feat(aiproxy): add aiproxy (#25003)

build/aiproxy/vars

@@ -0,0 +1 @@
+DESCRIPTION="Yunion Cloud AI Proxy Service"

build/docker/Dockerfile.aiproxy

@@ -0,0 +1,3 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v3.22.2-0
+
+ADD ./_output/alpine-build/bin/aiproxy /opt/yunion/bin/aiproxy

cmd/aiproxy/main.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/aiproxy/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
+
+func main() {
+	defer atexit.Handle()
+
+	service.StartService()
+}

cmd/climc/main.go

@@ -17,6 +17,7 @@ package main
 import (
 	"yunion.io/x/onecloud/cmd/climc/entry"
 	_ "yunion.io/x/onecloud/cmd/climc/shell"
+	_ "yunion.io/x/onecloud/cmd/climc/shell/aiproxy"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/ansible"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/apimap"
 	_ "yunion.io/x/onecloud/cmd/climc/shell/cloudevent"

cmd/climc/shell/aiproxy/ai_key.go

@@ -0,0 +1,31 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiKeys)
+	cmd.Create(new(apoptions.AiKeyCreateOptions))
+	cmd.List(new(apoptions.AiKeyListOptions))
+	cmd.Show(new(apoptions.AiKeyShowOptions))
+	cmd.Update(new(apoptions.AiKeyUpdateOptions))
+	cmd.Delete(new(apoptions.AiKeyDeleteOptions))
+	registerEnableDisable(cmd)
+}

cmd/climc/shell/aiproxy/ai_model.go

@@ -0,0 +1,31 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiModels)
+	cmd.Create(new(apoptions.AiModelCreateOptions))
+	cmd.List(new(apoptions.AiModelListOptions))
+	cmd.Show(new(apoptions.AiModelShowOptions))
+	cmd.Update(new(apoptions.AiModelUpdateOptions))
+	cmd.Delete(new(apoptions.AiModelDeleteOptions))
+	registerEnableDisable(cmd)
+}

cmd/climc/shell/aiproxy/ai_provider.go

@@ -0,0 +1,31 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiProviders)
+	cmd.Create(new(apoptions.AiProviderCreateOptions))
+	cmd.List(new(apoptions.AiProviderListOptions))
+	cmd.Show(new(apoptions.AiProviderShowOptions))
+	cmd.Update(new(apoptions.AiProviderUpdateOptions))
+	cmd.Delete(new(apoptions.AiProviderDeleteOptions))
+	registerEnableDisable(cmd)
+}

cmd/climc/shell/aiproxy/ai_proxy_node.go

@@ -0,0 +1,32 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiProxyNodes)
+	cmd.Create(new(apoptions.AiProxyNodeCreateOptions))
+	cmd.List(new(apoptions.AiProxyNodeListOptions))
+	cmd.Show(new(apoptions.AiProxyNodeShowOptions))
+	cmd.Update(new(apoptions.AiProxyNodeUpdateOptions))
+	cmd.Delete(new(apoptions.AiProxyNodeDeleteOptions))
+	registerEnableDisable(cmd)
+	cmd.PerformClass("register", new(apoptions.AiProxyNodeRegisterOptions))
+}

cmd/climc/shell/aiproxy/ai_routing.go

@@ -0,0 +1,32 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiRoutings)
+	cmd.Create(new(apoptions.AiRoutingCreateOptions))
+	cmd.List(new(apoptions.AiRoutingListOptions))
+	cmd.Show(new(apoptions.AiRoutingShowOptions))
+	cmd.Update(new(apoptions.AiRoutingUpdateOptions))
+	cmd.Delete(new(apoptions.AiRoutingDeleteOptions))
+	cmd.Perform("set-models", new(apoptions.AiRoutingSetModelsOptions))
+	registerEnableDisable(cmd)
+}

cmd/climc/shell/aiproxy/ai_routing_model.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiRoutingModels)
+	cmd.Create(new(apoptions.AiRoutingModelCreateOptions))
+	cmd.List(new(apoptions.AiRoutingModelListOptions))
+	cmd.Show(new(apoptions.AiRoutingModelShowOptions))
+	cmd.Update(new(apoptions.AiRoutingModelUpdateOptions))
+	cmd.Delete(new(apoptions.AiRoutingModelDeleteOptions))
+}

cmd/climc/shell/aiproxy/ai_virtual_key.go

@@ -0,0 +1,31 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+	apoptions "yunion.io/x/onecloud/pkg/mcclient/options/aiproxy"
+)
+
+func init() {
+	cmd := shell.NewResourceCmd(&apmodules.AiVirtualKeys)
+	cmd.Create(new(apoptions.AiVirtualKeyCreateOptions))
+	cmd.List(new(apoptions.AiVirtualKeyListOptions))
+	cmd.Show(new(apoptions.AiVirtualKeyShowOptions))
+	cmd.Update(new(apoptions.AiVirtualKeyUpdateOptions))
+	cmd.Delete(new(apoptions.AiVirtualKeyDeleteOptions))
+	registerEnableDisable(cmd)
+}

cmd/climc/shell/aiproxy/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package aiproxy registers climc commands for the aiproxy service.
+package aiproxy // import "yunion.io/x/onecloud/cmd/climc/shell/aiproxy"

cmd/climc/shell/aiproxy/enabled.go

@@ -0,0 +1,25 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	baseoptions "yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func registerEnableDisable(cmd *shell.ResourceCmd) {
+	cmd.Perform("enable", new(baseoptions.BaseIdOptions))
+	cmd.Perform("disable", new(baseoptions.BaseIdOptions))
+}

docs/aiproxy/functional-test-climc-mimo.md

@@ -0,0 +1,114 @@
+# aiproxy 功能测试（climc + 小米 MiMo）
+
+本文用 **climc** 配置 aiproxy 资源，并通过 **curl** 调用 `POST /v1/chat/completions` 验证 **xiaomi** catalog（`api.xiaomimimo.com`）。
+
+> **安全**：请勿将 MiMo API Key 写入脚本或提交到 Git。使用环境变量 `MIMO_API_KEY`。若 Key 曾泄露，请到小米开放平台轮换。
+
+通义千问（DashScope）测试见 [functional-test-climc.md](./functional-test-climc.md)。
+
+## 前置条件
+
+| 项 | 说明 |
+|----|------|
+| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` public endpoint |
+| 数据库 | 主节点已执行 `InitDB`，catalog 含 `xiaomi` 及 `mimo-*` 模型 |
+| 客户端 | 已 `source /etc/yunion/rcadmin`，`climc` 可用 |
+| 工具 | `jq` |
+| 网络 | aiproxy 节点能访问 `https://api.xiaomimimo.com` |
+
+```bash
+source /etc/yunion/rcadmin
+export CLIMC_OUTPUT_FORMAT=json
+bash scripts/test/aiproxy/aiproxy-functional-test.sh
+# 交互菜单中选择 xiaomi，输入 MiMo API Key
+```
+
+或快捷入口（默认选中 `xiaomi`）：
+
+```bash
+export MIMO_API_KEY='你的 MiMo API Key'
+bash scripts/test/aiproxy/aiproxy-functional-test-mimo.sh
+```
+
+预置模型：`export AIPROXY_FT_PROVIDER=xiaomi AIPROXY_FT_MODEL=mimo-v2.5-pro`
+
+按 provider 自动命名的资源（可覆盖 `AIPROXY_FT_KEY_NAME` 等）：
+
+| 变量 | 默认（xiaomi） |
+|------|----------------|
+| `AIPROXY_FT_KEY_NAME` | `aiproxy-ft-xiaomi` |
+| `AIPROXY_FT_VK_NAME` | `aiproxy-ft-xiaomi-vk` |
+| `AIPROXY_FT_ROUTING_NAME` | `aiproxy-ft-xiaomi-routing` |
+| `AIPROXY_FT_MODEL` | 交互默认 `mimo-v2-flash` |
+
+## 手动步骤摘要
+
+### Catalog
+
+```bash
+climc ai-provider-show xiaomi
+climc ai-model-show xiaomi-mimo-v2-flash
+```
+
+`config.base_url` 应为 `https://api.xiaomimimo.com`。
+
+### ai_key
+
+```bash
+climc ai-key-create mimo-ft \
+  --ai-provider-id xiaomi \
+  --secret "${MIMO_API_KEY}" \
+  --weight 10 \
+  --enabled
+```
+
+### 路由与 chat
+
+```bash
+climc ai-virtual-key-create aiproxy-mimo-ft-vk
+climc ai-routing-create aiproxy-mimo-ft-routing \
+  --priority 10 \
+  --models '[{"ai_provider_id":"xiaomi","ai_model_id":"mimo-v2-flash","priority":1}]'
+
+AIPROXY_URL="$(climc endpoint-list --service aiproxy --interface public --limit 1 \
+  --output-format json | jq -r '.data[0].url')"
+VK="$(climc ai-virtual-key-show aiproxy-mimo-ft-vk --output-format json | jq -r '.virtual_key')"
+
+curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \
+  -H "Authorization: Bearer ${VK}" \
+  -H "Content-Type: application/json" \
+  -d '{"model":"mimo-v2-flash","messages":[{"role":"user","content":"hi"}],"max_tokens":64}' | jq .
+```
+
+### 流式
+
+`scripts/test/aiproxy/aiproxy-functional-test-mimo.sh` 在非流式通过后默认执行 step 7（`stream: true`）。跳过：`export AIPROXY_FT_SKIP_STREAM=1`。
+
+```bash
+curl -k -sS -N -o /tmp/aiproxy-mimo-stream.sse \
+  "${AIPROXY_URL%/}/v1/chat/completions" \
+  -H "Authorization: Bearer ${VK}" \
+  -H "Content-Type: application/json" \
+  -d '{"model":"mimo-v2-flash","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}'
+```
+
+catalog 中其它模型：`mimo-v2.5-pro`、`mimo-v2-pro`、`mimo-v2.5`、`mimo-v2-omni`（id 形如 `xiaomi-mimo-v2.5-pro`）。
+
+## 常见问题
+
+**上游 401**  
+检查 `MIMO_API_KEY` 是否有效；确认 `ai_key` 已 `--enabled` 且 `ai_provider_id=xiaomi`。
+
+**`no ai_routing matched`**  
+virtual key 与 routing 须在同一 climc 项目下创建。
+
+**与 DashScope 脚本冲突**  
+MiMo 脚本使用独立的 vk/routing/key 名称；勿与 `aiproxy-ft-vk` 混用同一 routing 的 model 列表。
+
+## 清理
+
+```bash
+climc ai-routing-delete aiproxy-mimo-ft-routing
+climc ai-virtual-key-delete aiproxy-mimo-ft-vk
+climc ai-key-delete mimo-ft
+```

docs/aiproxy/functional-test-climc.md

@@ -0,0 +1,303 @@
+# aiproxy 功能测试（climc + 通义千问 DashScope）
+
+本文用 **climc** 配置 aiproxy 资源，并用 **curl** 调用 OpenAI 兼容接口 `POST /v1/chat/completions` 做端到端验证。
+
+> **安全**：请勿把 DashScope API Key 写进脚本、文档或提交到 Git。在 shell 里用环境变量 `DASHSCOPE_API_KEY` 传入。若 Key 曾在聊天/工单中泄露，请到阿里云控制台轮换。
+
+## 前置条件
+
+| 项 | 说明 |
+|----|------|
+| 服务 | aiproxy **主节点**已部署，Keystone 中已注册 `aiproxy` 服务及 public endpoint |
+| 数据库 | 主节点已执行 `InitDB`，catalog 中已有 `aliyun` provider 及 `qwen-*` 模型（首次启动 master 会自动 seed） |
+| 客户端 | 已 `source /etc/yunion/rcadmin`（或等价 rc 文件），`climc` 能正常 list |
+| 工具 | `jq`（脚本与下文 curl 示例用于解析 JSON） |
+| 网络 | aiproxy 节点能访问 `https://dashscope.aliyuncs.com` |
+
+### 一键脚本（交互式，推荐）
+
+从 catalog 选择 **模型提供商** 与 **model_key**，终端输入 API Key（或使用环境变量跳过输入），自动完成非流式 + 流式 chat：
+
+```bash
+source /etc/yunion/rcadmin
+export CLIMC_OUTPUT_FORMAT=json
+bash scripts/test/aiproxy/aiproxy-functional-test.sh
+```
+
+也可预置后减少交互（仍会选择模型、是否流式，除非全部用环境变量）：
+
+```bash
+export DASHSCOPE_API_KEY='你的 DashScope API Key'   # 或 AIPROXY_FT_API_KEY
+export AIPROXY_FT_PROVIDER=aliyun
+export AIPROXY_FT_MODEL=qwen-turbo
+bash scripts/test/aiproxy/aiproxy-functional-test.sh
+```
+
+通义快捷入口：`bash scripts/test/aiproxy/aiproxy-functional-test-qwen.sh`（默认 `aliyun`）。  
+小米 MiMo 见 [functional-test-climc-mimo.md](./functional-test-climc-mimo.md)（`aiproxy-functional-test-mimo.sh`）。
+
+非交互（CI）：
+
+```bash
+export AIPROXY_FT_NONINTERACTIVE=1
+export AIPROXY_FT_PROVIDER=aliyun
+export AIPROXY_FT_MODEL=qwen-turbo
+export AIPROXY_FT_API_KEY='...'
+export AIPROXY_FT_SKIP_STREAM=1   # 可选，跳过流式
+bash scripts/test/aiproxy/aiproxy-functional-test.sh
+```
+
+| 环境变量 | 说明 |
+|----------|------|
+| `AIPROXY_FT_PROVIDER` | `provider_key`（如 `aliyun`、`xiaomi`） |
+| `AIPROXY_FT_MODEL` | `model_key`（如 `qwen-turbo`） |
+| `AIPROXY_FT_API_KEY` | 上游 API Key（通用） |
+| `DASHSCOPE_API_KEY` / `MIMO_API_KEY` | 按提供商兼容的旧变量名 |
+| `AIPROXY_FT_SKIP_STREAM` | `1` 跳过流式；`0` 强制流式 |
+| `AIPROXY_URL` | 留空则从 endpoint-list 解析 |
+
+## 测试流程概览
+
+```mermaid
+flowchart LR
+  VK[ai_virtual_key] --> RT[ai_routing]
+  RT --> RM[ai_routing_model]
+  RM --> P[ai_provider aliyun]
+  RM --> M[ai_model qwen-turbo]
+  P --> K[ai_key secret]
+  K --> DS[DashScope API]
+```
+
+## 0a. ai_provider 创建测试脚本
+
+自定义 provider（非 catalog seed）创建与校验：
+
+```bash
+source /etc/yunion/rcadmin
+bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh
+```
+
+交互输入：资源名、`provider_key`、`base_url`、是否 `--enabled`。非交互示例：
+
+```bash
+export AIPROXY_PROVIDER_FT_NONINTERACTIVE=1
+export AIPROXY_PROVIDER_FT_NAME=my-vllm
+export AIPROXY_PROVIDER_FT_PROVIDER_KEY=my-vllm
+export AIPROXY_PROVIDER_FT_BASE_URL=http://127.0.0.1:8000/v1
+bash scripts/test/aiproxy/aiproxy-ai-provider-create-test.sh
+```
+
+`provider_key` 须全局唯一；与 InitDB catalog（如 `aliyun`）重复会失败。完整 config 可用 `AIPROXY_PROVIDER_FT_CONFIG='{"base_url":"..."}'`。
+
+## 0. ai_proxy_node（多副本 / 路由绑定）
+
+列出 aiproxy 实例节点（InitDB 后默认有 `primary`）：
+
+```bash
+climc ai-proxy-node-list
+climc ai-proxy-node-show primary
+```
+
+注册 standby 节点（与进程内 `register` 心跳相同，一般由 standby 自动调用；手工测试可用）：
+
+```bash
+climc ai-proxy-node-register --address https://standby-host:30938 --hb-timeout 120
+```
+
+手工创建/更新节点（需具备写权限策略）：
+
+```bash
+climc ai-proxy-node-create standby-1 \
+  --address https://standby-host:30938 \
+  --domain aiproxy-standby.example.com \
+  --hb-timeout 120 \
+  --enabled
+
+climc ai-proxy-node-update primary --address https://primary-host:30938 --domain aiproxy.example.com
+climc ai-proxy-node-enable primary
+climc ai-proxy-node-disable <node-id>
+```
+
+将 `ai_routing` 绑定到指定节点（chat 须走该节点 public endpoint）：
+
+```bash
+climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary
+```
+
+## 1. 检查 Keystone endpoint
+
+```bash
+climc endpoint-list --service aiproxy --interface public
+```
+
+应能看到当前 region 的 public URL（脚本会取第一条用于 curl）。
+
+## 2. 检查 catalog（InitDB seed）
+
+```bash
+climc ai-provider-list --provider-key aliyun
+climc ai-provider-show aliyun
+climc ai-model-list --ai-provider-id aliyun --model-key qwen-turbo
+```
+
+确认 `provider_key=aliyun`，`config.base_url` 含 `https://dashscope.aliyuncs.com/compatible-mode`，且存在模型 `model_key=qwen-turbo`（catalog 固定 id / name：`aliyun-qwen-turbo`）。
+
+## 3. 注册上游 API Key（ai_key）
+
+将 DashScope Key 存为 `ai_key`，供 chat 时按 provider 加权选取：
+
+```bash
+climc ai-key-create qwen-dashscope-ft \
+  --ai-provider-id aliyun \
+  --secret "${DASHSCOPE_API_KEY}" \
+  --weight 10 \
+  --enabled
+```
+
+校验：
+
+```bash
+climc ai-key-list --ai-provider-id aliyun
+climc ai-key-show qwen-dashscope-ft
+```
+
+确认 `ai_provider_id` 为 `aliyun`，且 **`enabled=true`**（`ai_key` 默认 disabled，创建时需 `--enabled`；若已存在但被禁用，执行 `climc ai-key-enable qwen-dashscope-ft`）。若曾用错误参数创建过同名 key，可更新：
+
+```bash
+climc ai-key-update qwen-dashscope-ft \
+  --ai-provider-id aliyun \
+  --secret "${DASHSCOPE_API_KEY}" \
+  --weight 10
+climc ai-key-enable qwen-dashscope-ft
+```
+
+（`secret` 在 API 中通常不回显，仅用于上游调用。）
+
+## 4. 创建 Virtual Key（客户端鉴权）
+
+```bash
+climc ai-virtual-key-create aiproxy-ft-vk
+```
+
+记下返回的 `virtual_key`（形如 `sk-...`）。查看：
+
+```bash
+climc ai-virtual-key-list
+climc ai-virtual-key-show aiproxy-ft-vk
+```
+
+Virtual key 归属当前 climc 用户的 **项目**；后续 `ai_routing` 须在同一项目（或共享到该项目）下。
+
+## 5. 创建项目路由（ai_routing + models）
+
+将项目内请求 `model=qwen-turbo` 指到 catalog 的 aliyun/qwen-turbo。
+
+`models` 里 `ai_model_id` 使用 catalog 固定 id（与 name 相同，如 `aliyun-qwen-turbo`），或在指定 `ai_provider_id` 时也可填 **model_key**（如 `qwen-turbo`）：
+
+```bash
+climc ai-routing-create aiproxy-ft-routing \
+  --priority 10 \
+  --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-turbo","priority":1}]'
+```
+
+或手工指定 name：
+
+```bash
+climc ai-routing-create aiproxy-ft-routing \
+  --priority 10 \
+  --models '[{"ai_provider_id":"aliyun","ai_model_id":"aliyun-qwen-turbo","priority":1}]'
+```
+
+查看绑定模型：
+
+```bash
+climc ai-routing-show aiproxy-ft-routing
+```
+
+也可事后调整：
+
+```bash
+climc ai-routing-set-models aiproxy-ft-routing \
+  --models '[{"ai_provider_id":"aliyun","ai_model_id":"qwen-plus","priority":1}]'
+```
+
+可选：将规则绑定到指定 aiproxy 实例（多副本时）：
+
+```bash
+# 仅当需要固定到 primary 等节点时
+climc ai-routing-update aiproxy-ft-routing --ai-proxy-node-id primary
+```
+
+## 6. Chat  completions（curl）
+
+climc 暂无 chat 子命令，用 public endpoint + virtual key 调用（`-k` 跳过 TLS 证书校验，适用于自签或内网 HTTPS）：
+
+```bash
+AIPROXY_URL="${AIPROXY_URL:-$(climc endpoint-list --service aiproxy --interface public --limit 1 \
+  --output-format json | jq -r '.data[0].url // empty')}"
+
+VK="$(climc ai-virtual-key-show aiproxy-ft-vk --output-format json \
+  | jq -r '.virtual_key')"
+
+curl -k -sS "${AIPROXY_URL%/}/v1/chat/completions" \
+  -H "Authorization: Bearer ${VK}" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "qwen-turbo",
+    "messages": [{"role": "user", "content": "用一句话介绍通义千问"}],
+    "max_tokens": 128
+  }' | jq .
+```
+
+**期望**：HTTP 200，JSON 含 `choices[0].message.content` 及 `usage`。
+
+## 6b. 流式 Chat（curl / 脚本 step 7）
+
+一键脚本在步骤 6 非流式成功后，默认继续执行流式校验（聚合 `choices[0].delta.content`）。跳过流式：
+
+```bash
+export AIPROXY_FT_SKIP_STREAM=1
+```
+
+手动 curl（SSE，`data: [DONE]` 结束）：
+
+```bash
+curl -k -sS -N -o /tmp/aiproxy-ft-stream.sse \
+  "${AIPROXY_URL%/}/v1/chat/completions" \
+  -H "Authorization: Bearer ${VK}" \
+  -H "Content-Type: application/json" \
+  -d '{"model":"qwen-turbo","stream":true,"messages":[{"role":"user","content":"hi"}],"max_tokens":64}'
+```
+
+期望：HTTP 200，响应体含 `data: {...}` 行且至少一条 `delta.content` 非空；最后为 `data: [DONE]`。
+
+## 7. 负向用例（可选）
+
+| 场景 | 操作 | 期望 |
+|------|------|------|
+| 错误 virtual key | `Authorization: Bearer sk-invalid` | 4xx，virtual key 无效 |
+| 无路由 | `climc ai-routing-disable aiproxy-ft-routing` 或删除后再 chat | 404，无匹配 routing |
+| 禁用 virtual key | `climc ai-virtual-key-disable aiproxy-ft-vk` | 4xx |
+| provider 限制 | create vk 时 `--limits '{"allowed_ai_provider_ids":["openai"]}'` | 4xx，provider 不允许 |
+
+## 8. 清理（可选）
+
+```bash
+climc ai-routing-delete aiproxy-ft-routing
+climc ai-virtual-key-delete aiproxy-ft-vk
+climc ai-key-delete qwen-dashscope-ft
+```
+
+## 常见问题
+
+**`no ai_routing matched for virtual key project`**  
+Virtual key 的 `project_id` 与 routing 所在项目不一致，或 routing 未 `enabled`、未共享到该项目。用同一 `climc` 项目上下文创建两者。
+
+**`no api_key for ai_provider`**  
+未创建 `ai_key`，且 `ai_provider.config` 里也没有 `api_key`。按步骤 3 创建 `ai_key`。
+
+**DashScope 401/403**  
+检查 `DASHSCOPE_API_KEY` 是否有效、是否开通对应模型。
+
+**多副本 `ai_routing` 绑定其它节点**  
+若 routing 指定了 `ai_proxy_node_id`，须访问该节点的 public endpoint，或去掉绑定。

pkg/aiproxy/handlers/chat_completions.go

@@ -0,0 +1,368 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers"
+	"yunion.io/x/onecloud/pkg/aiproxy/upstream"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+const headerAiVirtualKey = "X-Ai-Virtual-Key"
+
+func extractVirtualKey(r *http.Request) string {
+	if v := strings.TrimSpace(r.Header.Get(headerAiVirtualKey)); v != "" {
+		return v
+	}
+	authz := strings.TrimSpace(r.Header.Get("Authorization"))
+	parts := strings.SplitN(authz, " ", 2)
+	if len(parts) == 2 && strings.EqualFold(parts[0], "Bearer") {
+		return strings.TrimSpace(parts[1])
+	}
+	return ""
+}
+
+func upstreamErrorStatusCode(uerr *upstream.Error) int {
+	if uerr == nil || uerr.StatusCode <= 0 {
+		return 0
+	}
+	return uerr.StatusCode
+}
+
+func writeUpstreamError(w http.ResponseWriter, uerr *upstream.Error) {
+	status := http.StatusBadGateway
+	if uerr != nil && uerr.StatusCode > 0 {
+		status = uerr.StatusCode
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	if uerr != nil && len(uerr.Body) > 0 {
+		_, _ = w.Write(uerr.Body)
+		return
+	}
+	msg := "upstream request failed"
+	if uerr != nil {
+		msg = uerr.Error()
+	}
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"error": map[string]interface{}{
+			"message": msg,
+		},
+	})
+}
+
+func flushIf(w http.ResponseWriter) {
+	if f, ok := w.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+// chatCompletionsHandler implements OpenAI-compatible POST /openai/v1/chat/completions.
+// Auth is the ai_virtual_key only (Authorization: Bearer <vk> or X-Ai-Virtual-Key).
+// Upstream is resolved: ai_virtual_key -> project ai_routing -> ai_routing_model -> ai_key (by catalog model_key).
+func chatCompletionsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		httperrors.InvalidInputError(ctx, w, "only POST is supported")
+		return
+	}
+
+	defer r.Body.Close()
+	raw, err := io.ReadAll(r.Body)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "read body: %v", err)
+		return
+	}
+
+	body, err := jsonutils.Parse(raw)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err)
+		return
+	}
+	dict, ok := body.(*jsonutils.JSONDict)
+	if !ok {
+		httperrors.InvalidInputError(ctx, w, "body must be a JSON object")
+		return
+	}
+
+	vk := extractVirtualKey(r)
+	userCred := auth.AdminCredential()
+	up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	var vkLim *api.SAiVirtualKeyLimits
+	if up.MaxTokensPerRequest > 0 {
+		vkLim = &api.SAiVirtualKeyLimits{
+			MaxTokensPerRequest: up.MaxTokensPerRequest,
+		}
+	}
+	if err := models.EnforceVirtualKeyMaxTokens(dict, vkLim); err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	isStream, _ := dict.Bool("stream")
+	prov := providers.Get(up.ProviderKey)
+	if _, err := prov.BuildUpstreamRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict, isStream); err != nil {
+		httperrors.InvalidInputError(ctx, w, "provider request: %v", err)
+		return
+	}
+	timeout := 120 * time.Second
+	if isStream {
+		timeout = 2 * time.Hour
+	}
+
+	if !isStream {
+		resp, uerr := chatCompletionWithKeyFailover(ctx, up, dict, isStream, timeout)
+		if uerr != nil {
+			writeUpstreamError(w, uerr)
+			return
+		}
+		body := resp.Body
+		if norm, nerr := prov.NormalizeResponse(body); nerr == nil && len(norm) > 0 {
+			body = norm
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(body)
+		return
+	}
+
+	ch, uerr := chatCompletionStreamWithKeyFailover(ctx, up, dict, isStream, prov, timeout)
+	if uerr != nil {
+		writeUpstreamError(w, uerr)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+	w.WriteHeader(http.StatusOK)
+	flushIf(w)
+
+	streamOK := true
+	for chunk := range ch {
+		if chunk.Done {
+			break
+		}
+		if len(chunk.Data) == 0 {
+			continue
+		}
+		if isUpstreamErrorChunk(chunk.Data) {
+			streamOK = false
+			models.RecordAiKeyFailure(up.AiKeyId, parseUpstreamErrorStatus(chunk.Data))
+			_, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data)
+			flushIf(w)
+			break
+		}
+		_, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data)
+		flushIf(w)
+	}
+	_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
+	flushIf(w)
+	if streamOK {
+		models.RecordAiKeySuccess(up.AiKeyId)
+	}
+}
+
+func isUpstreamErrorChunk(data []byte) bool {
+	var wrap struct {
+		Error interface{} `json:"error"`
+	}
+	return json.Unmarshal(data, &wrap) == nil && wrap.Error != nil
+}
+
+func parseUpstreamErrorStatus(data []byte) int {
+	var wrap struct {
+		Error struct {
+			Code interface{} `json:"code"`
+		} `json:"error"`
+	}
+	if json.Unmarshal(data, &wrap) != nil {
+		return 0
+	}
+	switch c := wrap.Error.Code.(type) {
+	case float64:
+		return int(c)
+	case int:
+		return c
+	default:
+		return 0
+	}
+}
+
+func chatCompletionWithKeyFailover(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	dict *jsonutils.JSONDict,
+	stream bool,
+	timeout time.Duration,
+) (*upstream.Response, *upstream.Error) {
+	tried := make(map[string]bool)
+	if up.AiKeyId != "" {
+		tried[up.AiKeyId] = true
+	}
+	var last *upstream.Error
+	for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
+		upReq, err := buildProviderUpstream(up, dict, stream)
+		if err != nil {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
+		}
+		reqCtx, cancel := context.WithTimeout(ctx, timeout)
+		resp, uerr := upstream.ChatCompletion(reqCtx, upReq)
+		cancel()
+		if uerr == nil {
+			models.RecordAiKeySuccess(up.AiKeyId)
+			return resp, nil
+		}
+		last = uerr
+		status := upstreamErrorStatusCode(uerr)
+		models.RecordAiKeyFailure(up.AiKeyId, status)
+		if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
+			break
+		}
+		if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
+			break
+		}
+		if up.AiKeyId != "" {
+			tried[up.AiKeyId] = true
+		}
+	}
+	return nil, last
+}
+
+func chatCompletionStreamWithKeyFailover(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	dict *jsonutils.JSONDict,
+	stream bool,
+	prov providers.Provider,
+	timeout time.Duration,
+) (<-chan upstream.StreamChunk, *upstream.Error) {
+	tried := make(map[string]bool)
+	if up.AiKeyId != "" {
+		tried[up.AiKeyId] = true
+	}
+	var last *upstream.Error
+	for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
+		upReq, err := buildProviderUpstream(up, dict, stream)
+		if err != nil {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
+		}
+		reqCtx, cancel := context.WithTimeout(ctx, timeout)
+		ch, uerr := providerStreamChunks(reqCtx, up, upReq, prov)
+		cancel()
+		if uerr == nil {
+			return ch, nil
+		}
+		last = uerr
+		status := upstreamErrorStatusCode(uerr)
+		models.RecordAiKeyFailure(up.AiKeyId, status)
+		if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
+			break
+		}
+		if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
+			break
+		}
+		if up.AiKeyId != "" {
+			tried[up.AiKeyId] = true
+		}
+	}
+	return nil, last
+}
+
+func buildProviderUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict, isStream bool) (*upstream.Request, error) {
+	prov := providers.Get(up.ProviderKey)
+	httpReq, err := prov.BuildUpstreamRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict, isStream)
+	if err != nil {
+		return nil, err
+	}
+	return providers.ToUpstreamRequest(httpReq, up.APIKey), nil
+}
+
+func providerStreamChunks(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	upReq *upstream.Request,
+	prov providers.Provider,
+) (<-chan upstream.StreamChunk, *upstream.Error) {
+	chatCtx := &providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}
+	if providers.OpenAIStreamPassthrough(prov, chatCtx) {
+		return upstream.ChatCompletionStream(ctx, upReq)
+	}
+	rawCh, uerr := upstream.ChatCompletionStreamRaw(ctx, upReq)
+	if uerr != nil {
+		return nil, uerr
+	}
+	out := make(chan upstream.StreamChunk, 16)
+	go func() {
+		defer close(out)
+		state := &providers.StreamState{Model: up.UpstreamModel}
+		for evt := range rawCh {
+			chunks, err := prov.ConvertStreamEvent(evt.Event, evt.Data, state)
+			if err != nil {
+				msg, _ := json.Marshal(map[string]interface{}{
+					"error": map[string]interface{}{"message": err.Error()},
+				})
+				out <- upstream.StreamChunk{Data: msg}
+				return
+			}
+			for _, c := range chunks {
+				if len(c.Data) > 0 {
+					out <- upstream.StreamChunk{Data: c.Data}
+				}
+				if c.Done {
+					out <- upstream.StreamChunk{Done: true}
+					return
+				}
+			}
+		}
+	}()
+	return out, nil
+}

pkg/aiproxy/handlers/completions.go

@@ -0,0 +1,258 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers"
+	"yunion.io/x/onecloud/pkg/aiproxy/upstream"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+// completionsHandler implements OpenAI-compatible POST /openai/v1/completions.
+func completionsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		httperrors.InvalidInputError(ctx, w, "only POST is supported")
+		return
+	}
+
+	defer r.Body.Close()
+	raw, err := io.ReadAll(r.Body)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "read body: %v", err)
+		return
+	}
+
+	body, err := jsonutils.Parse(raw)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err)
+		return
+	}
+	dict, ok := body.(*jsonutils.JSONDict)
+	if !ok {
+		httperrors.InvalidInputError(ctx, w, "body must be a JSON object")
+		return
+	}
+
+	vk := extractVirtualKey(r)
+	userCred := auth.AdminCredential()
+	up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	var vkLim *api.SAiVirtualKeyLimits
+	if up.MaxTokensPerRequest > 0 {
+		vkLim = &api.SAiVirtualKeyLimits{
+			MaxTokensPerRequest: up.MaxTokensPerRequest,
+		}
+	}
+	if err := models.EnforceVirtualKeyMaxTokens(dict, vkLim); err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	compProv, err := providers.GetCompletions(up.ProviderKey)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "%v", err)
+		return
+	}
+
+	isStream, _ := dict.Bool("stream")
+	if _, err := compProv.BuildCompletionsRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict, isStream); err != nil {
+		httperrors.InvalidInputError(ctx, w, "provider request: %v", err)
+		return
+	}
+
+	timeout := 120 * time.Second
+	if isStream {
+		timeout = 2 * time.Hour
+	}
+
+	if !isStream {
+		resp, uerr := completionsWithKeyFailover(ctx, up, dict, isStream, timeout)
+		if uerr != nil {
+			writeUpstreamError(w, uerr)
+			return
+		}
+		out := resp.Body
+		if norm, nerr := compProv.NormalizeCompletionsResponse(out); nerr == nil && len(norm) > 0 {
+			out = norm
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write(out)
+		return
+	}
+
+	ch, uerr := completionsStreamWithKeyFailover(ctx, up, dict, isStream, compProv, timeout)
+	if uerr != nil {
+		writeUpstreamError(w, uerr)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/event-stream")
+	w.Header().Set("Cache-Control", "no-cache")
+	w.Header().Set("Connection", "keep-alive")
+	w.WriteHeader(http.StatusOK)
+	flushIf(w)
+
+	streamOK := true
+	for chunk := range ch {
+		if chunk.Done {
+			break
+		}
+		if len(chunk.Data) == 0 {
+			continue
+		}
+		if isUpstreamErrorChunk(chunk.Data) {
+			streamOK = false
+			models.RecordAiKeyFailure(up.AiKeyId, parseUpstreamErrorStatus(chunk.Data))
+			_, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data)
+			flushIf(w)
+			break
+		}
+		_, _ = fmt.Fprintf(w, "data: %s\n\n", chunk.Data)
+		flushIf(w)
+	}
+	_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
+	flushIf(w)
+	if streamOK {
+		models.RecordAiKeySuccess(up.AiKeyId)
+	}
+}
+
+func completionsWithKeyFailover(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	dict *jsonutils.JSONDict,
+	stream bool,
+	timeout time.Duration,
+) (*upstream.Response, *upstream.Error) {
+	tried := make(map[string]bool)
+	if up.AiKeyId != "" {
+		tried[up.AiKeyId] = true
+	}
+	var last *upstream.Error
+	for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
+		upReq, err := buildCompletionsUpstream(up, dict, stream)
+		if err != nil {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
+		}
+		reqCtx, cancel := context.WithTimeout(ctx, timeout)
+		resp, uerr := upstream.ChatCompletion(reqCtx, upReq)
+		cancel()
+		if uerr == nil {
+			models.RecordAiKeySuccess(up.AiKeyId)
+			return resp, nil
+		}
+		last = uerr
+		status := upstreamErrorStatusCode(uerr)
+		models.RecordAiKeyFailure(up.AiKeyId, status)
+		if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
+			break
+		}
+		if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
+			break
+		}
+		if up.AiKeyId != "" {
+			tried[up.AiKeyId] = true
+		}
+	}
+	return nil, last
+}
+
+func completionsStreamWithKeyFailover(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	dict *jsonutils.JSONDict,
+	stream bool,
+	compProv providers.CompletionsProvider,
+	timeout time.Duration,
+) (<-chan upstream.StreamChunk, *upstream.Error) {
+	tried := make(map[string]bool)
+	if up.AiKeyId != "" {
+		tried[up.AiKeyId] = true
+	}
+	var last *upstream.Error
+	for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
+		upReq, err := buildCompletionsUpstream(up, dict, stream)
+		if err != nil {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
+		}
+		reqCtx, cancel := context.WithTimeout(ctx, timeout)
+		var ch <-chan upstream.StreamChunk
+		var uerr *upstream.Error
+		if compProv.OpenAICompletionsStreamPassthrough() {
+			ch, uerr = upstream.ChatCompletionStream(reqCtx, upReq)
+		} else {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: "streaming completions not supported for provider"}
+		}
+		cancel()
+		if uerr == nil {
+			return ch, nil
+		}
+		last = uerr
+		status := upstreamErrorStatusCode(uerr)
+		models.RecordAiKeyFailure(up.AiKeyId, status)
+		if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
+			break
+		}
+		if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
+			break
+		}
+		if up.AiKeyId != "" {
+			tried[up.AiKeyId] = true
+		}
+	}
+	return nil, last
+}
+
+func buildCompletionsUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict, isStream bool) (*upstream.Request, error) {
+	compProv, err := providers.GetCompletions(up.ProviderKey)
+	if err != nil {
+		return nil, err
+	}
+	httpReq, err := compProv.BuildCompletionsRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict, isStream)
+	if err != nil {
+		return nil, err
+	}
+	return providers.ToUpstreamRequest(httpReq, up.APIKey), nil
+}

pkg/aiproxy/handlers/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers // import "yunion.io/x/onecloud/pkg/aiproxy/handlers"

pkg/aiproxy/handlers/embeddings.go

@@ -0,0 +1,146 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers"
+	"yunion.io/x/onecloud/pkg/aiproxy/upstream"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+// embeddingsHandler implements OpenAI-compatible POST /openai/v1/embeddings.
+func embeddingsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		httperrors.InvalidInputError(ctx, w, "only POST is supported")
+		return
+	}
+
+	defer r.Body.Close()
+	raw, err := io.ReadAll(r.Body)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "read body: %v", err)
+		return
+	}
+
+	body, err := jsonutils.Parse(raw)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err)
+		return
+	}
+	dict, ok := body.(*jsonutils.JSONDict)
+	if !ok {
+		httperrors.InvalidInputError(ctx, w, "body must be a JSON object")
+		return
+	}
+
+	vk := extractVirtualKey(r)
+	userCred := auth.AdminCredential()
+	up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	embProv := providers.GetEmbeddings(up.ProviderKey)
+	if _, err := embProv.BuildEmbeddingsRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict); err != nil {
+		httperrors.InvalidInputError(ctx, w, "provider request: %v", err)
+		return
+	}
+
+	resp, uerr := embeddingsWithKeyFailover(ctx, up, dict, 60*time.Second)
+	if uerr != nil {
+		writeUpstreamError(w, uerr)
+		return
+	}
+	out := resp.Body
+	if norm, nerr := embProv.NormalizeEmbeddingsResponse(out); nerr == nil && len(norm) > 0 {
+		out = norm
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write(out)
+}
+
+func embeddingsWithKeyFailover(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	dict *jsonutils.JSONDict,
+	timeout time.Duration,
+) (*upstream.Response, *upstream.Error) {
+	tried := make(map[string]bool)
+	if up.AiKeyId != "" {
+		tried[up.AiKeyId] = true
+	}
+	var last *upstream.Error
+	for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
+		upReq, err := buildEmbeddingsUpstream(up, dict)
+		if err != nil {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
+		}
+		reqCtx, cancel := context.WithTimeout(ctx, timeout)
+		resp, uerr := upstream.ChatCompletion(reqCtx, upReq)
+		cancel()
+		if uerr == nil {
+			models.RecordAiKeySuccess(up.AiKeyId)
+			return resp, nil
+		}
+		last = uerr
+		status := upstreamErrorStatusCode(uerr)
+		models.RecordAiKeyFailure(up.AiKeyId, status)
+		if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
+			break
+		}
+		if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
+			break
+		}
+		if up.AiKeyId != "" {
+			tried[up.AiKeyId] = true
+		}
+	}
+	return nil, last
+}
+
+func buildEmbeddingsUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict) (*upstream.Request, error) {
+	embProv := providers.GetEmbeddings(up.ProviderKey)
+	httpReq, err := embProv.BuildEmbeddingsRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict)
+	if err != nil {
+		return nil, err
+	}
+	return providers.ToUpstreamRequest(httpReq, up.APIKey), nil
+}

pkg/aiproxy/handlers/handlers.go

@@ -0,0 +1,75 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/options"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+)
+
+const openaiCompatAPIPrefix = "/ai/openai/v1"
+
+func InitHandlers(app *appsrv.Application, isSlave bool) {
+	db.InitAllManagers()
+	db.RegistUserCredCacheUpdater()
+
+	app_common.ExportOptionsHandler(app, &options.Options)
+
+	taskman.AddTaskHandler("", app, isSlave)
+
+	db.AddScopeResourceCountHandler("", app)
+
+	app.AddHandler2("POST", openaiCompatAPIPrefix+"/chat/completions", chatCompletionsHandler, nil, "aiproxy_openai_v1_chat_completions", nil)
+	app.AddHandler2("POST", openaiCompatAPIPrefix+"/completions", completionsHandler, nil, "aiproxy_openai_v1_completions", nil)
+	app.AddHandler2("POST", openaiCompatAPIPrefix+"/embeddings", embeddingsHandler, nil, "aiproxy_openai_v1_embeddings", nil)
+	app.AddHandler2("POST", openaiCompatAPIPrefix+"/images/generations", imagesGenerationsHandler, nil, "aiproxy_openai_v1_images_generations", nil)
+	app.AddHandler2("GET", openaiCompatAPIPrefix+"/models", modelsHandler, nil, "aiproxy_openai_v1_models", nil)
+	app.AddHandler2("GET", openaiCompatAPIPrefix+"/models/<model>", modelRetrieveHandler, nil, "aiproxy_openai_v1_models_retrieve", nil)
+
+	for _, manager := range []db.IModelManager{
+		taskman.TaskManager,
+		taskman.SubTaskManager,
+		taskman.TaskObjectManager,
+		taskman.ArchivedTaskManager,
+
+		db.SharedResourceManager,
+		db.UserCacheManager,
+		db.TenantCacheManager,
+	} {
+		db.RegisterModelManager(manager)
+	}
+
+	for _, manager := range []db.IModelManager{
+		db.OpsLog,
+		db.Metadata,
+
+		models.AiProviderManager,
+		models.AiModelManager,
+		models.AiKeyManager,
+		models.AiVirtualKeyManager,
+		models.AiRoutingManager,
+		models.AiRoutingModelManager,
+		models.AiProxyNodeManager,
+	} {
+		db.RegisterModelManager(manager)
+		handler := db.NewModelHandler(manager)
+		dispatcher.AddModelDispatcher("", app, handler, isSlave)
+	}
+}

pkg/aiproxy/handlers/images_generations.go

@@ -0,0 +1,146 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers"
+	"yunion.io/x/onecloud/pkg/aiproxy/upstream"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+// imagesGenerationsHandler implements OpenAI-compatible POST /openai/v1/images/generations.
+func imagesGenerationsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		httperrors.InvalidInputError(ctx, w, "only POST is supported")
+		return
+	}
+
+	defer r.Body.Close()
+	raw, err := io.ReadAll(r.Body)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "read body: %v", err)
+		return
+	}
+
+	body, err := jsonutils.Parse(raw)
+	if err != nil {
+		httperrors.InvalidInputError(ctx, w, "invalid JSON body: %v", err)
+		return
+	}
+	dict, ok := body.(*jsonutils.JSONDict)
+	if !ok {
+		httperrors.InvalidInputError(ctx, w, "body must be a JSON object")
+		return
+	}
+
+	vk := extractVirtualKey(r)
+	userCred := auth.AdminCredential()
+	up, err := models.ResolveChatUpstream(ctx, userCred, vk, dict)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	if err := models.TakeVirtualKeyRequestsPerMinute(up.VirtualKeyId, up.RequestsPerMinute); err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+
+	imgProv := providers.GetImages(up.ProviderKey)
+	if _, err := imgProv.BuildImagesGenerationsRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict); err != nil {
+		httperrors.InvalidInputError(ctx, w, "provider request: %v", err)
+		return
+	}
+
+	resp, uerr := imagesGenerationsWithKeyFailover(ctx, up, dict, 180*time.Second)
+	if uerr != nil {
+		writeUpstreamError(w, uerr)
+		return
+	}
+	out := resp.Body
+	if norm, nerr := imgProv.NormalizeImagesGenerationsResponse(out); nerr == nil && len(norm) > 0 {
+		out = norm
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write(out)
+}
+
+func imagesGenerationsWithKeyFailover(
+	ctx context.Context,
+	up *models.ChatUpstream,
+	dict *jsonutils.JSONDict,
+	timeout time.Duration,
+) (*upstream.Response, *upstream.Error) {
+	tried := make(map[string]bool)
+	if up.AiKeyId != "" {
+		tried[up.AiKeyId] = true
+	}
+	var last *upstream.Error
+	for attempt := 0; attempt < models.MaxAiKeyFailoverAttempts; attempt++ {
+		upReq, err := buildImagesGenerationsUpstream(up, dict)
+		if err != nil {
+			return nil, &upstream.Error{StatusCode: http.StatusBadRequest, Message: err.Error()}
+		}
+		reqCtx, cancel := context.WithTimeout(ctx, timeout)
+		resp, uerr := upstream.ChatCompletion(reqCtx, upReq)
+		cancel()
+		if uerr == nil {
+			models.RecordAiKeySuccess(up.AiKeyId)
+			return resp, nil
+		}
+		last = uerr
+		status := upstreamErrorStatusCode(uerr)
+		models.RecordAiKeyFailure(up.AiKeyId, status)
+		if up.AiKeyId == "" || !models.IsRetryableAiKeyUpstreamStatus(status) || attempt+1 >= models.MaxAiKeyFailoverAttempts {
+			break
+		}
+		if err := models.RepickUpstreamAPIKey(up, tried); err != nil {
+			break
+		}
+		if up.AiKeyId != "" {
+			tried[up.AiKeyId] = true
+		}
+	}
+	return nil, last
+}
+
+func buildImagesGenerationsUpstream(up *models.ChatUpstream, dict *jsonutils.JSONDict) (*upstream.Request, error) {
+	imgProv := providers.GetImages(up.ProviderKey)
+	httpReq, err := imgProv.BuildImagesGenerationsRequest(&providers.ChatContext{
+		ProviderKey:   up.ProviderKey,
+		BaseURL:       up.BaseURL,
+		APIKey:        up.APIKey,
+		UpstreamModel: up.UpstreamModel,
+	}, dict)
+	if err != nil {
+		return nil, err
+	}
+	return providers.ToUpstreamRequest(httpReq, up.APIKey), nil
+}

pkg/aiproxy/handlers/models.go

@@ -0,0 +1,84 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+// modelsHandler implements OpenAI-compatible GET /openai/v1/models.
+func modelsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		httperrors.InvalidInputError(ctx, w, "only GET is supported")
+		return
+	}
+	vk := extractVirtualKey(r)
+	userCred := auth.AdminCredential()
+	items, err := models.ListModelsForVirtualKey(ctx, userCred, vk)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	if items == nil {
+		items = []models.ModelsListEntry{}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"object": "list",
+		"data":   items,
+	})
+}
+
+// modelRetrieveHandler implements OpenAI-compatible GET /openai/v1/models/{model}.
+func modelRetrieveHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		httperrors.InvalidInputError(ctx, w, "only GET is supported")
+		return
+	}
+	params := appsrv.AppContextGetParams(ctx)
+	modelID := ""
+	if params != nil {
+		modelID = strings.TrimSpace(params.Params["<model>"])
+	}
+	if modelID == "" {
+		httperrors.InvalidInputError(ctx, w, "missing model id")
+		return
+	}
+	vk := extractVirtualKey(r)
+	userCred := auth.AdminCredential()
+	items, err := models.ListModelsForVirtualKey(ctx, userCred, vk)
+	if err != nil {
+		httperrors.GeneralServerError(ctx, w, err)
+		return
+	}
+	for _, item := range items {
+		if item.ID == modelID {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			_ = json.NewEncoder(w).Encode(item)
+			return
+		}
+	}
+	httperrors.NotFoundError(ctx, w, "model %q not found", modelID)
+}

pkg/aiproxy/models/ai_key_health.go

@@ -0,0 +1,135 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"sync"
+	"time"
+)
+
+const (
+	aiKeyHealthMaxScore       = 100
+	aiKeyHealthFailPenalty    = 25
+	aiKeyHealthSuccessBoost   = 10
+	aiKeyHealthCooldownAfter  = 3
+	aiKeyHealthCooldownPeriod = 60 * time.Second
+)
+
+type aiKeyHealthState struct {
+	score            int
+	consecutiveFails int
+	cooldownUntil    time.Time
+}
+
+var (
+	aiKeyHealthMu sync.RWMutex
+	aiKeyHealth   = map[string]*aiKeyHealthState{}
+)
+
+func getAiKeyHealth(keyId string) *aiKeyHealthState {
+	if keyId == "" {
+		return nil
+	}
+	aiKeyHealthMu.RLock()
+	st := aiKeyHealth[keyId]
+	aiKeyHealthMu.RUnlock()
+	if st != nil {
+		return st
+	}
+	aiKeyHealthMu.Lock()
+	defer aiKeyHealthMu.Unlock()
+	if st = aiKeyHealth[keyId]; st == nil {
+		st = &aiKeyHealthState{score: aiKeyHealthMaxScore}
+		aiKeyHealth[keyId] = st
+	}
+	return st
+}
+
+// dynamicAiKeyWeightMultiplier returns 0-100 applied to configured ai_key.weight (100 = full weight).
+func dynamicAiKeyWeightMultiplier(keyId string) int {
+	if keyId == "" {
+		return aiKeyHealthMaxScore
+	}
+	st := getAiKeyHealth(keyId)
+	now := time.Now()
+	aiKeyHealthMu.Lock()
+	defer aiKeyHealthMu.Unlock()
+	if !st.cooldownUntil.IsZero() && now.Before(st.cooldownUntil) {
+		return 0
+	}
+	if !st.cooldownUntil.IsZero() && !now.Before(st.cooldownUntil) {
+		st.cooldownUntil = time.Time{}
+		if st.score < aiKeyHealthMaxScore/2 {
+			st.score = aiKeyHealthMaxScore / 2
+		}
+	}
+	if st.score <= 0 {
+		return 0
+	}
+	if st.score > aiKeyHealthMaxScore {
+		return aiKeyHealthMaxScore
+	}
+	return st.score
+}
+
+// RecordAiKeySuccess boosts dynamic weight after a successful upstream call.
+func RecordAiKeySuccess(keyId string) {
+	if keyId == "" {
+		return
+	}
+	st := getAiKeyHealth(keyId)
+	aiKeyHealthMu.Lock()
+	defer aiKeyHealthMu.Unlock()
+	st.consecutiveFails = 0
+	st.cooldownUntil = time.Time{}
+	st.score += aiKeyHealthSuccessBoost
+	if st.score > aiKeyHealthMaxScore {
+		st.score = aiKeyHealthMaxScore
+	}
+}
+
+// RecordAiKeyFailure reduces dynamic weight when upstream rejects a key (429/401/5xx etc.).
+func RecordAiKeyFailure(keyId string, statusCode int) {
+	if keyId == "" || !IsRetryableAiKeyUpstreamStatus(statusCode) {
+		return
+	}
+	st := getAiKeyHealth(keyId)
+	aiKeyHealthMu.Lock()
+	defer aiKeyHealthMu.Unlock()
+	st.consecutiveFails++
+	st.score -= aiKeyHealthFailPenalty
+	if st.score < 0 {
+		st.score = 0
+	}
+	if st.consecutiveFails >= aiKeyHealthCooldownAfter {
+		st.cooldownUntil = time.Now().Add(aiKeyHealthCooldownPeriod)
+		st.score = 0
+	}
+}
+
+// IsRetryableAiKeyUpstreamStatus reports HTTP statuses that imply the api_key may be bad or overloaded.
+func IsRetryableAiKeyUpstreamStatus(statusCode int) bool {
+	if statusCode <= 0 {
+		return true
+	}
+	switch {
+	case statusCode == 401, statusCode == 403, statusCode == 429:
+		return true
+	case statusCode >= 500 && statusCode <= 599:
+		return true
+	default:
+		return false
+	}
+}

pkg/aiproxy/models/ai_key_resolve.go

@@ -0,0 +1,205 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"crypto/rand"
+	"math/big"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/httperrors"
+)
+
+func effectiveAiKeyRoutingWeight(r *api.SAiKeyRouting) int {
+	if r == nil || r.Weight <= 0 {
+		return 0
+	}
+	return r.Weight
+}
+
+// baseAiKeyWeight returns configured weight (column, else routing.weight, else 1).
+func baseAiKeyWeight(k *SAiKey) int {
+	if k == nil {
+		return 1
+	}
+	if k.Weight > 0 {
+		return k.Weight
+	}
+	if w := effectiveAiKeyRoutingWeight(k.Routing); w > 0 {
+		return w
+	}
+	return 1
+}
+
+// effectiveAiKeyWeight returns load-balance weight including dynamic penalty (差 key 降权).
+func effectiveAiKeyWeight(k *SAiKey) int {
+	base := baseAiKeyWeight(k)
+	if k == nil || base <= 0 {
+		return 0
+	}
+	mul := dynamicAiKeyWeightMultiplier(k.Id)
+	if mul <= 0 {
+		return 0
+	}
+	return base * mul / aiKeyHealthMaxScore
+}
+
+func aiKeyRoutingAcceptsModel(r *api.SAiKeyRouting, reqModel string) bool {
+	rm := strings.TrimSpace(reqModel)
+	if r == nil {
+		return true
+	}
+	for _, block := range r.BlockedModelKeys {
+		if modelPatternMatches(block, rm) {
+			return false
+		}
+	}
+	if len(r.AllowedModelKeys) > 0 {
+		ok := false
+		for _, allow := range r.AllowedModelKeys {
+			if modelPatternMatches(allow, rm) {
+				ok = true
+				break
+			}
+		}
+		if !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func pickWeightedAiKey(candidates []*SAiKey) *SAiKey {
+	if len(candidates) == 0 {
+		return nil
+	}
+	if len(candidates) == 1 {
+		return candidates[0]
+	}
+	total := 0
+	for _, k := range candidates {
+		total += effectiveAiKeyWeight(k)
+	}
+	if total <= 0 {
+		return candidates[0]
+	}
+	n, err := rand.Int(rand.Reader, big.NewInt(int64(total)))
+	if err != nil {
+		return candidates[0]
+	}
+	threshold := int(n.Int64()) + 1
+	acc := 0
+	for _, k := range candidates {
+		acc += effectiveAiKeyWeight(k)
+		if acc >= threshold {
+			return k
+		}
+	}
+	return candidates[len(candidates)-1]
+}
+
+type resolvedUpstreamAPIKey struct {
+	Secret   string
+	AiKeyId  string
+	FromRows bool
+}
+
+// MaxAiKeyFailoverAttempts is how many alternate ai_key rows to try per chat request.
+const MaxAiKeyFailoverAttempts = 8
+
+// resolveUpstreamAPIKey picks an ai_key (weighted + dynamic penalty) or provider.config api_key.
+func resolveUpstreamAPIKey(prov *SAiProvider, modelKey string) (*resolvedUpstreamAPIKey, error) {
+	return resolveUpstreamAPIKeyExcluding(prov, modelKey, nil)
+}
+
+func resolveUpstreamAPIKeyExcluding(prov *SAiProvider, modelKey string, exclude map[string]bool) (*resolvedUpstreamAPIKey, error) {
+	if prov == nil {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider is nil")
+	}
+	pid := strings.TrimSpace(prov.Id)
+	if pid == "" {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider id is empty")
+	}
+
+	keys := make([]SAiKey, 0, 32)
+	q := AiKeyManager.Query().Equals("ai_provider_id", pid).Equals("enabled", true)
+	err := q.All(&keys)
+	if err != nil {
+		return nil, errors.Wrap(err, "list ai_key for provider")
+	}
+
+	candidates := make([]*SAiKey, 0, len(keys))
+	hasSecretKey := false
+	for i := range keys {
+		k := &keys[i]
+		if strings.TrimSpace(k.Secret) == "" {
+			continue
+		}
+		hasSecretKey = true
+		if exclude != nil && exclude[k.Id] {
+			continue
+		}
+		if effectiveAiKeyWeight(k) <= 0 {
+			continue
+		}
+		if aiKeyRoutingAcceptsModel(k.Routing, modelKey) {
+			candidates = append(candidates, k)
+		}
+	}
+	if len(candidates) > 0 {
+		chosen := pickWeightedAiKey(candidates)
+		if chosen == nil {
+			return nil, errors.Wrap(httperrors.ErrInvalidStatus, "failed to pick ai_key")
+		}
+		return &resolvedUpstreamAPIKey{
+			Secret:   strings.TrimSpace(chosen.Secret),
+			AiKeyId:  chosen.Id,
+			FromRows: true,
+		}, nil
+	}
+	if hasSecretKey {
+		return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "no available ai_key for catalog model %q (check weight, cooldown, allowed_model_keys)", modelKey)
+	}
+	if prov.Config == nil {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty")
+	}
+	apiKey := strings.TrimSpace(prov.Config.ResolvedAPIKey())
+	if apiKey == "" {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "set api_key on ai_provider or add an enabled ai_key with secret for this provider")
+	}
+	return &resolvedUpstreamAPIKey{Secret: apiKey}, nil
+}
+
+// RepickUpstreamAPIKey selects another ai_key for the same provider/model, excluding already tried ids.
+func RepickUpstreamAPIKey(up *ChatUpstream, tried map[string]bool) error {
+	if up == nil || strings.TrimSpace(up.AiProviderId) == "" {
+		return errors.Wrap(httperrors.ErrInvalidStatus, "missing ai_provider on upstream")
+	}
+	provObj, err := AiProviderManager.FetchById(up.AiProviderId)
+	if err != nil {
+		return errors.Wrap(err, "fetch ai_provider for key repick")
+	}
+	prov := provObj.(*SAiProvider)
+	resolved, err := resolveUpstreamAPIKeyExcluding(prov, up.UpstreamModel, tried)
+	if err != nil {
+		return err
+	}
+	up.APIKey = resolved.Secret
+	up.AiKeyId = resolved.AiKeyId
+	return nil
+}

pkg/aiproxy/models/ai_keys.go

@@ -0,0 +1,165 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+// SAiKey stores a named upstream API key (or other secret material) for reuse by routing or providers.
+type SAiKey struct {
+	db.SEnabledStatusStandaloneResourceBase
+
+	// AiProviderId optionally associates this key with a catalog provider row.
+	AiProviderId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
+	// Secret holds raw key material; only privileged scopes should list it.
+	Secret string `width:"4096" charset:"ascii" nullable:"false" create:"required"`
+	// Weight is used for weighted random load balancing among matching keys (default 1).
+	Weight int `default:"1" nullable:"false" list:"user" create:"optional" update:"user"`
+	// Routing limits which request "model" values may use this key.
+	Routing *api.SAiKeyRouting `length:"medium" charset:"utf8" list:"user" create:"optional" update:"user"`
+}
+
+type SAiKeyManager struct {
+	db.SEnabledStatusStandaloneResourceBaseManager
+}
+
+var AiKeyManager *SAiKeyManager
+
+func init() {
+	AiKeyManager = &SAiKeyManager{
+		SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager(
+			SAiKey{},
+			"ai_keys_tbl",
+			"ai_key",
+			"ai_keys",
+		),
+	}
+	AiKeyManager.SetVirtualObject(AiKeyManager)
+}
+
+func (manager *SAiKeyManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiKeyListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter")
+	}
+	if id := strings.TrimSpace(query.AiProviderId); id != "" {
+		q = q.Equals("ai_provider_id", id)
+	}
+	return q, nil
+}
+
+func (manager *SAiKeyManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiKeyDetails {
+	rows := make([]api.AiKeyDetails, len(objs))
+	baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	providerIds := make([]string, len(objs))
+	for i := range objs {
+		rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i]
+		k := objs[i].(*SAiKey)
+		providerIds[i] = k.AiProviderId
+	}
+	providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds)
+	if err != nil {
+		log.Errorf("FetchIdNameMap2 ai_provider: %v", err)
+		return rows
+	}
+	for i := range rows {
+		rows[i].AiProviderName, _ = providerNames[providerIds[i]]
+	}
+	return rows
+}
+
+func (manager *SAiKeyManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiKeyCreateInput,
+) (api.AiKeyCreateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData")
+	}
+	if input.Weight < 0 {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "weight must be >= 0")
+	}
+	if input.Weight == 0 {
+		input.Weight = 1
+	}
+	if strings.TrimSpace(input.Secret) == "" {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "secret is required")
+	}
+	if strings.TrimSpace(input.AiProviderId) == "" {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "ai_provider_id is required")
+	}
+	prov, err := fetchEnabledAiProvider(ctx, userCred, input.AiProviderId)
+	if err != nil {
+		return input, err
+	}
+	input.AiProviderId = prov.Id
+	if input.Enabled == nil && input.Disabled == nil {
+		input.SetEnabled()
+	}
+	return input, nil
+}
+
+func (k *SAiKey) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input *api.AiKeyUpdateInput,
+) (*api.AiKeyUpdateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceBaseUpdateInput, err = k.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData")
+	}
+	if input.Weight < 0 {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "weight must be >= 0")
+	}
+	if pid := strings.TrimSpace(input.AiProviderId); pid != "" {
+		prov, err := fetchEnabledAiProvider(ctx, userCred, pid)
+		if err != nil {
+			return input, err
+		}
+		input.AiProviderId = prov.Id
+	}
+	return input, nil
+}

pkg/aiproxy/models/ai_models.go

@@ -0,0 +1,179 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+// SAiModel stores a model catalog row associated with an SAiProvider.
+type SAiModel struct {
+	db.SEnabledStatusStandaloneResourceBase
+
+	AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	// ModelKey is the model id sent to the upstream API (e.g. gpt-4o-mini, qwen-turbo).
+	ModelKey string `width:"256" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"`
+}
+
+type SAiModelManager struct {
+	db.SEnabledStatusStandaloneResourceBaseManager
+}
+
+var AiModelManager *SAiModelManager
+
+func init() {
+	AiModelManager = &SAiModelManager{
+		SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager(
+			SAiModel{},
+			"ai_models_tbl",
+			"ai_model",
+			"ai_models",
+		),
+	}
+	AiModelManager.SetVirtualObject(AiModelManager)
+}
+
+func (manager *SAiModelManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiModelListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter")
+	}
+	if id := strings.TrimSpace(query.AiProviderId); id != "" {
+		q = q.Equals("ai_provider_id", id)
+	}
+	if key := strings.TrimSpace(query.ModelKey); key != "" {
+		q = q.Equals("model_key", key)
+	}
+	return q, nil
+}
+
+func (manager *SAiModelManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiModelDetails {
+	rows := make([]api.AiModelDetails, len(objs))
+	baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	providerIds := make([]string, len(objs))
+	for i := range objs {
+		rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i]
+		m := objs[i].(*SAiModel)
+		providerIds[i] = m.AiProviderId
+	}
+	providerNames, err := db.FetchIdNameMap2(AiProviderManager, providerIds)
+	if err != nil {
+		log.Errorf("FetchIdNameMap2 ai_provider: %v", err)
+		return rows
+	}
+	for i := range rows {
+		rows[i].AiProviderName, _ = providerNames[providerIds[i]]
+	}
+	return rows
+}
+
+func (manager *SAiModelManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiModelCreateInput,
+) (api.AiModelCreateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData")
+	}
+
+	prov, err := fetchEnabledAiProvider(ctx, userCred, input.AiProviderId)
+	if err != nil {
+		return input, err
+	}
+	input.AiProviderId = prov.Id
+
+	mk, err := validateAiModelKey(input.ModelKey)
+	if err != nil {
+		return input, err
+	}
+	input.ModelKey = mk
+
+	if err := ensureAiModelKeyUniquePerProvider(ctx, prov.Id, mk, ""); err != nil {
+		return input, err
+	}
+
+	if strings.TrimSpace(input.Name) == "" {
+		input.Name = defaultAiModelName(prov.Name, mk)
+	}
+
+	return input, nil
+}
+
+func (m *SAiModel) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input *api.AiModelUpdateInput,
+) (*api.AiModelUpdateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceBaseUpdateInput, err = m.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData")
+	}
+
+	providerId := m.AiProviderId
+	if pid := strings.TrimSpace(input.AiProviderId); pid != "" {
+		prov, err := fetchEnabledAiProvider(ctx, userCred, pid)
+		if err != nil {
+			return input, err
+		}
+		providerId = prov.Id
+		input.AiProviderId = prov.Id
+	}
+
+	modelKey := m.ModelKey
+	if mk := strings.TrimSpace(input.ModelKey); mk != "" {
+		modelKey, err = validateAiModelKey(mk)
+		if err != nil {
+			return input, err
+		}
+		input.ModelKey = modelKey
+	}
+
+	if modelKey != m.ModelKey || providerId != m.AiProviderId {
+		if err := ensureAiModelKeyUniquePerProvider(ctx, providerId, modelKey, m.Id); err != nil {
+			return input, err
+		}
+	}
+
+	return input, nil
+}

pkg/aiproxy/models/ai_providers.go

@@ -0,0 +1,155 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+// SAiProvider stores an LLM provider catalog entry (routing key and OpenAI-compatible config).
+type SAiProvider struct {
+	db.SEnabledStatusStandaloneResourceBase
+
+	// ProviderKey selects the upstream adapter implementation (e.g. openai, vllm, aliyun).
+	// Multiple ai_provider rows may share the same provider_key with different config.
+	ProviderKey string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+	// Config is a JSON snapshot of provider connectivity (base_url, optional api_key).
+	Config *api.SAiProviderConfig `length:"long" charset:"utf8" list:"user" create:"optional" update:"user"`
+}
+
+type SAiProviderManager struct {
+	db.SEnabledStatusStandaloneResourceBaseManager
+}
+
+var AiProviderManager *SAiProviderManager
+
+func init() {
+	AiProviderManager = &SAiProviderManager{
+		SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager(
+			SAiProvider{},
+			"ai_providers_tbl",
+			"ai_provider",
+			"ai_providers",
+		),
+	}
+	AiProviderManager.SetVirtualObject(AiProviderManager)
+}
+
+func (manager *SAiProviderManager) InitializeData() error {
+	return SeedStandardCatalog(context.Background())
+}
+
+func (manager *SAiProviderManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiProviderListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter")
+	}
+	if key := strings.TrimSpace(query.ProviderKey); key != "" {
+		q = q.Equals("provider_key", key)
+	}
+	return q, nil
+}
+
+func (manager *SAiProviderManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiProviderDetails {
+	rows := make([]api.AiProviderDetails, len(objs))
+	baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	for i := range objs {
+		rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i]
+	}
+	return rows
+}
+
+func (manager *SAiProviderManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiProviderCreateInput,
+) (api.AiProviderCreateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData")
+	}
+
+	pk, err := validateAiCatalogIdentifier("provider_key", input.ProviderKey, maxAiProviderKeyLen)
+	if err != nil {
+		return input, err
+	}
+	input.ProviderKey = pk
+
+	input.Config = normalizeAiProviderConfig(input.Config)
+	if err := validateAiProviderConfig(input.Config); err != nil {
+		return input, err
+	}
+
+	if strings.TrimSpace(input.Name) == "" {
+		input.Name = pk
+	}
+
+	return input, nil
+}
+
+func (p *SAiProvider) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input *api.AiProviderUpdateInput,
+) (*api.AiProviderUpdateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceBaseUpdateInput, err = p.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData")
+	}
+
+	if pk := strings.TrimSpace(input.ProviderKey); pk != "" {
+		pk, err = validateAiCatalogIdentifier("provider_key", pk, maxAiProviderKeyLen)
+		if err != nil {
+			return input, err
+		}
+		input.ProviderKey = pk
+	}
+
+	if input.Config != nil {
+		input.Config = normalizeAiProviderConfig(input.Config)
+		if err := validateAiProviderConfig(input.Config); err != nil {
+			return input, err
+		}
+	}
+
+	return input, nil
+}

pkg/aiproxy/models/ai_proxy_nodes.go

@@ -0,0 +1,354 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/rbacscope"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/options"
+	"yunion.io/x/onecloud/pkg/apis"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+const (
+	defaultAiProxyNodeHbTimeout = 120
+	defaultPrimaryAiProxyNodeId = "primary"
+	maxAiProxyNodeDomainLen     = 256
+)
+
+// SAiProxyNode records an aiproxy instance reachable address and optional domain name.
+type SAiProxyNode struct {
+	db.SEnabledStatusStandaloneResourceBase
+
+	Address   string    `width:"256" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user" index:"true"`
+	Domain    string    `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"`
+	LastSeen  time.Time `nullable:"true" list:"user"`
+	HbTimeout int       `nullable:"false" default:"120" list:"user" create:"optional" update:"user"`
+}
+
+type SAiProxyNodeManager struct {
+	db.SEnabledStatusStandaloneResourceBaseManager
+}
+
+var AiProxyNodeManager *SAiProxyNodeManager
+
+func init() {
+	AiProxyNodeManager = &SAiProxyNodeManager{
+		SEnabledStatusStandaloneResourceBaseManager: db.NewEnabledStatusStandaloneResourceBaseManager(
+			SAiProxyNode{},
+			"ai_proxy_nodes_tbl",
+			"ai_proxy_node",
+			"ai_proxy_nodes",
+		),
+	}
+	AiProxyNodeManager.SetVirtualObject(AiProxyNodeManager)
+}
+
+func (manager *SAiProxyNodeManager) ResourceScope() rbacscope.TRbacScope {
+	return rbacscope.ScopeUser
+}
+
+func (manager *SAiProxyNodeManager) InitializeData() error {
+	ctx := context.Background()
+	addr, err := AdvertiseAddressFromOptions(nil)
+	if err != nil {
+		return err
+	}
+	node := SAiProxyNode{}
+	node.SetModelManager(manager, &node)
+	node.Id = defaultPrimaryAiProxyNodeId
+	node.Name = defaultPrimaryAiProxyNodeId
+	node.Description = "Default primary aiproxy node"
+	node.Address = addr
+	domain := ""
+	if existing, err := manager.FetchById(defaultPrimaryAiProxyNodeId); err == nil {
+		domain = strings.TrimSpace(existing.(*SAiProxyNode).Domain)
+	}
+	if domain == "" {
+		d, err := DomainFromApiServer(nil)
+		if err != nil {
+			return err
+		}
+		domain = d
+	}
+	node.Domain = domain
+	node.HbTimeout = defaultAiProxyNodeHbTimeout
+	node.LastSeen = time.Now()
+	node.SetEnabled(true)
+	node.Status = apis.STATUS_AVAILABLE
+	node.Progress = 100
+	if err := manager.TableSpec().InsertOrUpdate(ctx, &node); err != nil {
+		return errors.Wrap(err, "insert or update default primary ai_proxy_node")
+	}
+	return nil
+}
+
+func aiProxyNodeId(address string) string {
+	return stringutils2.GenId("aiproxy.node", address)
+}
+
+func normalizeAiProxyNodeAddress(raw string) (string, error) {
+	address := strings.TrimSpace(raw)
+	if address == "" {
+		return "", errors.Wrap(httperrors.ErrInputParameter, "address is required")
+	}
+	if strings.Contains(address, "://") {
+		u, err := url.Parse(address)
+		if err != nil {
+			return "", errors.Wrapf(httperrors.ErrInputParameter, "invalid address URL: %v", err)
+		}
+		if u.Scheme != "http" && u.Scheme != "https" {
+			return "", errors.Wrap(httperrors.ErrInputParameter, "address scheme must be http or https")
+		}
+		if strings.TrimSpace(u.Host) == "" {
+			return "", errors.Wrap(httperrors.ErrInputParameter, "address must include host")
+		}
+		return strings.TrimRight(address, "/"), nil
+	}
+	if _, _, err := net.SplitHostPort(address); err != nil {
+		return "", errors.Wrapf(httperrors.ErrInputParameter, "invalid address %q: %v", address, err)
+	}
+	return fmt.Sprintf("http://%s", address), nil
+}
+
+func normalizeAiProxyNodeDomain(domain string) (string, error) {
+	domain = strings.TrimSpace(domain)
+	if domain == "" {
+		return "", nil
+	}
+	if len(domain) > maxAiProxyNodeDomainLen {
+		return "", errors.Wrapf(httperrors.ErrInputParameter, "domain too long (max %d)", maxAiProxyNodeDomainLen)
+	}
+	if strings.Contains(domain, "://") || strings.ContainsAny(domain, "/:") {
+		return "", errors.Wrap(httperrors.ErrInputParameter, "domain must be a hostname without scheme or port")
+	}
+	return domain, nil
+}
+
+func aiProxyNodeDisplayName(address string) string {
+	u, err := url.Parse(address)
+	if err != nil || strings.TrimSpace(u.Host) == "" {
+		return address
+	}
+	return u.Host
+}
+
+// AdvertiseAddressFromOptions returns the service URL advertised by this instance.
+func AdvertiseAddressFromOptions(opts *options.SAiProxyOptions) (string, error) {
+	if opts == nil {
+		opts = &options.Options
+	}
+	if addr := strings.TrimRight(strings.TrimSpace(opts.AdvertiseAddress), "/"); addr != "" {
+		return normalizeAiProxyNodeAddress(addr)
+	}
+	scheme := "http"
+	if opts.EnableSsl {
+		scheme = "https"
+	}
+	host := strings.TrimSpace(opts.Address)
+	if host == "" || host == "0.0.0.0" {
+		host = "127.0.0.1"
+	}
+	return normalizeAiProxyNodeAddress(fmt.Sprintf("%s://%s:%d", scheme, host, opts.Port))
+}
+
+// DomainFromApiServer derives ai_proxy_node.domain from --api-server (hostname only).
+func DomainFromApiServer(opts *options.SAiProxyOptions) (string, error) {
+	if opts == nil {
+		opts = &options.Options
+	}
+	raw := strings.TrimSpace(opts.ApiServer)
+	if raw == "" {
+		return "", nil
+	}
+	host := raw
+	if strings.Contains(raw, "://") {
+		u, err := url.Parse(raw)
+		if err != nil {
+			return "", errors.Wrapf(err, "parse api_server %q", raw)
+		}
+		host = strings.TrimSpace(u.Hostname())
+	} else if strings.ContainsAny(raw, "/:") {
+		if h, _, err := net.SplitHostPort(raw); err == nil {
+			host = h
+		}
+	}
+	if host == "" {
+		return "", nil
+	}
+	return normalizeAiProxyNodeDomain(host)
+}
+
+func (manager *SAiProxyNodeManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiProxyNodeListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledStatusStandaloneResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ListItemFilter")
+	}
+	if addr := strings.TrimSpace(query.Address); addr != "" {
+		q = q.Equals("address", addr)
+	}
+	if domain := strings.TrimSpace(query.Domain); domain != "" {
+		q = q.Equals("domain", domain)
+	}
+	return q, nil
+}
+
+func (manager *SAiProxyNodeManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiProxyNodeDetails {
+	rows := make([]api.AiProxyNodeDetails, len(objs))
+	baseRows := manager.SEnabledStatusStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	for i := range objs {
+		rows[i].EnabledStatusStandaloneResourceDetails = baseRows[i]
+		node := objs[i].(*SAiProxyNode)
+		rows[i].IsActive = node.IsActive()
+	}
+	return rows
+}
+
+func (manager *SAiProxyNodeManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiProxyNodeCreateInput,
+) (api.AiProxyNodeCreateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceCreateInput, err = manager.SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.EnabledStatusStandaloneResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBaseManager.ValidateCreateData")
+	}
+	input.Address, err = normalizeAiProxyNodeAddress(input.Address)
+	if err != nil {
+		return input, err
+	}
+	input.Domain, err = normalizeAiProxyNodeDomain(input.Domain)
+	if err != nil {
+		return input, err
+	}
+	if input.HbTimeout <= 0 {
+		input.HbTimeout = defaultAiProxyNodeHbTimeout
+	}
+	if strings.TrimSpace(input.Name) == "" {
+		input.Name = aiProxyNodeDisplayName(input.Address)
+	}
+	return input, nil
+}
+
+func (manager *SAiProxyNodeManager) PerformRegister(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.AiProxyNodeRegisterInput,
+) (jsonutils.JSONObject, error) {
+	addr, err := normalizeAiProxyNodeAddress(input.Address)
+	if err != nil {
+		return nil, err
+	}
+	hbTimeout := input.HbTimeout
+	if hbTimeout <= 0 {
+		hbTimeout = defaultAiProxyNodeHbTimeout
+	}
+	nodeId := aiProxyNodeId(addr)
+	domain := ""
+	if existing, err := manager.FetchById(nodeId); err == nil {
+		domain = existing.(*SAiProxyNode).Domain
+	} else if errors.Cause(err) != sql.ErrNoRows {
+		return nil, errors.Wrap(err, "fetch ai_proxy_node")
+	}
+	node := SAiProxyNode{}
+	node.SetModelManager(manager, &node)
+	node.Id = nodeId
+	node.Name = aiProxyNodeDisplayName(addr)
+	node.Address = addr
+	node.Domain = domain
+	node.HbTimeout = hbTimeout
+	node.LastSeen = time.Now()
+	node.SetEnabled(true)
+	node.Status = apis.STATUS_AVAILABLE
+	node.Progress = 100
+	if err := manager.TableSpec().InsertOrUpdate(ctx, &node); err != nil {
+		return nil, errors.Wrap(err, "insert or update ai_proxy_node")
+	}
+	return jsonutils.Marshal(api.AiProxyNodeRegisterOutput{Id: node.Id}), nil
+}
+
+func (node *SAiProxyNode) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input *api.AiProxyNodeUpdateInput,
+) (*api.AiProxyNodeUpdateInput, error) {
+	var err error
+	input.EnabledStatusStandaloneResourceBaseUpdateInput, err = node.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, input.EnabledStatusStandaloneResourceBaseUpdateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SEnabledStatusStandaloneResourceBase.ValidateUpdateData")
+	}
+	if input.Address != "" {
+		input.Address, err = normalizeAiProxyNodeAddress(input.Address)
+		if err != nil {
+			return input, err
+		}
+	}
+	if query.Contains("domain") {
+		input.Domain, err = normalizeAiProxyNodeDomain(input.Domain)
+		if err != nil {
+			return input, err
+		}
+	}
+	if input.HbTimeout < 0 {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "hb_timeout must be >= 0")
+	}
+	return input, nil
+}
+
+func (node *SAiProxyNode) IsActive() bool {
+	if node.Id == defaultPrimaryAiProxyNodeId {
+		return node.GetEnabled()
+	}
+	if node.LastSeen.IsZero() {
+		return false
+	}
+	timeout := node.HbTimeout
+	if timeout <= 0 {
+		timeout = defaultAiProxyNodeHbTimeout
+	}
+	return int(time.Since(node.LastSeen).Seconds()) < timeout
+}

pkg/aiproxy/models/ai_routing_models.go

@@ -0,0 +1,337 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"database/sql"
+	stderrors "errors"
+	"fmt"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+// SAiRoutingModel binds a catalog model (and provider) to an ai_routing with per-entry priority.
+type SAiRoutingModel struct {
+	db.SStandaloneResourceBase
+
+	AiRoutingId  string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user" index:"true"`
+	AiProviderId string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+	AiModelId    string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"`
+	// Priority orders models within the same ai_routing (lower value = higher priority).
+	Priority int `default:"100" nullable:"false" list:"user" create:"optional" update:"user"`
+	// ModelPattern optionally matches the client request "model" (same rules as ai_routing.model_pattern).
+	ModelPattern string            `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"`
+	Enabled      tristate.TriState `default:"true" nullable:"false" list:"user" create:"optional" update:"user"`
+}
+
+type SAiRoutingModelManager struct {
+	db.SStandaloneResourceBaseManager
+}
+
+var AiRoutingModelManager *SAiRoutingModelManager
+
+func init() {
+	AiRoutingModelManager = &SAiRoutingModelManager{
+		SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager(
+			SAiRoutingModel{},
+			"ai_routing_models_tbl",
+			"ai_routing_model",
+			"ai_routing_models",
+		),
+	}
+	AiRoutingModelManager.SetVirtualObject(AiRoutingModelManager)
+}
+
+func (manager *SAiRoutingModelManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiRoutingModelListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
+	}
+	if id := strings.TrimSpace(query.AiRoutingId); id != "" {
+		q = q.Equals("ai_routing_id", id)
+	}
+	if id := strings.TrimSpace(query.AiProviderId); id != "" {
+		q = q.Equals("ai_provider_id", id)
+	}
+	if id := strings.TrimSpace(query.AiModelId); id != "" {
+		q = q.Equals("ai_model_id", id)
+	}
+	if query.Enabled != nil {
+		q = q.Equals("enabled", *query.Enabled)
+	}
+	return q, nil
+}
+
+func (manager *SAiRoutingModelManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiRoutingModelDetails {
+	rows := make([]api.AiRoutingModelDetails, len(objs))
+	baseRows := manager.SStandaloneResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	for i := range objs {
+		rows[i].StandaloneResourceDetails = baseRows[i]
+	}
+	return rows
+}
+
+func (manager *SAiRoutingModelManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiRoutingModelCreateInput,
+) (api.AiRoutingModelCreateInput, error) {
+	var err error
+	input.StandaloneResourceCreateInput, err = manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.StandaloneResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SStandaloneResourceBaseManager.ValidateCreateData")
+	}
+
+	routingId := strings.TrimSpace(input.AiRoutingId)
+	if routingId == "" {
+		return input, errors.Wrap(httperrors.ErrInputParameter, "ai_routing_id is required")
+	}
+	rObj, err := AiRoutingManager.FetchById(routingId)
+	if err != nil {
+		return input, errors.Wrap(err, "fetch ai_routing")
+	}
+	routing := rObj.(*SAiRouting)
+
+	providerId, modelId, err := resolveAiRoutingModelRefs(ctx, userCred, strings.TrimSpace(input.AiProviderId), strings.TrimSpace(input.AiModelId))
+	if err != nil {
+		return input, err
+	}
+	input.AiProviderId = providerId
+	input.AiModelId = modelId
+	input.AiRoutingId = routing.Id
+
+	if strings.TrimSpace(input.Name) == "" {
+		input.Name = fmt.Sprintf("%s-%s-%d", routing.Name, modelId, input.Priority)
+	}
+
+	if input.Enabled == nil {
+		enabled := true
+		input.Enabled = &enabled
+	}
+	return input, nil
+}
+
+func fetchAiModelRef(ctx context.Context, userCred mcclient.TokenCredential, providerIdOrName, modelIdOrName string) (*SAiModel, error) {
+	modelIdOrName = strings.TrimSpace(modelIdOrName)
+	if modelIdOrName == "" {
+		return nil, errors.Wrap(httperrors.ErrInputParameter, "ai_model_id is required")
+	}
+	mObj, err := AiModelManager.FetchByIdOrName(ctx, userCred, modelIdOrName)
+	if err == nil {
+		return mObj.(*SAiModel), nil
+	}
+	if !stderrors.Is(err, sql.ErrNoRows) {
+		return nil, errors.Wrap(err, "fetch ai_model")
+	}
+	if pk := strings.TrimSpace(providerIdOrName); pk != "" {
+		if mObj2, err2 := AiModelManager.FetchByIdOrName(ctx, userCred, catalogModelId(pk, modelIdOrName)); err2 == nil {
+			return mObj2.(*SAiModel), nil
+		}
+	}
+	providerIdOrName = strings.TrimSpace(providerIdOrName)
+	if providerIdOrName == "" {
+		return nil, errors.Wrap(err, "fetch ai_model")
+	}
+	prov, err := fetchEnabledAiProvider(ctx, userCred, providerIdOrName)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch ai_model by model_key")
+	}
+	mdl := SAiModel{}
+	q := AiModelManager.Query().Equals("ai_provider_id", prov.Id).Equals("model_key", modelIdOrName)
+	if err := q.First(&mdl); err != nil {
+		return nil, errors.Wrap(err, "fetch ai_model")
+	}
+	return &mdl, nil
+}
+
+func resolveAiRoutingModelRefs(ctx context.Context, userCred mcclient.TokenCredential, providerIdOrName, modelIdOrName string) (string, string, error) {
+	mdl, err := fetchAiModelRef(ctx, userCred, providerIdOrName, modelIdOrName)
+	if err != nil {
+		return "", "", err
+	}
+	if !mdl.GetEnabled() {
+		return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_model disabled")
+	}
+
+	providerId := strings.TrimSpace(providerIdOrName)
+	if providerId == "" {
+		providerId = mdl.AiProviderId
+	}
+	pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerId)
+	if err != nil {
+		return "", "", errors.Wrap(err, "fetch ai_provider")
+	}
+	prov := pObj.(*SAiProvider)
+	if !prov.GetEnabled() {
+		return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider disabled")
+	}
+	if mdl.AiProviderId != prov.Id {
+		return "", "", errors.Wrap(httperrors.ErrInputParameter, "ai_model does not belong to ai_provider")
+	}
+	return prov.Id, mdl.Id, nil
+}
+
+func routingModelItemPriority(priority, weight int) int {
+	if priority != 0 {
+		return priority
+	}
+	if weight != 0 {
+		return weight
+	}
+	return 100
+}
+
+func validateAiRoutingModelItems(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	items []api.AiRoutingModelItem,
+) ([]api.AiRoutingModelItem, error) {
+	if len(items) == 0 {
+		return nil, nil
+	}
+	out := make([]api.AiRoutingModelItem, len(items))
+	for i := range items {
+		item := items[i]
+		providerId, modelId, err := resolveAiRoutingModelRefs(ctx, userCred, strings.TrimSpace(item.AiProviderId), strings.TrimSpace(item.AiModelId))
+		if err != nil {
+			return nil, errors.Wrapf(err, "models[%d]", i)
+		}
+		item.AiProviderId = providerId
+		item.AiModelId = modelId
+		item.Priority = routingModelItemPriority(item.Priority, item.Weight)
+		item.Weight = 0
+		if item.Enabled == nil {
+			enabled := true
+			item.Enabled = &enabled
+		}
+		out[i] = item
+	}
+	return out, nil
+}
+
+func deleteAiRoutingModels(ctx context.Context, routingId string) error {
+	routingId = strings.TrimSpace(routingId)
+	if routingId == "" {
+		return errors.Wrap(httperrors.ErrInputParameter, "ai_routing_id is required")
+	}
+	_, err := sqlchemy.GetDB().Exec(
+		fmt.Sprintf("delete from %s where ai_routing_id = ?", AiRoutingModelManager.TableSpec().Name()),
+		routingId,
+	)
+	if err != nil {
+		return errors.Wrap(err, "delete ai_routing_models")
+	}
+	return nil
+}
+
+func createAiRoutingModels(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	routing *SAiRouting,
+	items []api.AiRoutingModelItem,
+) error {
+	if routing == nil || len(items) == 0 {
+		return nil
+	}
+	for i := range items {
+		item := items[i]
+		enabled := true
+		if item.Enabled != nil {
+			enabled = *item.Enabled
+		}
+		dataDict := jsonutils.NewDict()
+		dataDict.Set("ai_routing_id", jsonutils.NewString(routing.Id))
+		dataDict.Set("ai_provider_id", jsonutils.NewString(item.AiProviderId))
+		dataDict.Set("ai_model_id", jsonutils.NewString(item.AiModelId))
+		dataDict.Set("priority", jsonutils.NewInt(int64(item.Priority)))
+		if mp := strings.TrimSpace(item.ModelPattern); mp != "" {
+			dataDict.Set("model_pattern", jsonutils.NewString(mp))
+		}
+		dataDict.Set("enabled", jsonutils.JSONTrue)
+		if !enabled {
+			dataDict.Set("enabled", jsonutils.JSONFalse)
+		}
+		dataDict.Set("name", jsonutils.NewString(fmt.Sprintf("%s-%s-%d", routing.Name, item.AiModelId, item.Priority)))
+		if _, err := db.DoCreate(AiRoutingModelManager, ctx, userCred, nil, dataDict, ownerId); err != nil {
+			return errors.Wrapf(err, "create ai_routing_model[%d]", i)
+		}
+	}
+	return nil
+}
+
+func fetchAiRoutingModels(routingId string, enabledOnly bool) ([]SAiRoutingModel, error) {
+	entries := make([]SAiRoutingModel, 0, 8)
+	q := AiRoutingModelManager.Query().Equals("ai_routing_id", routingId)
+	if enabledOnly {
+		q = q.Equals("enabled", true)
+	}
+	err := q.Asc("priority").Asc("id").All(&entries)
+	if err != nil {
+		return nil, errors.Wrap(err, "list ai_routing_models")
+	}
+	return entries, nil
+}
+
+func fetchEnabledAiRoutingModels(routingId string) ([]SAiRoutingModel, error) {
+	return fetchAiRoutingModels(routingId, true)
+}
+
+// pickAiRoutingModel selects provider/model from ai_routing_models by request model name.
+func pickAiRoutingModel(ctx context.Context, userCred mcclient.TokenCredential, routing *SAiRouting, reqModel string) (providerId, modelId string, err error) {
+	if routing == nil {
+		return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing")
+	}
+	entries, err := fetchEnabledAiRoutingModels(routing.Id)
+	if err != nil {
+		return "", "", err
+	}
+	if len(entries) == 0 {
+		return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_routing has no ai_routing_models")
+	}
+	for i := range entries {
+		e := &entries[i]
+		if !modelPatternMatches(e.ModelPattern, reqModel) {
+			continue
+		}
+		return e.AiProviderId, e.AiModelId, nil
+	}
+	return "", "", errors.Wrapf(httperrors.ErrNotFound, "no ai_routing_model matched request model %q", reqModel)
+}

pkg/aiproxy/models/ai_routings.go

@@ -0,0 +1,236 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+// SAiRouting stores a project-scoped (and optionally shared) routing rule.
+type SAiRouting struct {
+	db.SSharableVirtualResourceBase
+	db.SEnabledResourceBase
+
+	Priority int `default:"100" nullable:"false" list:"user" create:"optional" update:"user"`
+	// ModelPattern optionally matches the requested model id (implementation-specific glob/prefix).
+	ModelPattern string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"`
+	// AiProxyNodeId optionally binds the rule to one aiproxy instance (ai_proxy_node id).
+	AiProxyNodeId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
+}
+
+type SAiRoutingManager struct {
+	db.SSharableVirtualResourceBaseManager
+	db.SEnabledResourceBaseManager
+}
+
+var AiRoutingManager *SAiRoutingManager
+
+func init() {
+	AiRoutingManager = &SAiRoutingManager{
+		SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager(
+			SAiRouting{},
+			"ai_routings_tbl",
+			"ai_routing",
+			"ai_routings",
+		),
+	}
+	AiRoutingManager.SetVirtualObject(AiRoutingManager)
+}
+
+func (manager *SAiRoutingManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiRoutingListInput,
+) (*sqlchemy.SQuery, error) {
+	q, err := manager.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.SharableVirtualResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter")
+	}
+	if v := strings.TrimSpace(query.ModelPattern); v != "" {
+		q = q.Equals("model_pattern", v)
+	}
+	if v := strings.TrimSpace(query.AiProxyNodeId); v != "" {
+		q = q.Equals("ai_proxy_node_id", v)
+	}
+	q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter")
+	}
+	return q, nil
+}
+
+func (manager *SAiRoutingManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiRoutingDetails {
+	rows := make([]api.AiRoutingDetails, len(objs))
+	sharableRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	routingIds := make([]string, len(objs))
+	for i := range objs {
+		rows[i].SharableVirtualResourceDetails = sharableRows[i]
+		routingIds[i] = objs[i].(*SAiRouting).Id
+	}
+	if fields == nil || fields.Contains("routing_models") {
+		for i, rid := range routingIds {
+			if rid == "" {
+				continue
+			}
+			entries, err := fetchAiRoutingModels(rid, false)
+			if err != nil {
+				continue
+			}
+			rows[i].RoutingModels = make([]api.AiRoutingModelDetails, len(entries))
+			for j := range entries {
+				e := entries[j]
+				rows[i].RoutingModels[j] = api.AiRoutingModelDetails{
+					AiRoutingId:  e.AiRoutingId,
+					AiProviderId: e.AiProviderId,
+					AiModelId:    e.AiModelId,
+					Priority:     e.Priority,
+					ModelPattern: e.ModelPattern,
+					Enabled:      e.Enabled.IsTrue(),
+				}
+			}
+		}
+	}
+	return rows
+}
+
+func (routing *SAiRouting) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformEnableInput) (jsonutils.JSONObject, error) {
+	if err := db.EnabledPerformEnable(routing, ctx, userCred, true); err != nil {
+		return nil, errors.Wrap(err, "EnabledPerformEnable")
+	}
+	return nil, nil
+}
+
+func (routing *SAiRouting) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformDisableInput) (jsonutils.JSONObject, error) {
+	if err := db.EnabledPerformEnable(routing, ctx, userCred, false); err != nil {
+		return nil, errors.Wrap(err, "EnabledPerformEnable")
+	}
+	return nil, nil
+}
+
+func (routing *SAiRouting) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input *api.AiRoutingUpdateInput,
+) (*api.AiRoutingUpdateInput, error) {
+	var err error
+	input.SharableVirtualResourceBaseUpdateInput, err = routing.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SSharableVirtualResourceBase.ValidateUpdateData")
+	}
+	if input.AiProxyNodeId != "" {
+		input.AiProxyNodeId, err = validateAiProxyNodeId(ctx, userCred, input.AiProxyNodeId)
+		if err != nil {
+			return input, err
+		}
+	} else if query.Contains("ai_proxy_node_id") {
+		input.AiProxyNodeId = ""
+	}
+	return input, nil
+}
+
+func (manager *SAiRoutingManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiRoutingCreateInput,
+) (api.AiRoutingCreateInput, error) {
+	var err error
+	input.SharableVirtualResourceCreateInput, err = manager.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ValidateCreateData")
+	}
+
+	validatedModels, err := validateAiRoutingModelItems(ctx, userCred, input.Models)
+	if err != nil {
+		return input, err
+	}
+	input.Models = validatedModels
+
+	input.AiProxyNodeId, err = validateAiProxyNodeId(ctx, userCred, input.AiProxyNodeId)
+	if err != nil {
+		return input, err
+	}
+
+	if input.Enabled == nil && input.Disabled == nil {
+		input.SetEnabled()
+	}
+	return input, nil
+}
+
+func (routing *SAiRouting) PostCreate(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) {
+	routing.SSharableVirtualResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
+	input := api.AiRoutingCreateInput{}
+	if err := data.Unmarshal(&input); err != nil {
+		log.Errorf("ai_routing PostCreate unmarshal models: %v", err)
+		return
+	}
+	if len(input.Models) == 0 {
+		return
+	}
+	if err := createAiRoutingModels(ctx, userCred, ownerId, routing, input.Models); err != nil {
+		log.Errorf("ai_routing %s create routing_models: %v", routing.Id, err)
+	}
+}
+
+// PerformSetModels replaces all ai_routing_models bound to this routing.
+func (routing *SAiRouting) PerformSetModels(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.AiRoutingSetModelsInput,
+) (jsonutils.JSONObject, error) {
+	items, err := validateAiRoutingModelItems(ctx, userCred, input.Models)
+	if err != nil {
+		return nil, err
+	}
+	if err := deleteAiRoutingModels(ctx, routing.Id); err != nil {
+		return nil, err
+	}
+	if len(items) == 0 {
+		return nil, nil
+	}
+	if err := createAiRoutingModels(ctx, userCred, routing.GetOwnerId(), routing, items); err != nil {
+		return nil, err
+	}
+	return nil, nil
+}

pkg/aiproxy/models/ai_virtual_keys.go

@@ -0,0 +1,314 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"github.com/google/uuid"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/pkg/util/rbacscope"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+const virtualKeyPrefix = "sk-"
+
+// SAiVirtualKey stores a client-facing virtual API key; upstream routing is resolved from project-scoped ai_routing rules.
+type SAiVirtualKey struct {
+	db.SVirtualResourceBase
+	db.SEnabledResourceBase
+
+	// OwnerId is the user that owns this virtual key within the project.
+	OwnerId string `width:"128" charset:"ascii" index:"true" list:"user" nullable:"false" create:"optional" update:"user"`
+	// VirtualKey is the opaque key id or prefix presented to clients (not the upstream provider secret).
+	VirtualKey string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"`
+	// Limits constrains allowed providers, per-request max_tokens, and request rate.
+	Limits *api.SAiVirtualKeyLimits `length:"medium" charset:"utf8" list:"user" create:"optional" update:"user"`
+}
+
+type SAiVirtualKeyManager struct {
+	db.SVirtualResourceBaseManager
+	db.SEnabledResourceBaseManager
+}
+
+var AiVirtualKeyManager *SAiVirtualKeyManager
+
+func init() {
+	AiVirtualKeyManager = &SAiVirtualKeyManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SAiVirtualKey{},
+			"ai_virtual_keys_tbl",
+			"ai_virtual_key",
+			"ai_virtual_keys",
+		),
+	}
+	AiVirtualKeyManager.SetVirtualObject(AiVirtualKeyManager)
+}
+
+func (m *SAiVirtualKey) GetOwnerId() mcclient.IIdentityProvider {
+	owner := db.SOwnerId{
+		UserId:    m.OwnerId,
+		DomainId:  m.DomainId,
+		ProjectId: m.ProjectId,
+	}
+	return &owner
+}
+
+func (manager *SAiVirtualKeyManager) NamespaceScope() rbacscope.TRbacScope {
+	return rbacscope.ScopeUser
+}
+
+func (m *SAiVirtualKey) IsOwner(userCred mcclient.TokenCredential) bool {
+	return userCred.GetUserId() == m.OwnerId
+}
+
+func (manager *SAiVirtualKeyManager) ResourceScope() rbacscope.TRbacScope {
+	return rbacscope.ScopeUser
+}
+
+func (manager *SAiVirtualKeyManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	return db.FetchUserInfo(ctx, data)
+}
+
+func (manager *SAiVirtualKeyManager) FilterByOwner(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	man db.FilterByOwnerProvider,
+	userCred mcclient.TokenCredential,
+	owner mcclient.IIdentityProvider,
+	scope rbacscope.TRbacScope,
+) *sqlchemy.SQuery {
+	if owner != nil && scope == rbacscope.ScopeUser {
+		if uid := strings.TrimSpace(owner.GetUserId()); uid != "" {
+			q = q.Equals("owner_id", uid)
+		}
+	}
+	return q
+}
+
+func (manager *SAiVirtualKeyManager) ListItemFilter(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiVirtualKeyListInput,
+) (*sqlchemy.SQuery, error) {
+	var err error
+	q, err = manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query.VirtualResourceListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SVirtualResourceBaseManager.ListItemFilter")
+	}
+	q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledResourceBaseListInput)
+	if err != nil {
+		return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter")
+	}
+	if v := strings.TrimSpace(query.VirtualKey); v != "" {
+		q = q.Equals("virtual_key", v)
+	}
+	userId := strings.TrimSpace(query.UserId)
+	if userId != "" {
+		if !db.IsAdminAllowList(userCred, manager).Result.IsAllow() {
+			return nil, httperrors.NewForbiddenError("only admin may filter by user_id")
+		}
+		uc, err := db.UserCacheManager.FetchUserByIdOrName(ctx, userId)
+		if err != nil {
+			return nil, errors.Wrap(err, "fetch user")
+		}
+		q = q.Equals("owner_id", uc.Id)
+	} else if !db.IsAdminAllowList(userCred, manager).Result.IsAllow() {
+		q = q.Equals("owner_id", userCred.GetUserId())
+	}
+	return q, nil
+}
+
+func (manager *SAiVirtualKeyManager) OrderByExtraFields(
+	ctx context.Context,
+	q *sqlchemy.SQuery,
+	userCred mcclient.TokenCredential,
+	query api.AiVirtualKeyListInput,
+) (*sqlchemy.SQuery, error) {
+	return manager.SVirtualResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.VirtualResourceListInput)
+}
+
+func (manager *SAiVirtualKeyManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
+	q, err := manager.SVirtualResourceBaseManager.QueryDistinctExtraField(q, field)
+	if err == nil {
+		return q, nil
+	}
+	return q, httperrors.ErrNotFound
+}
+
+func (manager *SAiVirtualKeyManager) FetchCustomizeColumns(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	objs []interface{},
+	fields stringutils2.SSortedStrings,
+	isList bool,
+) []api.AiVirtualKeyDetails {
+	rows := make([]api.AiVirtualKeyDetails, len(objs))
+	virtRows := manager.SVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
+	userIds := make([]string, len(objs))
+	for i := range objs {
+		rows[i].VirtualResourceDetails = virtRows[i]
+		vk := objs[i].(*SAiVirtualKey)
+		if strings.TrimSpace(vk.OwnerId) != "" {
+			userIds[i] = vk.OwnerId
+		}
+	}
+	userMaps, err := db.FetchIdNameMap2(db.UserCacheManager, userIds)
+	if err != nil {
+		log.Errorf("FetchIdNameMap2 fail: %v", err)
+		return rows
+	}
+	for i := range rows {
+		rows[i].OwnerName, _ = userMaps[userIds[i]]
+	}
+	return rows
+}
+
+func (m *SAiVirtualKey) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformEnableInput) (jsonutils.JSONObject, error) {
+	if err := db.EnabledPerformEnable(m, ctx, userCred, true); err != nil {
+		return nil, errors.Wrap(err, "EnabledPerformEnable")
+	}
+	return nil, nil
+}
+
+func (m *SAiVirtualKey) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformDisableInput) (jsonutils.JSONObject, error) {
+	if err := db.EnabledPerformEnable(m, ctx, userCred, false); err != nil {
+		return nil, errors.Wrap(err, "EnabledPerformEnable")
+	}
+	return nil, nil
+}
+
+func (manager *SAiVirtualKeyManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	input api.AiVirtualKeyCreateInput,
+) (api.AiVirtualKeyCreateInput, error) {
+	var err error
+	input.VirtualResourceCreateInput, err = manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.VirtualResourceCreateInput)
+	if err != nil {
+		return input, errors.Wrap(err, "SVirtualResourceBaseManager.ValidateCreateData")
+	}
+
+	if strings.TrimSpace(input.OwnerId) == "" {
+		input.OwnerId = userCred.GetUserId()
+	} else if !db.IsAdminAllowList(userCred, manager).Result.IsAllow() && input.OwnerId != userCred.GetUserId() {
+		return input, httperrors.NewForbiddenError("cannot create virtual key for another user")
+	}
+
+	if err := validateAiVirtualKeyLimits(ctx, userCred, input.Limits); err != nil {
+		return input, err
+	}
+
+	vk := strings.TrimSpace(input.VirtualKey)
+	if vk != "" {
+		if !strings.HasPrefix(vk, virtualKeyPrefix) {
+			return input, errors.Wrapf(httperrors.ErrInputParameter, "virtual_key must start with %q", virtualKeyPrefix)
+		}
+		if len(vk) > 128 {
+			return input, errors.Wrap(httperrors.ErrInputParameter, "virtual_key too long")
+		}
+		exists, err := aiVirtualKeyExists(vk)
+		if err != nil {
+			return input, err
+		}
+		if exists {
+			return input, errors.Wrap(httperrors.ErrConflict, "virtual_key already exists")
+		}
+		input.VirtualKey = vk
+	} else {
+		input.VirtualKey, err = generateUniqueVirtualKey()
+		if err != nil {
+			return input, err
+		}
+	}
+
+	if input.Enabled.IsNone() {
+		input.Enabled = tristate.True
+	}
+
+	return input, nil
+}
+
+func validateAiVirtualKeyLimits(ctx context.Context, userCred mcclient.TokenCredential, lim *api.SAiVirtualKeyLimits) error {
+	if lim == nil {
+		return nil
+	}
+	if lim.MaxTokensPerRequest < 0 {
+		return errors.Wrap(httperrors.ErrInputParameter, "limits.max_tokens_per_request must be >= 0")
+	}
+	if lim.RequestsPerMinute < 0 {
+		return errors.Wrap(httperrors.ErrInputParameter, "limits.requests_per_minute must be >= 0")
+	}
+	if len(lim.AllowedAiProviderIds) == 0 {
+		return nil
+	}
+	resolved := make([]string, 0, len(lim.AllowedAiProviderIds))
+	for _, idOrName := range lim.AllowedAiProviderIds {
+		idOrName = strings.TrimSpace(idOrName)
+		if idOrName == "" {
+			continue
+		}
+		pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, idOrName)
+		if err != nil {
+			return errors.Wrapf(err, "limits.allowed_ai_provider_ids: fetch %q", idOrName)
+		}
+		prov := pObj.(*SAiProvider)
+		if !prov.GetEnabled() {
+			return errors.Wrapf(httperrors.ErrInvalidStatus, "limits.allowed_ai_provider_ids: ai_provider %q disabled", idOrName)
+		}
+		resolved = append(resolved, prov.Id)
+	}
+	lim.AllowedAiProviderIds = resolved
+	return nil
+}
+
+func aiVirtualKeyExists(virtualKey string) (bool, error) {
+	cnt, err := AiVirtualKeyManager.Query().Equals("virtual_key", virtualKey).CountWithError()
+	if err != nil {
+		return false, errors.Wrap(err, "count ai_virtual_key")
+	}
+	return cnt > 0, nil
+}
+
+func generateUniqueVirtualKey() (string, error) {
+	const maxAttempts = 8
+	for i := 0; i < maxAttempts; i++ {
+		vk := virtualKeyPrefix + strings.ReplaceAll(uuid.New().String(), "-", "")
+		exists, err := aiVirtualKeyExists(vk)
+		if err != nil {
+			return "", err
+		}
+		if !exists {
+			return vk, nil
+		}
+	}
+	return "", errors.Wrap(httperrors.ErrConflict, "failed to generate unique virtual_key")
+}

pkg/aiproxy/models/aiproxy_catalog_validate.go

@@ -0,0 +1,181 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+const (
+	maxAiProviderKeyLen = 64
+	maxAiModelKeyLen    = 256
+)
+
+var aiCatalogIdentifierRe = regexp.MustCompile(`^[a-z0-9][a-z0-9_-]*$`)
+
+func validateAiCatalogIdentifier(field, value string, maxLen int) (string, error) {
+	v := strings.TrimSpace(value)
+	if v == "" {
+		return "", errors.Wrapf(httperrors.ErrInputParameter, "%s is required", field)
+	}
+	if len(v) > maxLen {
+		return "", errors.Wrapf(httperrors.ErrInputParameter, "%s too long (max %d)", field, maxLen)
+	}
+	if !aiCatalogIdentifierRe.MatchString(v) {
+		return "", errors.Wrapf(httperrors.ErrInputParameter, "%s must match [a-z0-9][a-z0-9_-]*", field)
+	}
+	return v, nil
+}
+
+func validateAiModelKey(modelKey string) (string, error) {
+	key := strings.TrimSpace(modelKey)
+	if key == "" {
+		return "", errors.Wrap(httperrors.ErrInputParameter, "model_key is required")
+	}
+	if len(key) > maxAiModelKeyLen {
+		return "", errors.Wrap(httperrors.ErrInputParameter, "model_key too long")
+	}
+	return key, nil
+}
+
+// catalogModelId returns a stable ai_model row id for catalog seed (readable when model_key is simple).
+// Format: {provider_key}-{slug(model_key)}; falls back to GenId for path-like or overlong keys.
+func catalogModelId(providerKey, modelKey string) string {
+	pk := strings.ToLower(strings.TrimSpace(providerKey))
+	mk := strings.TrimSpace(modelKey)
+	if pk == "" || mk == "" {
+		return stringutils2.GenId("aiproxy.ai_model", providerKey, modelKey)
+	}
+	slug := catalogModelKeySlug(mk)
+	id := pk + "-" + slug
+	const maxIdLen = 128
+	if len(id) > maxIdLen {
+		trim := maxIdLen - len(pk) - 1
+		if trim > 0 {
+			id = pk + "-" + slug[:trim]
+		} else {
+			id = pk[:maxIdLen]
+		}
+	}
+	if aiCatalogIdentifierRe.MatchString(id) {
+		return id
+	}
+	return stringutils2.GenId("aiproxy.ai_model", providerKey, modelKey)
+}
+
+func catalogModelKeySlug(modelKey string) string {
+	var b strings.Builder
+	lastDash := false
+	for _, r := range strings.TrimSpace(modelKey) {
+		switch {
+		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
+			b.WriteRune(r)
+			lastDash = false
+		case r >= 'A' && r <= 'Z':
+			b.WriteRune(r + ('a' - 'A'))
+			lastDash = false
+		case r == '-', r == '_', r == '.', r == '/':
+			if b.Len() > 0 && !lastDash {
+				b.WriteByte('-')
+				lastDash = true
+			}
+		}
+	}
+	s := strings.Trim(b.String(), "-")
+	if s == "" {
+		return "model"
+	}
+	return s
+}
+
+func validateAiProviderConfig(cfg *api.SAiProviderConfig) error {
+	if cfg == nil || cfg.IsZero() {
+		return nil
+	}
+	baseURL := cfg.ResolvedBaseURL()
+	if baseURL == "" {
+		return nil
+	}
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return errors.Wrapf(httperrors.ErrInputParameter, "config.base_url: invalid URL: %v", err)
+	}
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return errors.Wrap(httperrors.ErrInputParameter, "config.base_url must use http or https scheme")
+	}
+	if strings.TrimSpace(u.Host) == "" {
+		return errors.Wrap(httperrors.ErrInputParameter, "config.base_url must include a host")
+	}
+	return nil
+}
+
+func normalizeAiProviderConfig(cfg *api.SAiProviderConfig) *api.SAiProviderConfig {
+	if cfg == nil || cfg.IsZero() {
+		return cfg
+	}
+	out := &api.SAiProviderConfig{}
+	if base := cfg.ResolvedBaseURL(); base != "" {
+		out.BaseURL = base
+	}
+	if key := cfg.ResolvedAPIKey(); key != "" {
+		out.APIKey = key
+	}
+	return out
+}
+
+func ensureAiModelKeyUniquePerProvider(ctx context.Context, providerId, modelKey, excludeId string) error {
+	q := AiModelManager.Query().Equals("ai_provider_id", providerId).Equals("model_key", modelKey)
+	if excludeId != "" {
+		q = q.NotEquals("id", excludeId)
+	}
+	cnt, err := q.CountWithError()
+	if err != nil {
+		return errors.Wrap(err, "count ai_model by provider and model_key")
+	}
+	if cnt > 0 {
+		return errors.Wrapf(httperrors.ErrConflict, "model_key %q already exists for ai_provider", modelKey)
+	}
+	return nil
+}
+
+func fetchEnabledAiProvider(ctx context.Context, userCred mcclient.TokenCredential, idOrName string) (*SAiProvider, error) {
+	idOrName = strings.TrimSpace(idOrName)
+	if idOrName == "" {
+		return nil, errors.Wrap(httperrors.ErrInputParameter, "ai_provider_id is required")
+	}
+	pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, idOrName)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch ai_provider")
+	}
+	prov := pObj.(*SAiProvider)
+	if !prov.GetEnabled() {
+		return nil, errors.Wrapf(httperrors.ErrInvalidStatus, "ai_provider %q is disabled", idOrName)
+	}
+	return prov, nil
+}
+
+func defaultAiModelName(providerName, modelKey string) string {
+	return catalogModelId(providerName, modelKey)
+}

pkg/aiproxy/models/catalog_seed.go

@@ -0,0 +1,229 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+)
+
+// standardCatalogProviderKeys lists built-in provider_key values seeded at InitDB.
+var standardCatalogProviderKeys = []string{
+	"anthropic",
+	"azure",
+	"bedrock",
+	"cerebras",
+	"cohere",
+	"gemini",
+	"groq",
+	"mistral",
+	"ollama",
+	"openai",
+	"parasail",
+	"perplexity",
+	"sgl",
+	"vertex",
+	"openrouter",
+	"elevenlabs",
+	"huggingface",
+	"nebius",
+	"xai",
+	"replicate",
+	"vllm",
+	"runway",
+	"fireworks",
+	"aliyun",
+	"baidu",
+	"xiaomi",
+}
+
+// defaultPublicBaseURL returns a well-known public API base for OpenAI-compatible upstreams.
+// Empty string means no default in catalog (user must set base_url in provider config).
+func defaultPublicBaseURL(providerKey string) string {
+	switch strings.ToLower(strings.TrimSpace(providerKey)) {
+	case "openai":
+		return "https://api.openai.com"
+	case "anthropic":
+		return "https://api.anthropic.com"
+	case "azure", "bedrock", "sgl", "vertex":
+		return ""
+	case "cerebras":
+		return "https://api.cerebras.ai"
+	case "cohere":
+		return "https://api.cohere.ai"
+	case "gemini":
+		return "https://generativelanguage.googleapis.com/v1beta"
+	case "groq":
+		return "https://api.groq.com/openai"
+	case "mistral":
+		return "https://api.mistral.ai"
+	case "ollama":
+		return "http://127.0.0.1:11434"
+	case "vllm":
+		return "http://127.0.0.1:8000"
+	case "parasail":
+		return "https://api.parasail.io"
+	case "perplexity":
+		return "https://api.perplexity.ai"
+	case "openrouter":
+		return "https://openrouter.ai/api"
+	case "elevenlabs":
+		return "https://api.elevenlabs.io"
+	case "huggingface":
+		return "https://router.huggingface.co"
+	case "nebius":
+		return "https://api.tokenfactory.nebius.com"
+	case "xai":
+		return "https://api.x.ai"
+	case "replicate":
+		return "https://api.replicate.com"
+	case "runway":
+		return "https://api.dev.runwayml.com"
+	case "fireworks":
+		return "https://api.fireworks.ai/inference"
+	case "aliyun":
+		return "https://dashscope.aliyuncs.com/compatible-mode"
+	case "baidu":
+		return "https://qianfan.baidubce.com/v2"
+	case "xiaomi":
+		return "https://api.xiaomimimo.com"
+	default:
+		return ""
+	}
+}
+
+func standardProviderConfig(providerKey string) *api.SAiProviderConfig {
+	if u := defaultPublicBaseURL(providerKey); u != "" {
+		return &api.SAiProviderConfig{BaseURL: u}
+	}
+	return nil
+}
+
+const placeholderCatalogModelKey = "default"
+
+func catalogProviderId(providerKey string) string {
+	return providerKey
+}
+
+func catalogProviderExists(providerId string) (bool, error) {
+	cnt, err := AiProviderManager.Query().Equals("id", providerId).CountWithError()
+	if err != nil {
+		return false, errors.Wrap(err, "count catalog ai_provider")
+	}
+	return cnt > 0, nil
+}
+
+func catalogModelExists(modelId string) (bool, error) {
+	cnt, err := AiModelManager.Query().Equals("id", modelId).CountWithError()
+	if err != nil {
+		return false, errors.Wrap(err, "count catalog ai_model")
+	}
+	return cnt > 0, nil
+}
+
+func insertCatalogProvider(ctx context.Context, providerKey, description string, cfg *api.SAiProviderConfig) error {
+	providerId := catalogProviderId(providerKey)
+	exists, err := catalogProviderExists(providerId)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	prov := SAiProvider{}
+	prov.SetModelManager(AiProviderManager, &prov)
+	prov.Id = providerId
+	prov.Name = providerKey
+	prov.ProviderKey = providerKey
+	prov.Description = description
+	prov.Config = cfg
+	prov.SetEnabled(true)
+	prov.Status = apis.STATUS_AVAILABLE
+	prov.Progress = 100
+	if err := AiProviderManager.TableSpec().Insert(ctx, &prov); err != nil {
+		return errors.Wrapf(err, "insert ai_provider %s", providerKey)
+	}
+	return nil
+}
+
+func insertCatalogModel(ctx context.Context, providerId, providerKey, modelKey, description string) error {
+	modelId := catalogModelId(providerKey, modelKey)
+	exists, err := catalogModelExists(modelId)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return nil
+	}
+
+	m := SAiModel{}
+	m.SetModelManager(AiModelManager, &m)
+	m.Id = modelId
+	m.Name = modelId
+	m.AiProviderId = providerId
+	m.ModelKey = modelKey
+	m.Description = description
+	m.SetEnabled(true)
+	m.Status = apis.STATUS_AVAILABLE
+	m.Progress = 100
+	if err := AiModelManager.TableSpec().Insert(ctx, &m); err != nil {
+		return errors.Wrapf(err, "insert ai_model %s/%s", providerKey, modelKey)
+	}
+	return nil
+}
+
+func ensureSeedModelsEntries(ctx context.Context, providerId, providerKey string, entries []catalogSeedModel) error {
+	if len(entries) == 0 {
+		return insertCatalogModel(ctx, providerId, providerKey, placeholderCatalogModelKey,
+			"Catalog seed placeholder; replace with concrete model_key values or use a provider with a built-in catalog.")
+	}
+	for i := range entries {
+		if err := insertCatalogModel(ctx, providerId, providerKey, entries[i].ModelKey, entries[i].Description); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func ensureSeedProvider(ctx context.Context, providerKey string) error {
+	providerKey = strings.TrimSpace(providerKey)
+	providerId := catalogProviderId(providerKey)
+	if err := insertCatalogProvider(ctx, providerKey,
+		fmt.Sprintf("Standard provider catalog entry: %s", providerKey),
+		standardProviderConfig(providerKey)); err != nil {
+		return err
+	}
+	return ensureSeedModelsEntries(ctx, providerId, providerKey, catalogSeedModelsForProvider(providerKey))
+}
+
+// SeedStandardCatalog inserts built-in ai_provider / ai_model catalog rows on first init only.
+// Existing rows are left unchanged so user config survives service restarts.
+func SeedStandardCatalog(ctx context.Context) error {
+	for _, pk := range standardCatalogProviderKeys {
+		if err := ensureSeedProvider(ctx, pk); err != nil {
+			return err
+		}
+	}
+	log.Infof("aiproxy: standard catalog seed completed (%d providers)", len(standardCatalogProviderKeys))
+	return nil
+}

pkg/aiproxy/models/catalog_seed_models.go

@@ -0,0 +1,286 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+// catalogSeedModel is one row to insert into ai_models when seeding a standard provider.
+// ModelKey is the id sent to the upstream API (no "provider/" prefix).
+type catalogSeedModel struct {
+	ModelKey    string
+	Description string
+}
+
+// catalogSeedModelsForProvider returns known public model ids for seeding.
+// Curated from vendor/provider docs; extend as products ship.
+// Providers without a list return nil and the seeder inserts model_key "default".
+func catalogSeedModelsForProvider(providerKey string) []catalogSeedModel {
+	switch providerKey {
+	case "anthropic":
+		return []catalogSeedModel{
+			{ModelKey: "claude-opus-4-20250514", Description: "Anthropic Claude Opus 4"},
+			{ModelKey: "claude-sonnet-4-20250514", Description: "Anthropic Claude Sonnet 4"},
+			{ModelKey: "claude-3-7-sonnet-20250219", Description: "Anthropic Claude 3.7 Sonnet"},
+			{ModelKey: "claude-3-5-sonnet-20241022", Description: "Anthropic Claude 3.5 Sonnet"},
+			{ModelKey: "claude-3-5-haiku-20241022", Description: "Anthropic Claude 3.5 Haiku"},
+			{ModelKey: "claude-3-opus-20240229", Description: "Anthropic Claude 3 Opus"},
+			{ModelKey: "claude-3-haiku-20240307", Description: "Anthropic Claude 3 Haiku"},
+		}
+	case "azure":
+		// Azure OpenAI uses deployment names; these match common Azure OpenAI deployment ids.
+		return []catalogSeedModel{
+			{ModelKey: "gpt-4o", Description: "Azure OpenAI GPT-4o deployment"},
+			{ModelKey: "gpt-4o-mini", Description: "Azure OpenAI GPT-4o mini deployment"},
+			{ModelKey: "gpt-4", Description: "Azure OpenAI GPT-4 deployment"},
+			{ModelKey: "gpt-35-turbo", Description: "Azure OpenAI GPT-3.5 Turbo deployment"},
+			{ModelKey: "o3-mini", Description: "Azure OpenAI o3-mini deployment"},
+		}
+	case "bedrock":
+		return []catalogSeedModel{
+			{ModelKey: "anthropic.claude-3-5-sonnet-20241022-v2:0", Description: "Bedrock Claude 3.5 Sonnet"},
+			{ModelKey: "anthropic.claude-3-5-haiku-20241022-v1:0", Description: "Bedrock Claude 3.5 Haiku"},
+			{ModelKey: "anthropic.claude-3-opus-20240229-v1:0", Description: "Bedrock Claude 3 Opus"},
+			{ModelKey: "anthropic.claude-3-sonnet-20240229-v1:0", Description: "Bedrock Claude 3 Sonnet"},
+			{ModelKey: "anthropic.claude-3-haiku-20240307-v1:0", Description: "Bedrock Claude 3 Haiku"},
+			{ModelKey: "meta.llama3-70b-instruct-v1:0", Description: "Bedrock Llama 3 70B Instruct"},
+			{ModelKey: "meta.llama3-8b-instruct-v1:0", Description: "Bedrock Llama 3 8B Instruct"},
+			{ModelKey: "mistral.mistral-large-2402-v1:0", Description: "Bedrock Mistral Large"},
+			{ModelKey: "amazon.titan-text-express-v1", Description: "Bedrock Amazon Titan Text Express"},
+		}
+	case "cerebras":
+		return []catalogSeedModel{
+			{ModelKey: "llama3.1-8b", Description: "Cerebras Llama 3.1 8B"},
+			{ModelKey: "llama3.1-70b", Description: "Cerebras Llama 3.1 70B"},
+			{ModelKey: "llama-3.3-70b", Description: "Cerebras Llama 3.3 70B"},
+		}
+	case "cohere":
+		return []catalogSeedModel{
+			{ModelKey: "command-r-plus", Description: "Cohere Command R+"},
+			{ModelKey: "command-r", Description: "Cohere Command R"},
+			{ModelKey: "command-a", Description: "Cohere Command A"},
+			{ModelKey: "command", Description: "Cohere Command"},
+			{ModelKey: "command-light", Description: "Cohere Command Light"},
+			{ModelKey: "embed-english-v3.0", Description: "Cohere Embed English v3"},
+			{ModelKey: "embed-multilingual-v3.0", Description: "Cohere Embed Multilingual v3"},
+		}
+	case "elevenlabs":
+		return []catalogSeedModel{
+			{ModelKey: "eleven_multilingual_v2", Description: "ElevenLabs multilingual v2"},
+			{ModelKey: "eleven_turbo_v2_5", Description: "ElevenLabs Turbo v2.5"},
+			{ModelKey: "eleven_flash_v2_5", Description: "ElevenLabs Flash v2.5"},
+			{ModelKey: "eleven_multilingual_v1", Description: "ElevenLabs multilingual v1"},
+		}
+	case "fireworks":
+		return []catalogSeedModel{
+			{ModelKey: "accounts/fireworks/models/llama-v3p1-8b-instruct", Description: "Fireworks Llama 3.1 8B Instruct"},
+			{ModelKey: "accounts/fireworks/models/llama-v3p1-70b-instruct", Description: "Fireworks Llama 3.1 70B Instruct"},
+			{ModelKey: "accounts/fireworks/models/llama-v3p3-70b-instruct", Description: "Fireworks Llama 3.3 70B Instruct"},
+			{ModelKey: "accounts/fireworks/models/mixtral-8x7b-instruct", Description: "Fireworks Mixtral 8x7B Instruct"},
+		}
+	case "gemini":
+		return []catalogSeedModel{
+			{ModelKey: "gemini-2.0-flash", Description: "Google Gemini 2.0 Flash"},
+			{ModelKey: "gemini-2.0-flash-lite", Description: "Google Gemini 2.0 Flash-Lite"},
+			{ModelKey: "gemini-1.5-pro", Description: "Google Gemini 1.5 Pro"},
+			{ModelKey: "gemini-1.5-flash", Description: "Google Gemini 1.5 Flash"},
+			{ModelKey: "gemini-1.5-flash-8b", Description: "Google Gemini 1.5 Flash 8B"},
+			{ModelKey: "gemini-embedding-001", Description: "Google Gemini Embedding 001"},
+		}
+	case "groq":
+		return []catalogSeedModel{
+			{ModelKey: "llama-3.3-70b-versatile", Description: "Groq Llama 3.3 70B Versatile"},
+			{ModelKey: "llama-3.1-8b-instant", Description: "Groq Llama 3.1 8B Instant"},
+			{ModelKey: "llama-3.1-70b-versatile", Description: "Groq Llama 3.1 70B Versatile"},
+			{ModelKey: "mixtral-8x7b-32768", Description: "Groq Mixtral 8x7B"},
+			{ModelKey: "gemma2-9b-it", Description: "Groq Gemma2 9B IT"},
+		}
+	case "huggingface":
+		return []catalogSeedModel{
+			{ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "HF Llama 3.1 8B Instruct"},
+			{ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "HF Llama 3.1 70B Instruct"},
+			{ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "HF Mistral 7B Instruct"},
+			{ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "HF Qwen2.5 72B Instruct"},
+		}
+	case "mistral":
+		return []catalogSeedModel{
+			{ModelKey: "mistral-large-latest", Description: "Mistral Large (latest)"},
+			{ModelKey: "mistral-small-latest", Description: "Mistral Small (latest)"},
+			{ModelKey: "pixtral-12b-2409", Description: "Mistral Pixtral 12B"},
+			{ModelKey: "codestral-latest", Description: "Mistral Codestral (latest)"},
+			{ModelKey: "ministral-8b-latest", Description: "Mistral Ministral 8B"},
+			{ModelKey: "open-mistral-nemo", Description: "Mistral Open Mistral Nemo"},
+			{ModelKey: "mixtral-8x22b", Description: "Mistral Mixtral 8x22B"},
+			{ModelKey: "mixtral-8x7b", Description: "Mistral Mixtral 8x7B"},
+		}
+	case "nebius":
+		return []catalogSeedModel{
+			{ModelKey: "deepseek-ai/DeepSeek-V3", Description: "Nebius DeepSeek V3"},
+			{ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "Nebius Qwen2.5 72B Instruct"},
+			{ModelKey: "meta-llama/Llama-3.3-70B-Instruct", Description: "Nebius Llama 3.3 70B Instruct"},
+		}
+	case "ollama":
+		return []catalogSeedModel{
+			{ModelKey: "llama3.2", Description: "Ollama Llama 3.2"},
+			{ModelKey: "llama3.1", Description: "Ollama Llama 3.1"},
+			{ModelKey: "mistral", Description: "Ollama Mistral"},
+			{ModelKey: "qwen2.5", Description: "Ollama Qwen 2.5"},
+			{ModelKey: "codellama", Description: "Ollama Code Llama"},
+			{ModelKey: "phi3", Description: "Ollama Phi 3"},
+		}
+	case "vllm":
+		return []catalogSeedModel{
+			{ModelKey: "meta-llama/Meta-Llama-3.1-8B-Instruct", Description: "vLLM Llama 3.1 8B Instruct"},
+			{ModelKey: "meta-llama/Meta-Llama-3.1-70B-Instruct", Description: "vLLM Llama 3.1 70B Instruct"},
+			{ModelKey: "Qwen/Qwen2.5-7B-Instruct", Description: "vLLM Qwen2.5 7B Instruct"},
+			{ModelKey: "Qwen/Qwen2.5-72B-Instruct", Description: "vLLM Qwen2.5 72B Instruct"},
+			{ModelKey: "mistralai/Mistral-7B-Instruct-v0.3", Description: "vLLM Mistral 7B Instruct"},
+		}
+	case "openai":
+		return []catalogSeedModel{
+			{ModelKey: "gpt-5-nano", Description: "OpenAI GPT-5 nano"},
+			{ModelKey: "gpt-5-mini", Description: "OpenAI GPT-5 mini"},
+			{ModelKey: "gpt-5", Description: "OpenAI GPT-5"},
+			{ModelKey: "gpt-5.1", Description: "OpenAI GPT-5.1"},
+			{ModelKey: "gpt-5.1-mini", Description: "OpenAI GPT-5.1 mini"},
+			{ModelKey: "gpt-5.1-codex", Description: "OpenAI GPT-5.1 Codex"},
+			{ModelKey: "gpt-5.1-codex-max", Description: "OpenAI GPT-5.1 Codex Max"},
+			{ModelKey: "gpt-5.2", Description: "OpenAI GPT-5.2"},
+			{ModelKey: "gpt-5.2-pro", Description: "OpenAI GPT-5.2 pro"},
+			{ModelKey: "gpt-5.2-codex", Description: "OpenAI GPT-5.2 Codex"},
+			{ModelKey: "gpt-4.1", Description: "OpenAI GPT-4.1"},
+			{ModelKey: "gpt-4.1-mini", Description: "OpenAI GPT-4.1 mini"},
+			{ModelKey: "gpt-4.1-nano", Description: "OpenAI GPT-4.1 nano"},
+			{ModelKey: "gpt-4o", Description: "OpenAI GPT-4o"},
+			{ModelKey: "gpt-4o-mini", Description: "OpenAI GPT-4o mini"},
+			{ModelKey: "chatgpt-4o-latest", Description: "OpenAI ChatGPT-4o latest"},
+			{ModelKey: "gpt-4-turbo", Description: "OpenAI GPT-4 Turbo"},
+			{ModelKey: "gpt-4", Description: "OpenAI GPT-4"},
+			{ModelKey: "gpt-3.5-turbo", Description: "OpenAI GPT-3.5 Turbo"},
+			{ModelKey: "o1", Description: "OpenAI o1"},
+			{ModelKey: "o1-mini", Description: "OpenAI o1-mini"},
+			{ModelKey: "o1-preview", Description: "OpenAI o1-preview"},
+			{ModelKey: "o3", Description: "OpenAI o3"},
+			{ModelKey: "o3-mini", Description: "OpenAI o3-mini"},
+			{ModelKey: "o4-mini", Description: "OpenAI o4-mini"},
+			{ModelKey: "text-embedding-3-small", Description: "OpenAI text-embedding-3-small"},
+			{ModelKey: "text-embedding-3-large", Description: "OpenAI text-embedding-3-large"},
+			{ModelKey: "text-embedding-ada-002", Description: "OpenAI text-embedding-ada-002"},
+		}
+	case "openrouter":
+		return []catalogSeedModel{
+			{ModelKey: "openai/gpt-4o", Description: "OpenRouter OpenAI GPT-4o"},
+			{ModelKey: "openai/gpt-4o-mini", Description: "OpenRouter OpenAI GPT-4o mini"},
+			{ModelKey: "anthropic/claude-3.5-sonnet", Description: "OpenRouter Claude 3.5 Sonnet"},
+			{ModelKey: "anthropic/claude-3.5-haiku", Description: "OpenRouter Claude 3.5 Haiku"},
+			{ModelKey: "google/gemini-2.0-flash-001", Description: "OpenRouter Gemini 2.0 Flash"},
+			{ModelKey: "meta-llama/llama-3.3-70b-instruct", Description: "OpenRouter Llama 3.3 70B Instruct"},
+			{ModelKey: "mistralai/mistral-large", Description: "OpenRouter Mistral Large"},
+		}
+	case "perplexity":
+		return []catalogSeedModel{
+			{ModelKey: "sonar", Description: "Perplexity Sonar"},
+			{ModelKey: "sonar-pro", Description: "Perplexity Sonar Pro"},
+			{ModelKey: "sonar-reasoning", Description: "Perplexity Sonar Reasoning"},
+			{ModelKey: "llama-3.1-sonar-small-128k-online", Description: "Perplexity Llama 3.1 Sonar Small online"},
+			{ModelKey: "llama-3.1-sonar-large-128k-online", Description: "Perplexity Llama 3.1 Sonar Large online"},
+		}
+	case "replicate":
+		return []catalogSeedModel{
+			{ModelKey: "meta/meta-llama-3-8b-instruct", Description: "Replicate Meta Llama 3 8B Instruct"},
+			{ModelKey: "meta/meta-llama-3-70b-instruct", Description: "Replicate Meta Llama 3 70B Instruct"},
+			{ModelKey: "mistralai/mixtral-8x7b-instruct-v0.1", Description: "Replicate Mixtral 8x7B Instruct"},
+		}
+	case "runway":
+		return []catalogSeedModel{
+			{ModelKey: "gen3a_turbo", Description: "Runway Gen-3 Alpha Turbo"},
+			{ModelKey: "gen3a", Description: "Runway Gen-3 Alpha"},
+			{ModelKey: "gen4_aleph", Description: "Runway Gen-4 Aleph"},
+		}
+	case "vertex":
+		return []catalogSeedModel{
+			{ModelKey: "gemini-2.0-flash", Description: "Vertex AI Gemini 2.0 Flash"},
+			{ModelKey: "gemini-1.5-pro", Description: "Vertex AI Gemini 1.5 Pro"},
+			{ModelKey: "gemini-1.5-flash", Description: "Vertex AI Gemini 1.5 Flash"},
+			{ModelKey: "publishers/google/models/gemini-1.5-pro", Description: "Vertex publisher path Gemini 1.5 Pro"},
+		}
+	case "xai":
+		return []catalogSeedModel{
+			{ModelKey: "grok-3", Description: "xAI Grok 3"},
+			{ModelKey: "grok-3-mini", Description: "xAI Grok 3 mini"},
+			{ModelKey: "grok-2-latest", Description: "xAI Grok 2 latest"},
+			{ModelKey: "grok-2-1212", Description: "xAI Grok 2 1212"},
+			{ModelKey: "grok-beta", Description: "xAI Grok beta"},
+		}
+	case "aliyun":
+		return aliyunQwenSeedModels()
+	case "baidu":
+		return baiduErnieSeedModels()
+	case "xiaomi":
+		return xiaomiMimoSeedModels()
+	default:
+		return nil
+	}
+}
+
+func aliyunQwenSeedModels() []catalogSeedModel {
+	return []catalogSeedModel{
+		{ModelKey: "qwen-turbo", Description: "Alibaba Qwen Turbo"},
+		{ModelKey: "qwen-plus", Description: "Alibaba Qwen Plus"},
+		{ModelKey: "qwen-max", Description: "Alibaba Qwen Max"},
+		{ModelKey: "qwen-long", Description: "Alibaba Qwen Long context"},
+		{ModelKey: "qwen-vl-max", Description: "Alibaba Qwen-VL Max"},
+		{ModelKey: "qwen-vl-plus", Description: "Alibaba Qwen-VL Plus"},
+		{ModelKey: "qwen-vl-ocr", Description: "Alibaba Qwen-VL OCR"},
+		{ModelKey: "qwen2.5-0.5b-instruct", Description: "Alibaba Qwen2.5 0.5B Instruct"},
+		{ModelKey: "qwen2.5-1.5b-instruct", Description: "Alibaba Qwen2.5 1.5B Instruct"},
+		{ModelKey: "qwen2.5-3b-instruct", Description: "Alibaba Qwen2.5 3B Instruct"},
+		{ModelKey: "qwen2.5-7b-instruct", Description: "Alibaba Qwen2.5 7B Instruct"},
+		{ModelKey: "qwen2.5-14b-instruct", Description: "Alibaba Qwen2.5 14B Instruct"},
+		{ModelKey: "qwen2.5-32b-instruct", Description: "Alibaba Qwen2.5 32B Instruct"},
+		{ModelKey: "qwen2.5-72b-instruct", Description: "Alibaba Qwen2.5 72B Instruct"},
+		{ModelKey: "qwen2.5-coder-7b-instruct", Description: "Alibaba Qwen2.5 Coder 7B Instruct"},
+		{ModelKey: "qwen2.5-coder-32b-instruct", Description: "Alibaba Qwen2.5 Coder 32B Instruct"},
+		{ModelKey: "qwen3-30b-a3b", Description: "Alibaba Qwen3 30B A3B MoE"},
+		{ModelKey: "qwen3-32b", Description: "Alibaba Qwen3 32B"},
+		{ModelKey: "qwen3-235b-a22b", Description: "Alibaba Qwen3 235B A22B MoE"},
+		{ModelKey: "qwen-math-plus", Description: "Alibaba Qwen Math Plus"},
+		{ModelKey: "qwen-coder-plus", Description: "Alibaba Qwen Coder Plus"},
+		{ModelKey: "text-embedding-v3", Description: "Alibaba text-embedding-v3"},
+		{ModelKey: "text-embedding-v4", Description: "Alibaba text-embedding-v4"},
+	}
+}
+
+func baiduErnieSeedModels() []catalogSeedModel {
+	return []catalogSeedModel{
+		{ModelKey: "ernie-4.0-turbo-8k", Description: "Baidu ERNIE 4.0 Turbo 8K"},
+		{ModelKey: "ernie-4.0-8k", Description: "Baidu ERNIE 4.0 8K"},
+		{ModelKey: "ernie-4.0-turbo-128k", Description: "Baidu ERNIE 4.0 Turbo 128K"},
+		{ModelKey: "ernie-3.5-8k", Description: "Baidu ERNIE 3.5 8K"},
+		{ModelKey: "ernie-3.5-128k", Description: "Baidu ERNIE 3.5 128K"},
+		{ModelKey: "ernie-speed-128k", Description: "Baidu ERNIE Speed 128K"},
+		{ModelKey: "ernie-lite-8k", Description: "Baidu ERNIE Lite 8K"},
+		{ModelKey: "ernie-char-8k", Description: "Baidu ERNIE Character 8K"},
+		{ModelKey: "embedding-v1", Description: "Baidu Wenxin embedding-v1"},
+		{ModelKey: "tao-8k", Description: "Baidu ERNIE Tao 8K"},
+	}
+}
+
+func xiaomiMimoSeedModels() []catalogSeedModel {
+	return []catalogSeedModel{
+		{ModelKey: "mimo-v2.5-pro", Description: "Xiaomi MiMo 2.5 Pro (flagship text)"},
+		{ModelKey: "mimo-v2-pro", Description: "Xiaomi MiMo 2 Pro"},
+		{ModelKey: "mimo-v2.5", Description: "Xiaomi MiMo 2.5 (multimodal text)"},
+		{ModelKey: "mimo-v2-omni", Description: "Xiaomi MiMo 2 Omni (multimodal)"},
+		{ModelKey: "mimo-v2-flash", Description: "Xiaomi MiMo 2 Flash (fast)"},
+	}
+}

pkg/aiproxy/models/chat_upstream.go

@@ -0,0 +1,255 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"database/sql"
+	stderrors "errors"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/rbacscope"
+
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+// ChatUpstream holds resolved upstream and the model id to send.
+type ChatUpstream struct {
+	BaseURL       string
+	APIKey        string
+	UpstreamModel string
+	ProviderKey   string
+	AiProviderId  string
+	AiKeyId       string
+
+	// VirtualKeyId and usage/rate snapshots come from the matched ai_virtual_key row.
+	VirtualKeyId        string
+	MaxTokensPerRequest int
+	RequestsPerMinute   int
+}
+
+func modelPatternMatches(pattern, requestedModel string) bool {
+	pattern = strings.TrimSpace(pattern)
+	if pattern == "" {
+		return true
+	}
+	rm := strings.TrimSpace(requestedModel)
+	if strings.HasSuffix(pattern, "*") {
+		return strings.HasPrefix(rm, strings.TrimSuffix(pattern, "*"))
+	}
+	return strings.EqualFold(pattern, rm)
+}
+
+func virtualKeyAllowsProvider(vk *SAiVirtualKey, prov *SAiProvider) bool {
+	if vk == nil || prov == nil {
+		return false
+	}
+	if vk.Limits == nil || len(vk.Limits.AllowedAiProviderIds) == 0 {
+		return true
+	}
+	for _, idOrName := range vk.Limits.AllowedAiProviderIds {
+		idOrName = strings.TrimSpace(idOrName)
+		if idOrName == "" {
+			continue
+		}
+		if idOrName == prov.Id || strings.EqualFold(idOrName, prov.Name) {
+			return true
+		}
+	}
+	return false
+}
+
+func loadEnabledVirtualKey(virtualKey string) (*SAiVirtualKey, error) {
+	virtualKey = strings.TrimSpace(virtualKey)
+	if virtualKey == "" {
+		return nil, errors.Wrap(httperrors.ErrInputParameter, "missing virtual key (Authorization: Bearer <vk> or X-Ai-Virtual-Key)")
+	}
+	vk := SAiVirtualKey{}
+	qvk := AiVirtualKeyManager.Query().Equals("virtual_key", virtualKey).Equals("enabled", true)
+	err := qvk.First(&vk)
+	if err != nil {
+		if stderrors.Is(err, sql.ErrNoRows) {
+			return nil, errors.Wrap(httperrors.ErrInvalidStatus, "virtual key not found or disabled")
+		}
+		return nil, errors.Wrap(err, "query ai_virtual_key")
+	}
+	if strings.TrimSpace(vk.ProjectId) == "" {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "virtual key has no project")
+	}
+	return &vk, nil
+}
+
+// listProjectRoutingsForVirtualKey returns enabled ai_routing rows owned by or shared with the virtual key's project.
+func listProjectRoutingsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredential, vk *SAiVirtualKey) ([]SAiRouting, error) {
+	if vk == nil {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "nil virtual key")
+	}
+	routings := make([]SAiRouting, 0, 16)
+	qr := AiRoutingManager.Query().Equals("enabled", true)
+	qr = AiRoutingManager.FilterByOwner(ctx, qr, AiRoutingManager, userCred, vk.GetOwnerId(), rbacscope.ScopeProject)
+	qr = qr.Asc("priority")
+	if err := qr.All(&routings); err != nil {
+		return nil, errors.Wrap(err, "list ai_routings for virtual key project")
+	}
+	return routings, nil
+}
+
+// pickRoutingForRequest chooses the first matching ai_routing (lowest priority value wins)
+// on the current aiproxy instance. When a matched rule is bound to another node, returns forbidden.
+func pickRoutingForRequest(routings []SAiRouting, reqModel, currentNodeId string) (*SAiRouting, error) {
+	var boundElsewhere *SAiRouting
+	for i := range routings {
+		r := &routings[i]
+		if !modelPatternMatches(r.ModelPattern, reqModel) {
+			continue
+		}
+		if !proxyNodeScopeMatches(r.AiProxyNodeId, currentNodeId) {
+			if boundElsewhere == nil && strings.TrimSpace(r.AiProxyNodeId) != "" {
+				boundElsewhere = r
+			}
+			continue
+		}
+		return r, nil
+	}
+	if boundElsewhere != nil {
+		return nil, errors.Wrapf(httperrors.ErrForbidden,
+			"ai_routing %q is bound to ai_proxy_node %q; use that instance endpoint",
+			boundElsewhere.Name, boundElsewhere.AiProxyNodeId)
+	}
+	return nil, nil
+}
+
+type resolvedCatalogModel struct {
+	provider *SAiProvider
+	model    *SAiModel
+}
+
+// resolveCatalogModelFromRouting picks ai_routing_models for the routing and loads catalog provider/model rows.
+func resolveCatalogModelFromRouting(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	vk *SAiVirtualKey,
+	routing *SAiRouting,
+	reqModel string,
+) (*resolvedCatalogModel, error) {
+	if routing == nil {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing")
+	}
+	providerId, modelId, err := pickAiRoutingModel(ctx, userCred, routing, reqModel)
+	if err != nil {
+		return nil, err
+	}
+
+	pObj, err := AiProviderManager.FetchByIdOrName(ctx, userCred, providerId)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch ai_provider")
+	}
+	prov := pObj.(*SAiProvider)
+	if !prov.GetEnabled() {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider disabled")
+	}
+	if !virtualKeyAllowsProvider(vk, prov) {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider not allowed for this virtual key")
+	}
+
+	mObj, err := AiModelManager.FetchByIdOrName(ctx, userCred, modelId)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetch ai_model")
+	}
+	mdl := mObj.(*SAiModel)
+	if !mdl.GetEnabled() {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model disabled")
+	}
+	if strings.TrimSpace(mdl.AiProviderId) != "" && mdl.AiProviderId != prov.Id {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model does not belong to resolved ai_provider")
+	}
+	return &resolvedCatalogModel{provider: prov, model: mdl}, nil
+}
+
+// ResolveChatUpstream resolves upstream URL, API key, and catalog model_key for a chat request:
+//  1. ai_virtual_key (auth + project scope)
+//  2. ai_routing in that project (model_pattern / optional proxy-node scope, priority)
+//  3. ai_routing_model -> ai_provider + ai_model
+//  4. ai_key rows for that provider matching the catalog model_key (weight), else provider.config api_key
+func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential, virtualKey string, body *jsonutils.JSONDict) (*ChatUpstream, error) {
+	vk, err := loadEnabledVirtualKey(virtualKey)
+	if err != nil {
+		return nil, err
+	}
+
+	reqModel, _ := body.GetString("model")
+	if strings.TrimSpace(reqModel) == "" {
+		return nil, errors.Wrap(httperrors.ErrInputParameter, "missing model in JSON body")
+	}
+
+	routings, err := listProjectRoutingsForVirtualKey(ctx, userCred, vk)
+	if err != nil {
+		return nil, err
+	}
+	routing, err := pickRoutingForRequest(routings, reqModel, CurrentProxyNodeId())
+	if err != nil {
+		return nil, err
+	}
+	if routing == nil {
+		return nil, errors.Wrap(httperrors.ErrNotFound, "no ai_routing matched for virtual key project on this aiproxy node")
+	}
+
+	resolved, err := resolveCatalogModelFromRouting(ctx, userCred, vk, routing, reqModel)
+	if err != nil {
+		return nil, err
+	}
+	prov := resolved.provider
+	mdl := resolved.model
+
+	if prov.Config == nil {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config is empty")
+	}
+	baseURL := prov.Config.ResolvedBaseURL()
+	if baseURL == "" {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_provider.config must include base_url")
+	}
+
+	upstreamModel := strings.TrimSpace(mdl.ModelKey)
+	if upstreamModel == "" {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_model.model_key is empty")
+	}
+
+	// Keys are scoped to ai_provider; routing on each ai_key matches the resolved catalog model_key.
+	keyRes, err := resolveUpstreamAPIKey(prov, upstreamModel)
+	if err != nil {
+		return nil, err
+	}
+	if keyRes == nil || keyRes.Secret == "" {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no api_key for ai_provider and catalog model")
+	}
+
+	up := &ChatUpstream{
+		BaseURL:       baseURL,
+		APIKey:        keyRes.Secret,
+		UpstreamModel: upstreamModel,
+		ProviderKey:   prov.ProviderKey,
+		AiProviderId:  prov.Id,
+		AiKeyId:       keyRes.AiKeyId,
+		VirtualKeyId:  vk.Id,
+	}
+	if vk.Limits != nil {
+		up.MaxTokensPerRequest = vk.Limits.MaxTokensPerRequest
+		up.RequestsPerMinute = vk.Limits.RequestsPerMinute
+	}
+	return up, nil
+}

pkg/aiproxy/models/doc.go

@@ -0,0 +1 @@
+package models // import "yunion.io/x/onecloud/pkg/aiproxy/models"

pkg/aiproxy/models/initdb.go

@@ -0,0 +1,46 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+func InitDB() error {
+	for _, manager := range []db.IModelManager{
+		/*
+		 * Important!!!
+		 * initialization order matters, do not change the order
+		 */
+		db.Metadata,
+
+		AiProviderManager,
+		AiModelManager,
+		AiKeyManager,
+		AiVirtualKeyManager,
+		AiRoutingManager,
+		AiRoutingModelManager,
+		AiProxyNodeManager,
+	} {
+		err := manager.InitializeData()
+		if err != nil {
+			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
+			return err
+		}
+	}
+	return nil
+}

pkg/aiproxy/models/list_models.go

@@ -0,0 +1,190 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"sort"
+	"strings"
+	"time"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+// ModelsListEntry is one OpenAI-compatible model object in GET /openai/v1/models.
+type ModelsListEntry struct {
+	ID      string `json:"id"`
+	Object  string `json:"object"`
+	Created int64  `json:"created"`
+	OwnedBy string `json:"owned_by"`
+}
+
+// ListModelsForVirtualKey returns OpenAI-compatible model ids reachable by the virtual key
+// on the current aiproxy node (project ai_routing -> ai_routing_model -> ai_model).
+func ListModelsForVirtualKey(ctx context.Context, userCred mcclient.TokenCredential, virtualKey string) ([]ModelsListEntry, error) {
+	vk, err := loadEnabledVirtualKey(virtualKey)
+	if err != nil {
+		return nil, err
+	}
+	rpm := 0
+	if vk.Limits != nil {
+		rpm = vk.Limits.RequestsPerMinute
+	}
+	if err := TakeVirtualKeyRequestsPerMinute(vk.Id, rpm); err != nil {
+		return nil, err
+	}
+	routings, err := listProjectRoutingsForVirtualKey(ctx, userCred, vk)
+	if err != nil {
+		return nil, err
+	}
+	currentNode := CurrentProxyNodeId()
+	routingIds := make([]string, 0, len(routings))
+	for i := range routings {
+		if proxyNodeScopeMatches(routings[i].AiProxyNodeId, currentNode) {
+			routingIds = append(routingIds, routings[i].Id)
+		}
+	}
+	if len(routingIds) == 0 {
+		return nil, nil
+	}
+
+	entries := make([]SAiRoutingModel, 0, 16)
+	q := AiRoutingModelManager.Query().In("ai_routing_id", routingIds).Equals("enabled", true)
+	if err := q.All(&entries); err != nil {
+		return nil, errors.Wrap(err, "list ai_routing_models")
+	}
+	if len(entries) == 0 {
+		return nil, nil
+	}
+
+	providerIds := make([]string, 0, len(entries))
+	modelIds := make([]string, 0, len(entries))
+	for i := range entries {
+		providerIds = append(providerIds, entries[i].AiProviderId)
+		modelIds = append(modelIds, entries[i].AiModelId)
+	}
+
+	providers, err := fetchEnabledAiProvidersByIds(providerIds)
+	if err != nil {
+		return nil, err
+	}
+	modelsById, err := fetchEnabledAiModelsByIds(modelIds)
+	if err != nil {
+		return nil, err
+	}
+
+	seen := make(map[string]ModelsListEntry, len(entries))
+	created := time.Now().Unix()
+	for i := range entries {
+		e := &entries[i]
+		prov := providers[e.AiProviderId]
+		mdl := modelsById[e.AiModelId]
+		if prov == nil || mdl == nil {
+			continue
+		}
+		if !virtualKeyAllowsProvider(vk, prov) {
+			continue
+		}
+		id := clientFacingModelID(e, mdl)
+		if id == "" {
+			continue
+		}
+		if _, ok := seen[id]; ok {
+			continue
+		}
+		seen[id] = ModelsListEntry{
+			ID:      id,
+			Object:  "model",
+			Created: created,
+			OwnedBy: strings.TrimSpace(prov.ProviderKey),
+		}
+	}
+	if len(seen) == 0 {
+		return nil, nil
+	}
+	out := make([]ModelsListEntry, 0, len(seen))
+	for _, item := range seen {
+		out = append(out, item)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].ID < out[j].ID
+	})
+	return out, nil
+}
+
+func clientFacingModelID(entry *SAiRoutingModel, mdl *SAiModel) string {
+	if entry != nil {
+		if mp := strings.TrimSpace(entry.ModelPattern); mp != "" && !strings.Contains(mp, "*") {
+			return mp
+		}
+	}
+	if mdl != nil {
+		return strings.TrimSpace(mdl.ModelKey)
+	}
+	return ""
+}
+
+func fetchEnabledAiProvidersByIds(ids []string) (map[string]*SAiProvider, error) {
+	ids = uniqueNonEmptyStrings(ids)
+	if len(ids) == 0 {
+		return map[string]*SAiProvider{}, nil
+	}
+	rows := make([]SAiProvider, 0, len(ids))
+	q := AiProviderManager.Query().In("id", ids).Equals("enabled", true)
+	if err := q.All(&rows); err != nil {
+		return nil, errors.Wrap(err, "list ai_providers")
+	}
+	out := make(map[string]*SAiProvider, len(rows))
+	for i := range rows {
+		out[rows[i].Id] = &rows[i]
+	}
+	return out, nil
+}
+
+func fetchEnabledAiModelsByIds(ids []string) (map[string]*SAiModel, error) {
+	ids = uniqueNonEmptyStrings(ids)
+	if len(ids) == 0 {
+		return map[string]*SAiModel{}, nil
+	}
+	rows := make([]SAiModel, 0, len(ids))
+	q := AiModelManager.Query().In("id", ids).Equals("enabled", true)
+	if err := q.All(&rows); err != nil {
+		return nil, errors.Wrap(err, "list ai_models")
+	}
+	out := make(map[string]*SAiModel, len(rows))
+	for i := range rows {
+		out[rows[i].Id] = &rows[i]
+	}
+	return out, nil
+}
+
+func uniqueNonEmptyStrings(in []string) []string {
+	seen := make(map[string]struct{}, len(in))
+	out := make([]string, 0, len(in))
+	for _, s := range in {
+		s = strings.TrimSpace(s)
+		if s == "" {
+			continue
+		}
+		if _, ok := seen[s]; ok {
+			continue
+		}
+		seen[s] = struct{}{}
+		out = append(out, s)
+	}
+	return out
+}

pkg/aiproxy/models/list_models_test.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import "testing"
+
+func TestClientFacingModelID(t *testing.T) {
+	mdl := &SAiModel{ModelKey: "gpt-4o-mini"}
+	if got := clientFacingModelID(&SAiRoutingModel{ModelPattern: "fast"}, mdl); got != "fast" {
+		t.Fatalf("expected alias fast, got %q", got)
+	}
+	if got := clientFacingModelID(&SAiRoutingModel{ModelPattern: "gpt-*"}, mdl); got != "gpt-4o-mini" {
+		t.Fatalf("expected catalog model_key for wildcard pattern, got %q", got)
+	}
+	if got := clientFacingModelID(&SAiRoutingModel{}, mdl); got != "gpt-4o-mini" {
+		t.Fatalf("expected catalog model_key, got %q", got)
+	}
+}
+
+func TestUniqueNonEmptyStrings(t *testing.T) {
+	out := uniqueNonEmptyStrings([]string{"a", "a", "", "b", "b"})
+	if len(out) != 2 || out[0] != "a" || out[1] != "b" {
+		t.Fatalf("unexpected dedupe result: %#v", out)
+	}
+}

pkg/aiproxy/models/proxy_node_local.go

@@ -0,0 +1,71 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/options"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+var localProxyNodeId string
+
+// InitLocalProxyNodeId records the ai_proxy_node id for this running aiproxy process.
+func InitLocalProxyNodeId(opts *options.SAiProxyOptions, isSlave bool) error {
+	if isSlave {
+		addr, err := AdvertiseAddressFromOptions(opts)
+		if err != nil {
+			return err
+		}
+		localProxyNodeId = aiProxyNodeId(addr)
+		return nil
+	}
+	localProxyNodeId = defaultPrimaryAiProxyNodeId
+	return nil
+}
+
+// CurrentProxyNodeId returns the ai_proxy_node id of this process.
+func CurrentProxyNodeId() string {
+	return localProxyNodeId
+}
+
+func validateAiProxyNodeId(ctx context.Context, userCred mcclient.TokenCredential, idOrName string) (string, error) {
+	idOrName = strings.TrimSpace(idOrName)
+	if idOrName == "" {
+		return "", nil
+	}
+	obj, err := AiProxyNodeManager.FetchByIdOrName(ctx, userCred, idOrName)
+	if err != nil {
+		return "", errors.Wrap(err, "fetch ai_proxy_node")
+	}
+	node := obj.(*SAiProxyNode)
+	if !node.GetEnabled() {
+		return "", errors.Wrapf(httperrors.ErrInvalidStatus, "ai_proxy_node %q is disabled", idOrName)
+	}
+	return node.Id, nil
+}
+
+func proxyNodeScopeMatches(routingNodeId, currentNodeId string) bool {
+	routingNodeId = strings.TrimSpace(routingNodeId)
+	if routingNodeId == "" {
+		return true
+	}
+	return routingNodeId == strings.TrimSpace(currentNodeId)
+}

pkg/aiproxy/models/virtual_key_guard.go

@@ -0,0 +1,63 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"strings"
+	"sync"
+
+	"golang.org/x/time/rate"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/httperrors"
+)
+
+var vkRpmLimiters sync.Map // virtual key id -> *rate.Limiter
+
+// TakeVirtualKeyRequestsPerMinute enforces an approximate per-minute request budget per virtual key (in-process).
+func TakeVirtualKeyRequestsPerMinute(vkId string, rpm int) error {
+	if rpm <= 0 || strings.TrimSpace(vkId) == "" {
+		return nil
+	}
+	limAny, _ := vkRpmLimiters.LoadOrStore(vkId, rate.NewLimiter(rate.Limit(float64(rpm))/60.0, rpm))
+	lim := limAny.(*rate.Limiter)
+	if !lim.Allow() {
+		return errors.Wrap(httperrors.ErrTooManyRequests, "virtual key request rate exceeded")
+	}
+	return nil
+}
+
+// EnforceVirtualKeyMaxTokens caps or injects max_tokens from virtual key limits.
+func EnforceVirtualKeyMaxTokens(body *jsonutils.JSONDict, lim *api.SAiVirtualKeyLimits) error {
+	if lim == nil || lim.MaxTokensPerRequest <= 0 {
+		return nil
+	}
+	cap := int64(lim.MaxTokensPerRequest)
+	if body.Contains("max_tokens") {
+		mt, err := body.Int("max_tokens")
+		if err != nil {
+			return errors.Wrap(httperrors.ErrInputParameter, "invalid max_tokens")
+		}
+		if mt > cap {
+			return errors.Wrap(httperrors.ErrInputParameter, "max_tokens exceeds virtual key limit")
+		}
+		return nil
+	}
+	body.Set("max_tokens", jsonutils.NewInt(cap))
+	return nil
+}

pkg/aiproxy/options/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options // import "yunion.io/x/onecloud/pkg/aiproxy/options"

pkg/aiproxy/options/options.go

@@ -0,0 +1,45 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+)
+
+type SAiProxyOptions struct {
+	common_options.CommonOptions
+	common_options.DBOptions
+
+	AdvertiseAddress             string `help:"Standby node address advertised to clients, e.g. http://10.0.0.2:30889; default derives from bind address and port" default:""`
+	NodeHeartbeatIntervalSeconds int    `help:"Interval in seconds for standby node registration heartbeat" default:"60"`
+}
+
+var (
+	Options SAiProxyOptions
+)
+
+func OnOptionsChange(oldO, newO interface{}) bool {
+	oldOpts := oldO.(*SAiProxyOptions)
+	newOpts := newO.(*SAiProxyOptions)
+
+	changed := false
+	if common_options.OnCommonOptionsChange(&oldOpts.CommonOptions, &newOpts.CommonOptions) {
+		changed = true
+	}
+	if common_options.OnDBOptionsChange(&oldOpts.DBOptions, &newOpts.DBOptions) {
+		changed = true
+	}
+	return changed
+}

pkg/aiproxy/policy/defaults.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	"yunion.io/x/pkg/util/rbacscope"
+
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	common_policy "yunion.io/x/onecloud/pkg/cloudcommon/policy"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+)
+
+const (
+	PolicyActionGet  = common_policy.PolicyActionGet
+	PolicyActionList = common_policy.PolicyActionList
+)
+
+var (
+	predefinedDefaultPolicies = []rbacutils.SRbacPolicy{
+		{
+			Auth:  true,
+			Scope: rbacscope.ScopeUser,
+			Rules: []rbacutils.SRbacRule{
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "ai_proxy_nodes",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  api.SERVICE_TYPE,
+					Resource: "ai_proxy_nodes",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
+			},
+		},
+	}
+)
+
+func Init() {
+	if consts.IsEnableDefaultPolicy() {
+		common_policy.AppendDefaultPolicies(predefinedDefaultPolicies)
+	}
+}

pkg/aiproxy/policy/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy // import "yunion.io/x/onecloud/pkg/aiproxy/policy"

pkg/aiproxy/policy/resources.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package policy
+
+import (
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	common_policy "yunion.io/x/onecloud/pkg/cloudcommon/policy"
+)
+
+var (
+	aiproxySystemResources = []string{}
+	aiproxyDomainResources = []string{}
+	aiproxyUserResources   = []string{
+		"ai_proxy_nodes",
+	}
+)
+
+func init() {
+	common_policy.RegisterSystemResources(api.SERVICE_TYPE, aiproxySystemResources)
+	common_policy.RegisterDomainResources(api.SERVICE_TYPE, aiproxyDomainResources)
+	common_policy.RegisterUserResources(api.SERVICE_TYPE, aiproxyUserResources)
+}

pkg/aiproxy/providerapi/doc.go

@@ -0,0 +1 @@
+package providerapi // import "yunion.io/x/onecloud/pkg/aiproxy/providerapi"

pkg/aiproxy/providerapi/stream.go

@@ -0,0 +1,20 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providerapi
+
+// ContextualStreamPassthrough allows providers to choose SSE passthrough per request.
+type ContextualStreamPassthrough interface {
+	OpenAIStreamPassthroughForContext(ctx *ChatContext) bool
+}

pkg/aiproxy/providerapi/types.go

@@ -0,0 +1,84 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package providerapi defines shared types and interfaces for aiproxy provider adapters.
+package providerapi // import "yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+
+import (
+	"yunion.io/x/jsonutils"
+)
+
+// ChatContext holds resolved upstream connectivity for one proxied request.
+type ChatContext struct {
+	ProviderKey   string
+	BaseURL       string
+	APIKey        string
+	UpstreamModel string
+}
+
+// HTTPRequest is the wire-format call sent to an upstream provider.
+type HTTPRequest struct {
+	Method  string
+	URL     string
+	Headers map[string]string
+	Body    []byte
+}
+
+// StreamChunk is one normalized OpenAI chat.completion.chunk SSE payload (JSON only, no "data:" prefix).
+type StreamChunk struct {
+	Data []byte
+	Done bool
+}
+
+// StreamState carries per-stream conversion state for providers that emit non-OpenAI SSE.
+type StreamState struct {
+	Model           string
+	ResponseID      string
+	ChunkIndex      int
+	TextStarted     bool
+	ToolIndex       int
+	ToolID          string
+	ToolName        string
+	ToolArgsPending string
+	InToolBlock     bool
+}
+
+// Provider converts OpenAI chat/completions to a provider-native HTTP call and
+// normalizes responses back to OpenAI format.
+type Provider interface {
+	Key() string
+	BuildUpstreamRequest(ctx *ChatContext, body *jsonutils.JSONDict, stream bool) (*HTTPRequest, error)
+	NormalizeResponse(body []byte) ([]byte, error)
+	OpenAIStreamPassthrough() bool
+	ConvertStreamEvent(eventType string, payload []byte, state *StreamState) ([]StreamChunk, error)
+}
+
+// EmbeddingsProvider converts OpenAI /v1/embeddings requests to provider-native APIs.
+type EmbeddingsProvider interface {
+	BuildEmbeddingsRequest(ctx *ChatContext, body *jsonutils.JSONDict) (*HTTPRequest, error)
+	NormalizeEmbeddingsResponse(body []byte) ([]byte, error)
+}
+
+// ImagesProvider converts OpenAI /v1/images/generations requests to provider-native APIs.
+type ImagesProvider interface {
+	BuildImagesGenerationsRequest(ctx *ChatContext, body *jsonutils.JSONDict) (*HTTPRequest, error)
+	NormalizeImagesGenerationsResponse(body []byte) ([]byte, error)
+}
+
+// CompletionsProvider converts OpenAI /v1/completions requests to provider-native APIs.
+type CompletionsProvider interface {
+	BuildCompletionsRequest(ctx *ChatContext, body *jsonutils.JSONDict, stream bool) (*HTTPRequest, error)
+	NormalizeCompletionsResponse(body []byte) ([]byte, error)
+	OpenAICompletionsStreamPassthrough() bool
+}

pkg/aiproxy/providers/aliyun/aliyun.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aliyun
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+func patchEnableThinkingFalse(body *jsonutils.JSONDict, stream bool) {
+	if stream {
+		return
+	}
+	if _, err := body.Get("enable_thinking"); err == nil {
+		return
+	}
+	body.Set("enable_thinking", jsonutils.JSONFalse)
+}
+
+// New returns the Aliyun (DashScope compatible-mode) provider adapter.
+func New() providerapi.Provider {
+	return openai.NewCompat("aliyun", patchEnableThinkingFalse)
+}

pkg/aiproxy/providers/aliyun/doc.go

@@ -0,0 +1 @@
+package aliyun // import "yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun"

pkg/aiproxy/providers/anthropic/anthropic.go

@@ -0,0 +1,285 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package anthropic
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+const apiVersion = "2023-06-01"
+
+type provider struct{}
+
+// New returns the Anthropic Messages API provider adapter.
+func New() providerapi.Provider {
+	return &provider{}
+}
+
+func (p *provider) Key() string {
+	return "anthropic"
+}
+
+func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	msgs, system, err := openai.ParseMessages(body)
+	if err != nil {
+		return nil, err
+	}
+	anthropicMsgs, err := openai.MessagesToAnthropic(msgs)
+	if err != nil {
+		return nil, err
+	}
+	maxTokens := 4096
+	if v, ok := openai.IntParam(body, "max_tokens", "max_completion_tokens"); ok {
+		maxTokens = v
+	}
+	reqBody := map[string]interface{}{
+		"model":      ctx.UpstreamModel,
+		"max_tokens": maxTokens,
+		"messages":   anthropicMsgs,
+	}
+	if system != "" {
+		reqBody["system"] = system
+	}
+	if stream {
+		reqBody["stream"] = true
+	}
+	if v, ok := openai.FloatParam(body, "temperature"); ok {
+		reqBody["temperature"] = v
+	}
+	if v, ok := openai.FloatParam(body, "top_p"); ok {
+		reqBody["top_p"] = v
+	}
+	if tools, toolChoice, err := openai.ExtractTools(body); err != nil {
+		return nil, err
+	} else if len(tools) > 0 {
+		reqBody["tools"] = openai.ToolsToAnthropic(tools)
+		if tc := openai.ToolChoiceToAnthropic(toolChoice); tc != nil {
+			reqBody["tool_choice"] = tc
+		}
+	}
+	raw, err := openai.MarshalJSON(reqBody)
+	if err != nil {
+		return nil, err
+	}
+	base := strings.TrimSpace(ctx.BaseURL)
+	if base == "" {
+		base = "https://api.anthropic.com"
+	}
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    openai.JoinURL(base, "/v1/messages"),
+		Headers: map[string]string{
+			"x-api-key":         strings.TrimSpace(ctx.APIKey),
+			"anthropic-version": apiVersion,
+			"Content-Type":      "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func (p *provider) NormalizeResponse(body []byte) ([]byte, error) {
+	var resp struct {
+		ID      string `json:"id"`
+		Model   string `json:"model"`
+		Content []struct {
+			Type  string                 `json:"type"`
+			Text  string                 `json:"text"`
+			ID    string                 `json:"id"`
+			Name  string                 `json:"name"`
+			Input map[string]interface{} `json:"input"`
+		} `json:"content"`
+		StopReason string `json:"stop_reason"`
+		Usage      struct {
+			InputTokens  int `json:"input_tokens"`
+			OutputTokens int `json:"output_tokens"`
+		} `json:"usage"`
+	}
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return body, nil
+	}
+	blocks := make([]openai.AnthropicBlock, len(resp.Content))
+	for i, c := range resp.Content {
+		blocks[i] = openai.AnthropicBlock{
+			Type:  c.Type,
+			Text:  c.Text,
+			ID:    c.ID,
+			Name:  c.Name,
+			Input: c.Input,
+		}
+	}
+	msg := openai.AnthropicBlocksToAssistant(blocks)
+	out, err := openai.MarshalJSON(openai.NewChatCompletionWithTools(
+		resp.Model,
+		resp.ID,
+		msg,
+		resp.StopReason,
+		resp.Usage.InputTokens,
+		resp.Usage.OutputTokens,
+	))
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (p *provider) OpenAIStreamPassthrough() bool {
+	return false
+}
+
+func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) {
+	if state == nil || len(payload) == 0 {
+		return nil, nil
+	}
+	var wrap struct {
+		Type string `json:"type"`
+	}
+	if err := json.Unmarshal(payload, &wrap); err != nil {
+		return nil, nil
+	}
+	switch wrap.Type {
+	case "message_start":
+		var start struct {
+			Message struct {
+				ID    string `json:"id"`
+				Model string `json:"model"`
+			} `json:"message"`
+		}
+		if err := json.Unmarshal(payload, &start); err == nil {
+			if start.Message.ID != "" {
+				state.ResponseID = start.Message.ID
+			}
+			if start.Message.Model != "" {
+				state.Model = start.Message.Model
+			}
+		}
+		return nil, nil
+	case "content_block_start":
+		var start struct {
+			ContentBlock struct {
+				Type string `json:"type"`
+				ID   string `json:"id"`
+				Name string `json:"name"`
+			} `json:"content_block"`
+		}
+		if err := json.Unmarshal(payload, &start); err != nil {
+			return nil, nil
+		}
+		if start.ContentBlock.Type != "tool_use" {
+			return nil, nil
+		}
+		state.InToolBlock = true
+		state.ToolID = start.ContentBlock.ID
+		state.ToolName = start.ContentBlock.Name
+		state.ToolArgsPending = ""
+		chunk, err := openai.MarshalJSON(openai.NewStreamChunkToolDelta(
+			state.Model, state.ResponseID, state.ToolIndex,
+			openai.ToolCall{
+				ID:   state.ToolID,
+				Type: "function",
+				Function: openai.ToolFunction{
+					Name: state.ToolName,
+				},
+			}, "",
+		))
+		if err != nil {
+			return nil, err
+		}
+		return []providerapi.StreamChunk{{Data: chunk}}, nil
+	case "content_block_delta":
+		var delta struct {
+			Delta struct {
+				Type        string `json:"type"`
+				Text        string `json:"text"`
+				PartialJSON string `json:"partial_json"`
+			} `json:"delta"`
+		}
+		if err := json.Unmarshal(payload, &delta); err != nil {
+			return nil, nil
+		}
+		if delta.Delta.Type == "input_json_delta" && delta.Delta.PartialJSON != "" {
+			state.ToolArgsPending += delta.Delta.PartialJSON
+			chunk, err := openai.MarshalJSON(openai.NewStreamChunkToolDelta(
+				state.Model, state.ResponseID, state.ToolIndex,
+				openai.ToolCall{
+					Function: openai.ToolFunction{
+						Arguments: delta.Delta.PartialJSON,
+					},
+				}, "",
+			))
+			if err != nil {
+				return nil, err
+			}
+			return []providerapi.StreamChunk{{Data: chunk}}, nil
+		}
+		if delta.Delta.Text == "" {
+			return nil, nil
+		}
+		chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, delta.Delta.Text, ""))
+		if err != nil {
+			return nil, err
+		}
+		return []providerapi.StreamChunk{{Data: chunk}}, nil
+	case "content_block_stop":
+		if state.InToolBlock {
+			state.InToolBlock = false
+			state.ToolIndex++
+			state.ToolArgsPending = ""
+		}
+		return nil, nil
+	case "message_delta":
+		var end struct {
+			Delta struct {
+				StopReason string `json:"stop_reason"`
+			} `json:"delta"`
+		}
+		if err := json.Unmarshal(payload, &end); err != nil {
+			return nil, nil
+		}
+		chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, "", end.Delta.StopReason))
+		if err != nil {
+			return nil, err
+		}
+		return []providerapi.StreamChunk{{Data: chunk}}, nil
+	default:
+		return nil, nil
+	}
+}
+
+func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	return nil, fmt.Errorf("provider %q does not support embeddings", p.Key())
+}
+
+func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	return nil, fmt.Errorf("provider %q does not support embeddings", p.Key())
+}
+
+func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	return nil, fmt.Errorf("provider %q does not support images/generations", p.Key())
+}
+
+func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) {
+	return nil, fmt.Errorf("provider %q does not support images/generations", p.Key())
+}

pkg/aiproxy/providers/anthropic/doc.go

@@ -0,0 +1 @@
+package anthropic // import "yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic"

pkg/aiproxy/providers/azure/azure.go

@@ -0,0 +1,117 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+type provider struct{}
+
+// New returns the Azure OpenAI provider adapter.
+func New() providerapi.Provider {
+	return &provider{}
+}
+
+func (p *provider) Key() string {
+	return "azure"
+}
+
+func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	return p.buildRequest(ctx, body, "chat/completions")
+}
+
+func (p *provider) NormalizeResponse(body []byte) ([]byte, error) {
+	return body, nil
+}
+
+func (p *provider) OpenAIStreamPassthrough() bool {
+	return true
+}
+
+func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) {
+	return nil, nil
+}
+
+func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	return p.buildRequest(ctx, body, "embeddings")
+}
+
+func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	return body, nil
+}
+
+func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	return p.buildRequest(ctx, body, "images/generations")
+}
+
+func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) {
+	return body, nil
+}
+
+func (p *provider) buildRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, action string) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	dup := openai.CloneBodyWithModel(body, ctx.UpstreamModel)
+	url, err := deploymentURL(ctx.BaseURL, ctx.UpstreamModel, body, action)
+	if err != nil {
+		return nil, err
+	}
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    url,
+		Headers: map[string]string{
+			"api-key":      strings.TrimSpace(ctx.APIKey),
+			"Content-Type": "application/json",
+		},
+		Body: []byte(dup.String()),
+	}, nil
+}
+
+func deploymentURL(baseURL, deployment string, body *jsonutils.JSONDict, action string) (string, error) {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if base == "" {
+		return "", fmt.Errorf("azure provider requires base_url (resource endpoint)")
+	}
+	url := base
+	suffix := "/" + action
+	if strings.Contains(url, "/"+action) {
+		return url, nil
+	}
+	if strings.HasSuffix(url, "/v1") {
+		return url + suffix, nil
+	}
+	if strings.Contains(url, "/openai/deployments/") {
+		return url + suffix, nil
+	}
+	url = openai.JoinURL(url, fmt.Sprintf("openai/deployments/%s/%s", deployment, action))
+	if body != nil {
+		if v, err := body.Get("api-version"); err == nil {
+			ver, _ := v.GetString()
+			if ver != "" {
+				url = url + "?api-version=" + ver
+			}
+		}
+	}
+	return url, nil
+}

pkg/aiproxy/providers/azure/doc.go

@@ -0,0 +1 @@
+package azure // import "yunion.io/x/onecloud/pkg/aiproxy/providers/azure"

pkg/aiproxy/providers/baidu/baidu.go

@@ -0,0 +1,103 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package baidu
+
+import (
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+type provider struct {
+	v2 *openai.Compat
+}
+
+// New returns the Baidu Wenxin / Qianfan provider adapter.
+func New() providerapi.Provider {
+	return &provider{v2: openai.NewCompat("baidu")}
+}
+
+func (p *provider) Key() string {
+	return "baidu"
+}
+
+func (p *provider) useV2(ctx *providerapi.ChatContext) bool {
+	if ctx == nil {
+		return true
+	}
+	return useQianfanV2(ctx.BaseURL)
+}
+
+func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	if p.useV2(ctx) {
+		return p.v2.BuildUpstreamRequest(ctx, body, stream)
+	}
+	return buildWenxinV1ChatRequest(ctx, body, stream)
+}
+
+func (p *provider) NormalizeResponse(body []byte) ([]byte, error) {
+	if p.v2 != nil {
+		// Try wenxin v1 only when response looks non-OpenAI.
+		if out, err := normalizeWenxinV1ChatResponse(body); err != nil {
+			return nil, err
+		} else if string(out) != string(body) {
+			return out, nil
+		}
+	}
+	return body, nil
+}
+
+func (p *provider) OpenAIStreamPassthrough() bool {
+	return false
+}
+
+func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) {
+	return convertWenxinV1StreamEvent(payload, state)
+}
+
+func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if p.useV2(ctx) {
+		return p.v2.BuildEmbeddingsRequest(ctx, body)
+	}
+	return buildWenxinV1EmbeddingsRequest(ctx, body)
+}
+
+func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	if out, err := normalizeWenxinV1EmbeddingsResponse(body); err != nil {
+		return nil, err
+	} else if len(out) > 0 && string(out) != string(body) {
+		return out, nil
+	}
+	return body, nil
+}
+
+func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if p.useV2(ctx) {
+		return p.v2.BuildImagesGenerationsRequest(ctx, body)
+	}
+	return nil, fmt.Errorf("provider %q wenxin v1 does not support images/generations; use qianfan v2 base_url", p.Key())
+}
+
+func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) {
+	return body, nil
+}
+
+// OpenAIStreamPassthroughForContext reports whether upstream SSE is already OpenAI-compatible.
+func (p *provider) OpenAIStreamPassthroughForContext(ctx *providerapi.ChatContext) bool {
+	return p.useV2(ctx)
+}

pkg/aiproxy/providers/baidu/baidu_test.go

@@ -0,0 +1,116 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package baidu
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+func TestQianfanV2ChatBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("ernie-3.5-8k"), "model")
+	userMsg := jsonutils.NewDict()
+	userMsg.Set("role", jsonutils.NewString("user"))
+	userMsg.Set("content", jsonutils.NewString("你好"))
+	body.Add(jsonutils.NewArray(userMsg), "messages")
+
+	p := New()
+	req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{
+		BaseURL:       "https://qianfan.baidubce.com/v2",
+		APIKey:        "bce-v3/ALTAK-test",
+		UpstreamModel: "ernie-3.5-8k",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://qianfan.baidubce.com/v2/chat/completions" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+	if req.Headers["Authorization"] != "Bearer bce-v3/ALTAK-test" {
+		t.Fatalf("missing bearer auth: %#v", req.Headers)
+	}
+}
+
+func TestWenxinV1ChatBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	userMsg := jsonutils.NewDict()
+	userMsg.Set("role", jsonutils.NewString("user"))
+	userMsg.Set("content", jsonutils.NewString("你好"))
+	body.Add(jsonutils.NewArray(userMsg), "messages")
+
+	p := New()
+	req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{
+		BaseURL:       "https://aip.baidubce.com",
+		APIKey:        "test-access-token",
+		UpstreamModel: "eb-instant",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(req.URL, "wenxinworkshop/chat/eb-instant") {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestWenxinV1NormalizeResponse(t *testing.T) {
+	p := New()
+	out, err := p.NormalizeResponse([]byte(`{
+		"id":"as-1",
+		"result":"你好",
+		"usage":{"prompt_tokens":2,"completion_tokens":1,"total_tokens":3}
+	}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp["object"] != "chat.completion" {
+		t.Fatalf("unexpected object: %#v", resp["object"])
+	}
+}
+
+func TestStreamPassthroughV2(t *testing.T) {
+	p := New().(providerapi.ContextualStreamPassthrough)
+	if !p.OpenAIStreamPassthroughForContext(&providerapi.ChatContext{
+		BaseURL: "https://qianfan.baidubce.com/v2",
+	}) {
+		t.Fatal("qianfan v2 should passthrough SSE")
+	}
+	if p.OpenAIStreamPassthroughForContext(&providerapi.ChatContext{
+		BaseURL: "https://aip.baidubce.com",
+	}) {
+		t.Fatal("wenxin v1 should not passthrough SSE")
+	}
+}
+
+func TestUseQianfanV2(t *testing.T) {
+	if !useQianfanV2("") {
+		t.Fatal("empty base should default to v2")
+	}
+	if !useQianfanV2("https://qianfan.baidubce.com/v2") {
+		t.Fatal("qianfan host should be v2")
+	}
+	if useQianfanV2("https://aip.baidubce.com") {
+		t.Fatal("aip host should be v1")
+	}
+}

pkg/aiproxy/providers/baidu/doc.go

@@ -0,0 +1 @@
+package baidu // import "yunion.io/x/onecloud/pkg/aiproxy/providers/baidu"

pkg/aiproxy/providers/baidu/token.go

@@ -0,0 +1,146 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package baidu
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	defaultTokenURL = "https://aip.baidubce.com/oauth/2.0/token"
+	tokenSkew       = 5 * time.Minute
+)
+
+type cachedToken struct {
+	value  string
+	expiry time.Time
+}
+
+var tokenCache sync.Map
+
+// ResolveAccessToken returns a Wenxin access_token.
+// apiKey may be a raw access_token, or "APIKey:SecretKey" for OAuth exchange.
+func ResolveAccessToken(apiKey string) (string, error) {
+	apiKey = strings.TrimSpace(apiKey)
+	if apiKey == "" {
+		return "", fmt.Errorf("empty baidu api key")
+	}
+	if !strings.Contains(apiKey, ":") {
+		return apiKey, nil
+	}
+	parts := strings.SplitN(apiKey, ":", 2)
+	clientID := strings.TrimSpace(parts[0])
+	clientSecret := strings.TrimSpace(parts[1])
+	if clientID == "" || clientSecret == "" {
+		return "", fmt.Errorf("invalid baidu api key format, want access_token or APIKey:SecretKey")
+	}
+	cacheKey := clientID + ":" + clientSecret
+	if v, ok := tokenCache.Load(cacheKey); ok {
+		entry := v.(cachedToken)
+		if time.Now().Before(entry.expiry.Add(-tokenSkew)) {
+			return entry.value, nil
+		}
+	}
+	token, expiresIn, err := fetchAccessToken(clientID, clientSecret)
+	if err != nil {
+		return "", err
+	}
+	if expiresIn <= 0 {
+		expiresIn = 30 * 24 * time.Hour
+	}
+	tokenCache.Store(cacheKey, cachedToken{
+		value:  token,
+		expiry: time.Now().Add(expiresIn),
+	})
+	return token, nil
+}
+
+func fetchAccessToken(clientID, clientSecret string) (string, time.Duration, error) {
+	q := url.Values{}
+	q.Set("grant_type", "client_credentials")
+	q.Set("client_id", clientID)
+	q.Set("client_secret", clientSecret)
+	req, err := http.NewRequest(http.MethodPost, defaultTokenURL+"?"+q.Encode(), nil)
+	if err != nil {
+		return "", 0, err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", 0, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return "", 0, err
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return "", 0, fmt.Errorf("baidu oauth HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
+	}
+	var wrap struct {
+		AccessToken string `json:"access_token"`
+		ExpiresIn   int64  `json:"expires_in"`
+		Error       string `json:"error"`
+		ErrorDesc   string `json:"error_description"`
+	}
+	if err := json.Unmarshal(body, &wrap); err != nil {
+		return "", 0, err
+	}
+	if wrap.AccessToken == "" {
+		msg := wrap.ErrorDesc
+		if msg == "" {
+			msg = wrap.Error
+		}
+		if msg == "" {
+			msg = string(body)
+		}
+		return "", 0, fmt.Errorf("baidu oauth failed: %s", msg)
+	}
+	return wrap.AccessToken, time.Duration(wrap.ExpiresIn) * time.Second, nil
+}
+
+func wenxinBaseURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if base == "" {
+		return "https://aip.baidubce.com"
+	}
+	return base
+}
+
+func wenxinChatURL(baseURL, model, accessToken string) string {
+	path := fmt.Sprintf("/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/%s", strings.TrimSpace(model))
+	u := wenxinBaseURL(baseURL) + path
+	return u + "?access_token=" + url.QueryEscape(accessToken)
+}
+
+func wenxinEmbeddingsURL(baseURL, model, accessToken string) string {
+	path := fmt.Sprintf("/rpc/2.0/ai_custom/v1/wenxinworkshop/embeddings/%s", strings.TrimSpace(model))
+	u := wenxinBaseURL(baseURL) + path
+	return u + "?access_token=" + url.QueryEscape(accessToken)
+}
+
+func useQianfanV2(baseURL string) bool {
+	base := strings.ToLower(strings.TrimSpace(baseURL))
+	if base == "" {
+		return true
+	}
+	return strings.Contains(base, "qianfan.baidubce.com")
+}

pkg/aiproxy/providers/baidu/wenxin_v1.go

@@ -0,0 +1,235 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package baidu
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+func buildWenxinV1ChatRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	token, err := ResolveAccessToken(ctx.APIKey)
+	if err != nil {
+		return nil, err
+	}
+	msgs, _, err := openai.ParseMessages(body)
+	if err != nil {
+		return nil, err
+	}
+	wenxinMsgs := make([]map[string]interface{}, 0, len(msgs))
+	for _, m := range msgs {
+		role := strings.ToLower(strings.TrimSpace(m.Role))
+		if role == "tool" {
+			role = "user"
+		}
+		text := openai.MessageTextContent(m.Content)
+		if text == "" {
+			continue
+		}
+		wenxinMsgs = append(wenxinMsgs, map[string]interface{}{
+			"role":    role,
+			"content": text,
+		})
+	}
+	if len(wenxinMsgs) == 0 {
+		return nil, fmt.Errorf("no convertible messages for wenxin")
+	}
+	reqBody := map[string]interface{}{
+		"messages": wenxinMsgs,
+		"stream":   stream,
+	}
+	if v, ok := openai.IntParam(body, "max_tokens", "max_completion_tokens"); ok {
+		reqBody["max_output_tokens"] = v
+	}
+	if v, ok := openai.FloatParam(body, "temperature"); ok {
+		reqBody["temperature"] = v
+	}
+	if v, ok := openai.FloatParam(body, "top_p"); ok {
+		reqBody["top_p"] = v
+	}
+	raw, err := openai.MarshalJSON(reqBody)
+	if err != nil {
+		return nil, err
+	}
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    wenxinChatURL(ctx.BaseURL, ctx.UpstreamModel, token),
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func normalizeWenxinV1ChatResponse(body []byte) ([]byte, error) {
+	var probe struct {
+		Choices json.RawMessage `json:"choices"`
+		Result  *string         `json:"result"`
+	}
+	if err := json.Unmarshal(body, &probe); err != nil {
+		return body, nil
+	}
+	if len(probe.Choices) > 0 {
+		return body, nil
+	}
+	if probe.Result == nil {
+		return body, nil
+	}
+	var resp struct {
+		ID      string `json:"id"`
+		Object  string `json:"object"`
+		Created int64  `json:"created"`
+		Result  string `json:"result"`
+		Usage   struct {
+			PromptTokens     int `json:"prompt_tokens"`
+			CompletionTokens int `json:"completion_tokens"`
+			TotalTokens      int `json:"total_tokens"`
+		} `json:"usage"`
+		ErrorCode int    `json:"error_code"`
+		ErrorMsg  string `json:"error_msg"`
+	}
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return body, nil
+	}
+	if resp.ErrorCode != 0 || resp.ErrorMsg != "" {
+		return nil, fmt.Errorf("wenxin error %d: %s", resp.ErrorCode, resp.ErrorMsg)
+	}
+	if resp.Result == "" && resp.ID == "" {
+		return body, nil
+	}
+	out, err := openai.MarshalJSON(openai.NewChatCompletion(
+		"",
+		resp.ID,
+		resp.Result,
+		"stop",
+		resp.Usage.PromptTokens,
+		resp.Usage.CompletionTokens,
+	))
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func convertWenxinV1StreamEvent(payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) {
+	if state == nil || len(payload) == 0 {
+		return nil, nil
+	}
+	var chunk struct {
+		ID      string `json:"id"`
+		Result  string `json:"result"`
+		IsEnd   bool   `json:"is_end"`
+		IsTrunc bool   `json:"is_truncated"`
+	}
+	if err := json.Unmarshal(payload, &chunk); err != nil {
+		return nil, nil
+	}
+	if chunk.ID != "" {
+		state.ResponseID = chunk.ID
+	}
+	finish := ""
+	if chunk.IsEnd {
+		finish = "stop"
+	}
+	if chunk.Result == "" && finish == "" {
+		return nil, nil
+	}
+	data, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, chunk.Result, finish))
+	if err != nil {
+		return nil, err
+	}
+	return []providerapi.StreamChunk{{Data: data}}, nil
+}
+
+func buildWenxinV1EmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	token, err := ResolveAccessToken(ctx.APIKey)
+	if err != nil {
+		return nil, err
+	}
+	texts, rawInput, isText, err := openai.ParseEmbeddingInput(body)
+	if err != nil {
+		return nil, err
+	}
+	var reqBody map[string]interface{}
+	if isText {
+		reqBody = map[string]interface{}{"input": texts}
+	} else {
+		if err := json.Unmarshal(rawInput, &reqBody); err != nil {
+			return nil, fmt.Errorf("invalid embeddings input")
+		}
+	}
+	raw, err := openai.MarshalJSON(reqBody)
+	if err != nil {
+		return nil, err
+	}
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    wenxinEmbeddingsURL(ctx.BaseURL, ctx.UpstreamModel, token),
+		Headers: map[string]string{
+			"Content-Type": "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func normalizeWenxinV1EmbeddingsResponse(body []byte) ([]byte, error) {
+	var probe struct {
+		Object  string          `json:"object"`
+		Choices json.RawMessage `json:"choices"`
+	}
+	if err := json.Unmarshal(body, &probe); err == nil && (probe.Object == "list" || len(probe.Choices) > 0) {
+		return body, nil
+	}
+	var resp struct {
+		Data []struct {
+			Embedding []float64 `json:"embedding"`
+			Index     int       `json:"index"`
+		} `json:"data"`
+		Usage struct {
+			PromptTokens int `json:"prompt_tokens"`
+			TotalTokens  int `json:"total_tokens"`
+		} `json:"usage"`
+		ErrorCode int    `json:"error_code"`
+		ErrorMsg  string `json:"error_msg"`
+	}
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return body, nil
+	}
+	if resp.ErrorCode != 0 || resp.ErrorMsg != "" {
+		return nil, fmt.Errorf("wenxin embeddings error %d: %s", resp.ErrorCode, resp.ErrorMsg)
+	}
+	vectors := make([][]float64, len(resp.Data))
+	for i := range resp.Data {
+		vectors[i] = resp.Data[i].Embedding
+	}
+	promptTokens := resp.Usage.PromptTokens
+	if promptTokens == 0 {
+		promptTokens = resp.Usage.TotalTokens
+	}
+	return openai.NewEmbeddingsResponse("", vectors, promptTokens)
+}

pkg/aiproxy/providers/bridge.go

@@ -0,0 +1,33 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"yunion.io/x/onecloud/pkg/aiproxy/upstream"
+)
+
+// ToUpstreamRequest maps a provider HTTPRequest to the shared upstream client shape.
+func ToUpstreamRequest(r *HTTPRequest, fallbackAPIKey string) *upstream.Request {
+	if r == nil {
+		return nil
+	}
+	return &upstream.Request{
+		BaseURL: "",
+		URL:     r.URL,
+		APIKey:  fallbackAPIKey,
+		Headers: r.Headers,
+		Body:    r.Body,
+	}
+}

pkg/aiproxy/providers/cohere/cohere.go

@@ -0,0 +1,94 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cohere
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+type provider struct {
+	*openai.Compat
+}
+
+// New returns the Cohere provider adapter (OpenAI-compatible chat, native embeddings).
+func New() providerapi.Provider {
+	return &provider{Compat: openai.NewCompat("cohere")}
+}
+
+func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	texts, _, isText, err := openai.ParseEmbeddingInput(body)
+	if err != nil {
+		return nil, err
+	}
+	if !isText {
+		return nil, fmt.Errorf("cohere embeddings requires string or string[] input")
+	}
+	reqBody := map[string]interface{}{
+		"model":           ctx.UpstreamModel,
+		"texts":           texts,
+		"input_type":      embeddingInputType(body),
+		"embedding_types": []string{"float"},
+	}
+	raw, err := openai.MarshalJSON(reqBody)
+	if err != nil {
+		return nil, err
+	}
+	base := strings.TrimSpace(ctx.BaseURL)
+	if base == "" {
+		base = "https://api.cohere.ai"
+	}
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    openai.JoinURL(base, "/v2/embed"),
+		Headers: map[string]string{
+			"Authorization": "Bearer " + strings.TrimSpace(ctx.APIKey),
+			"Content-Type":  "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func embeddingInputType(body *jsonutils.JSONDict) string {
+	if body == nil {
+		return "search_document"
+	}
+	if v, err := body.GetString("input_type"); err == nil && strings.TrimSpace(v) != "" {
+		return strings.TrimSpace(v)
+	}
+	return "search_document"
+}
+
+func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	var resp struct {
+		Embeddings struct {
+			Float [][]float64 `json:"float"`
+		} `json:"embeddings"`
+	}
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return body, nil
+	}
+	return openai.NewEmbeddingsResponse("", resp.Embeddings.Float, 0)
+}

pkg/aiproxy/providers/cohere/doc.go

@@ -0,0 +1 @@
+package cohere // import "yunion.io/x/onecloud/pkg/aiproxy/providers/cohere"

pkg/aiproxy/providers/completions.go

@@ -0,0 +1,28 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"fmt"
+)
+
+// GetCompletions returns the legacy completions adapter for providerKey.
+func GetCompletions(providerKey string) (CompletionsProvider, error) {
+	p := Get(providerKey)
+	if cp, ok := p.(CompletionsProvider); ok {
+		return cp, nil
+	}
+	return nil, fmt.Errorf("provider %q does not support /v1/completions", providerKey)
+}

pkg/aiproxy/providers/doc.go

@@ -0,0 +1,5 @@
+// Package providers converts OpenAI-compatible API requests to upstream AI provider APIs.
+//
+// Vendor-specific adapters live in subdirectories (openai/, anthropic/, gemini/, ...).
+// Shared types are defined in providerapi; this package exposes registry lookup helpers.
+package providers // import "yunion.io/x/onecloud/pkg/aiproxy/providers"

pkg/aiproxy/providers/embeddings.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+var defaultEmbeddings EmbeddingsProvider = openai.DefaultEmbeddingCompat()
+
+// GetEmbeddings returns the embeddings adapter for providerKey, or OpenAI-compatible passthrough.
+func GetEmbeddings(providerKey string) EmbeddingsProvider {
+	p := Get(providerKey)
+	if ep, ok := p.(EmbeddingsProvider); ok {
+		return ep
+	}
+	return defaultEmbeddings
+}

pkg/aiproxy/providers/embeddings_test.go

@@ -0,0 +1,121 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"encoding/json"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestOpenAIEmbeddingsCompatBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("text-embedding-3-small"), "model")
+	body.Add(jsonutils.NewString("hello"), "input")
+
+	p := GetEmbeddings("openai")
+	req, err := p.BuildEmbeddingsRequest(&ChatContext{
+		BaseURL:       "https://api.openai.com",
+		APIKey:        "sk-test",
+		UpstreamModel: "text-embedding-3-small",
+	}, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://api.openai.com/v1/embeddings" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestGeminiEmbeddingsBuildSingle(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("text-embedding-004"), "model")
+	body.Add(jsonutils.NewString("hello world"), "input")
+
+	p := GetEmbeddings("gemini")
+	req, err := p.BuildEmbeddingsRequest(&ChatContext{
+		BaseURL:       "https://generativelanguage.googleapis.com/v1beta",
+		APIKey:        "key",
+		UpstreamModel: "text-embedding-004",
+	}, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://generativelanguage.googleapis.com/v1beta/models/text-embedding-004:embedContent" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestGeminiEmbeddingsNormalize(t *testing.T) {
+	p := GetEmbeddings("gemini")
+	out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embedding":{"values":[0.1,0.2]}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp["object"] != "list" {
+		t.Fatalf("unexpected object: %#v", resp["object"])
+	}
+}
+
+func TestCohereEmbeddingsBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("embed-english-v3.0"), "model")
+	body.Add(jsonutils.NewArray(jsonutils.NewString("a"), jsonutils.NewString("b")), "input")
+
+	p := GetEmbeddings("cohere")
+	req, err := p.BuildEmbeddingsRequest(&ChatContext{
+		BaseURL:       "https://api.cohere.ai",
+		APIKey:        "key",
+		UpstreamModel: "embed-english-v3.0",
+	}, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://api.cohere.ai/v2/embed" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestCohereEmbeddingsNormalize(t *testing.T) {
+	p := GetEmbeddings("cohere")
+	out, err := p.NormalizeEmbeddingsResponse([]byte(`{"embeddings":{"float":[[0.1],[0.2]]}}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var resp struct {
+		Data []struct {
+			Index int `json:"index"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		t.Fatal(err)
+	}
+	if len(resp.Data) != 2 {
+		t.Fatalf("expected 2 embeddings, got %d", len(resp.Data))
+	}
+}
+
+func TestAnthropicEmbeddingsUnsupported(t *testing.T) {
+	p := GetEmbeddings("anthropic")
+	_, err := p.BuildEmbeddingsRequest(&ChatContext{ProviderKey: "anthropic"}, jsonutils.NewDict())
+	if err == nil {
+		t.Fatal("expected error for anthropic embeddings")
+	}
+}

pkg/aiproxy/providers/gemini/doc.go

@@ -0,0 +1 @@
+package gemini // import "yunion.io/x/onecloud/pkg/aiproxy/providers/gemini"

pkg/aiproxy/providers/gemini/gemini.go

@@ -0,0 +1,367 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gemini
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/google/uuid"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+type provider struct{}
+
+// New returns the Google Gemini provider adapter.
+func New() providerapi.Provider {
+	return &provider{}
+}
+
+func (p *provider) Key() string {
+	return "gemini"
+}
+
+func (p *provider) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	msgs, system, err := openai.ParseMessages(body)
+	if err != nil {
+		return nil, err
+	}
+	contents, err := openai.MessagesToGemini(msgs)
+	if err != nil {
+		return nil, err
+	}
+	genConfig := map[string]interface{}{}
+	if v, ok := openai.IntParam(body, "max_tokens", "max_completion_tokens"); ok {
+		genConfig["maxOutputTokens"] = v
+	}
+	if v, ok := openai.FloatParam(body, "temperature"); ok {
+		genConfig["temperature"] = v
+	}
+	if v, ok := openai.FloatParam(body, "top_p"); ok {
+		genConfig["topP"] = v
+	}
+	reqBody := map[string]interface{}{
+		"contents": contents,
+	}
+	if len(genConfig) > 0 {
+		reqBody["generationConfig"] = genConfig
+	}
+	if system != "" {
+		reqBody["systemInstruction"] = map[string]interface{}{
+			"parts": []map[string]interface{}{
+				{"text": system},
+			},
+		}
+	}
+	if tools, _, err := openai.ExtractTools(body); err != nil {
+		return nil, err
+	} else if gemTools := openai.ToolsToGemini(tools); len(gemTools) > 0 {
+		reqBody["tools"] = gemTools
+	}
+	raw, err := openai.MarshalJSON(reqBody)
+	if err != nil {
+		return nil, err
+	}
+	base := strings.TrimRight(strings.TrimSpace(ctx.BaseURL), "/")
+	if base == "" {
+		base = "https://generativelanguage.googleapis.com/v1beta"
+	}
+	action := "generateContent"
+	if stream {
+		action = "streamGenerateContent"
+	}
+	modelPath := fmt.Sprintf("/models/%s:%s", ctx.UpstreamModel, action)
+	url := openai.JoinURL(base, modelPath)
+	if stream {
+		url += "?alt=sse"
+	}
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    url,
+		Headers: map[string]string{
+			"x-goog-api-key": strings.TrimSpace(ctx.APIKey),
+			"Content-Type":   "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func (p *provider) NormalizeResponse(body []byte) ([]byte, error) {
+	var resp struct {
+		Candidates []struct {
+			Content struct {
+				Parts []openai.GeminiPart `json:"parts"`
+			} `json:"content"`
+			FinishReason string `json:"finishReason"`
+		} `json:"candidates"`
+		UsageMetadata struct {
+			PromptTokenCount     int `json:"promptTokenCount"`
+			CandidatesTokenCount int `json:"candidatesTokenCount"`
+		} `json:"usageMetadata"`
+	}
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return body, nil
+	}
+	finish := "stop"
+	msg := openai.AssistantMessage{}
+	if len(resp.Candidates) > 0 {
+		msg = openai.GeminiPartsToAssistant(resp.Candidates[0].Content.Parts)
+		if resp.Candidates[0].FinishReason != "" {
+			finish = resp.Candidates[0].FinishReason
+		}
+	}
+	out, err := openai.MarshalJSON(openai.NewChatCompletionWithTools(
+		"",
+		"gemini-"+uuid.New().String(),
+		msg,
+		finish,
+		resp.UsageMetadata.PromptTokenCount,
+		resp.UsageMetadata.CandidatesTokenCount,
+	))
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (p *provider) OpenAIStreamPassthrough() bool {
+	return false
+}
+
+func (p *provider) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) {
+	if state == nil || len(payload) == 0 {
+		return nil, nil
+	}
+	var resp struct {
+		Candidates []struct {
+			Content struct {
+				Parts []openai.GeminiPart `json:"parts"`
+			} `json:"content"`
+			FinishReason string `json:"finishReason"`
+		} `json:"candidates"`
+	}
+	if err := json.Unmarshal(payload, &resp); err != nil {
+		return nil, nil
+	}
+	if len(resp.Candidates) == 0 {
+		return nil, nil
+	}
+	c := resp.Candidates[0]
+	if state.ResponseID == "" {
+		state.ResponseID = "gemini-" + uuid.New().String()
+	}
+	var chunks []providerapi.StreamChunk
+	msg := openai.GeminiPartsToAssistant(c.Content.Parts)
+	if msg.Content != "" {
+		chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, msg.Content, ""))
+		if err != nil {
+			return nil, err
+		}
+		chunks = append(chunks, providerapi.StreamChunk{Data: chunk})
+	}
+	for _, tc := range msg.ToolCalls {
+		chunk, err := openai.MarshalJSON(openai.NewStreamChunkToolDelta(
+			state.Model, state.ResponseID, state.ToolIndex, tc, "",
+		))
+		if err != nil {
+			return nil, err
+		}
+		chunks = append(chunks, providerapi.StreamChunk{Data: chunk})
+		state.ToolIndex++
+	}
+	if c.FinishReason != "" {
+		chunk, err := openai.MarshalJSON(openai.NewStreamChunk(state.Model, state.ResponseID, 0, "", c.FinishReason))
+		if err != nil {
+			return nil, err
+		}
+		chunks = append(chunks, providerapi.StreamChunk{Data: chunk})
+	}
+	if len(chunks) == 0 {
+		return nil, nil
+	}
+	return chunks, nil
+}
+
+func (p *provider) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	texts, _, isText, err := openai.ParseEmbeddingInput(body)
+	if err != nil {
+		return nil, err
+	}
+	if !isText {
+		return openai.DefaultEmbeddingCompat().BuildEmbeddingsRequest(ctx, body)
+	}
+	base := strings.TrimRight(strings.TrimSpace(ctx.BaseURL), "/")
+	if base == "" {
+		base = "https://generativelanguage.googleapis.com/v1beta"
+	}
+	modelRef := modelRef(ctx.UpstreamModel)
+	var raw []byte
+	var action string
+	if len(texts) == 1 {
+		action = "embedContent"
+		reqBody := map[string]interface{}{
+			"content": map[string]interface{}{
+				"parts": []map[string]interface{}{{"text": texts[0]}},
+			},
+		}
+		raw, err = openai.MarshalJSON(reqBody)
+	} else {
+		action = "batchEmbedContents"
+		requests := make([]map[string]interface{}, len(texts))
+		for i, text := range texts {
+			requests[i] = map[string]interface{}{
+				"model": modelRef,
+				"content": map[string]interface{}{
+					"parts": []map[string]interface{}{{"text": text}},
+				},
+			}
+		}
+		raw, err = openai.MarshalJSON(map[string]interface{}{"requests": requests})
+	}
+	if err != nil {
+		return nil, err
+	}
+	url := openai.JoinURL(base, fmt.Sprintf("/models/%s:%s", ctx.UpstreamModel, action))
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    url,
+		Headers: map[string]string{
+			"x-goog-api-key": strings.TrimSpace(ctx.APIKey),
+			"Content-Type":   "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func (p *provider) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	var single struct {
+		Embedding struct {
+			Values []float64 `json:"values"`
+		} `json:"embedding"`
+	}
+	if err := json.Unmarshal(body, &single); err == nil && len(single.Embedding.Values) > 0 {
+		return openai.NewEmbeddingsResponse("", [][]float64{single.Embedding.Values}, 0)
+	}
+	var batch struct {
+		Embeddings []struct {
+			Values []float64 `json:"values"`
+		} `json:"embeddings"`
+	}
+	if err := json.Unmarshal(body, &batch); err != nil {
+		return body, nil
+	}
+	vectors := make([][]float64, len(batch.Embeddings))
+	for i := range batch.Embeddings {
+		vectors[i] = batch.Embeddings[i].Values
+	}
+	return openai.NewEmbeddingsResponse("", vectors, 0)
+}
+
+func modelRef(model string) string {
+	model = strings.TrimSpace(model)
+	if strings.HasPrefix(model, "models/") {
+		return model
+	}
+	return "models/" + model
+}
+
+func (p *provider) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	prompt, err := openai.ParseImagePrompt(body)
+	if err != nil {
+		return nil, err
+	}
+	base := strings.TrimRight(strings.TrimSpace(ctx.BaseURL), "/")
+	if base == "" {
+		base = "https://generativelanguage.googleapis.com/v1beta"
+	}
+	reqBody := map[string]interface{}{
+		"instances": []map[string]interface{}{
+			{"prompt": prompt},
+		},
+		"parameters": map[string]interface{}{
+			"sampleCount": openai.ImageCount(body),
+			"aspectRatio": openai.SizeToAspectRatio(openai.ImageSize(body)),
+		},
+	}
+	raw, err := openai.MarshalJSON(reqBody)
+	if err != nil {
+		return nil, err
+	}
+	url := openai.JoinURL(base, fmt.Sprintf("/models/%s:predict", ctx.UpstreamModel))
+	return &providerapi.HTTPRequest{
+		Method: http.MethodPost,
+		URL:    url,
+		Headers: map[string]string{
+			"x-goog-api-key": strings.TrimSpace(ctx.APIKey),
+			"Content-Type":   "application/json",
+		},
+		Body: raw,
+	}, nil
+}
+
+func (p *provider) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) {
+	var predict struct {
+		Predictions []struct {
+			BytesBase64Encoded string `json:"bytesBase64Encoded"`
+		} `json:"predictions"`
+	}
+	if err := json.Unmarshal(body, &predict); err == nil && len(predict.Predictions) > 0 {
+		items := make([]openai.ImageItem, 0, len(predict.Predictions))
+		for _, pred := range predict.Predictions {
+			if pred.BytesBase64Encoded == "" {
+				continue
+			}
+			items = append(items, openai.ImageItem{B64: pred.BytesBase64Encoded})
+		}
+		if len(items) > 0 {
+			return openai.NewImagesGenerationsResponse(items)
+		}
+	}
+	var generated struct {
+		GeneratedImages []struct {
+			Image struct {
+				ImageBytes string `json:"imageBytes"`
+			} `json:"image"`
+		} `json:"generatedImages"`
+	}
+	if err := json.Unmarshal(body, &generated); err == nil && len(generated.GeneratedImages) > 0 {
+		items := make([]openai.ImageItem, 0, len(generated.GeneratedImages))
+		for _, img := range generated.GeneratedImages {
+			if img.Image.ImageBytes == "" {
+				continue
+			}
+			items = append(items, openai.ImageItem{B64: img.Image.ImageBytes})
+		}
+		if len(items) > 0 {
+			return openai.NewImagesGenerationsResponse(items)
+		}
+	}
+	return body, nil
+}

pkg/aiproxy/providers/images.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+var defaultImages ImagesProvider = openai.DefaultImagesCompat()
+
+// GetImages returns the images adapter for providerKey, or OpenAI-compatible passthrough.
+func GetImages(providerKey string) ImagesProvider {
+	p := Get(providerKey)
+	if ip, ok := p.(ImagesProvider); ok {
+		return ip
+	}
+	return defaultImages
+}

pkg/aiproxy/providers/images_test.go

@@ -0,0 +1,121 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestOpenAIImagesCompatBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("dall-e-3"), "model")
+	body.Add(jsonutils.NewString("a red cat"), "prompt")
+
+	p := GetImages("openai")
+	req, err := p.BuildImagesGenerationsRequest(&ChatContext{
+		BaseURL:       "https://api.openai.com",
+		APIKey:        "sk-test",
+		UpstreamModel: "dall-e-3",
+	}, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://api.openai.com/v1/images/generations" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestGeminiImagesBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("imagen-3.0-generate-002"), "model")
+	body.Add(jsonutils.NewString("sunset over mountains"), "prompt")
+	body.Add(jsonutils.NewString("1792x1024"), "size")
+	body.Add(jsonutils.NewInt(2), "n")
+
+	p := GetImages("gemini")
+	req, err := p.BuildImagesGenerationsRequest(&ChatContext{
+		BaseURL:       "https://generativelanguage.googleapis.com/v1beta",
+		APIKey:        "key",
+		UpstreamModel: "imagen-3.0-generate-002",
+	}, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://generativelanguage.googleapis.com/v1beta/models/imagen-3.0-generate-002:predict" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+	var wire struct {
+		Parameters struct {
+			SampleCount int    `json:"sampleCount"`
+			AspectRatio string `json:"aspectRatio"`
+		} `json:"parameters"`
+	}
+	if err := json.Unmarshal(req.Body, &wire); err != nil {
+		t.Fatal(err)
+	}
+	if wire.Parameters.SampleCount != 2 || wire.Parameters.AspectRatio != "16:9" {
+		t.Fatalf("unexpected parameters: %+v", wire.Parameters)
+	}
+}
+
+func TestGeminiImagesNormalize(t *testing.T) {
+	p := GetImages("gemini")
+	out, err := p.NormalizeImagesGenerationsResponse([]byte(`{"predictions":[{"bytesBase64Encoded":"abc123"}]}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var resp struct {
+		Data []struct {
+			B64 string `json:"b64_json"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		t.Fatal(err)
+	}
+	if len(resp.Data) != 1 || resp.Data[0].B64 != "abc123" {
+		t.Fatalf("unexpected response: %+v", resp)
+	}
+}
+
+func TestAzureImagesBuild(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("dall-e-3"), "model")
+	body.Add(jsonutils.NewString("test"), "prompt")
+
+	p := GetImages("azure")
+	req, err := p.BuildImagesGenerationsRequest(&ChatContext{
+		BaseURL:       "https://example.openai.azure.com",
+		APIKey:        "key",
+		UpstreamModel: "dall-e-3",
+	}, body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(req.URL, "images/generations") {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestAnthropicImagesUnsupported(t *testing.T) {
+	p := GetImages("anthropic")
+	_, err := p.BuildImagesGenerationsRequest(&ChatContext{ProviderKey: "anthropic"}, jsonutils.NewDict())
+	if err == nil {
+		t.Fatal("expected error for anthropic images")
+	}
+}

pkg/aiproxy/providers/openai/compat.go

@@ -0,0 +1,102 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"fmt"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+var (
+	defaultEmbeddings = NewEmbeddingCompat()
+	defaultImages     = NewImagesCompat()
+)
+
+// Compat forwards OpenAI-shaped JSON to OpenAI-compatible upstreams.
+type Compat struct {
+	ProviderKey string
+	Patches     []PatchFunc
+}
+
+// NewCompat returns an OpenAI-compatible provider for the given catalog provider_key.
+func NewCompat(providerKey string, patches ...PatchFunc) *Compat {
+	return &Compat{ProviderKey: providerKey, Patches: patches}
+}
+
+func (p *Compat) Key() string {
+	return p.ProviderKey
+}
+
+func (p *Compat) buildBody(body *jsonutils.JSONDict, upstreamModel string, stream bool) *jsonutils.JSONDict {
+	if len(p.Patches) == 0 {
+		return CloneBodyWithModel(body, upstreamModel)
+	}
+	return PatchBody(CloneBodyWithModel(body, upstreamModel), stream, p.Patches...)
+}
+
+func (p *Compat) BuildUpstreamRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	dup := p.buildBody(body, ctx.UpstreamModel, stream)
+	return &providerapi.HTTPRequest{
+		Method:  http.MethodPost,
+		URL:     ChatCompletionsURL(ctx.BaseURL),
+		Headers: BearerAuthHeaders(ctx.APIKey),
+		Body:    []byte(dup.String()),
+	}, nil
+}
+
+func (p *Compat) NormalizeResponse(body []byte) ([]byte, error) {
+	return body, nil
+}
+
+func (p *Compat) OpenAIStreamPassthrough() bool {
+	return true
+}
+
+func (p *Compat) ConvertStreamEvent(eventType string, payload []byte, state *providerapi.StreamState) ([]providerapi.StreamChunk, error) {
+	return nil, nil
+}
+
+func (p *Compat) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	return defaultEmbeddings.BuildEmbeddingsRequest(ctx, body)
+}
+
+func (p *Compat) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	return defaultEmbeddings.NormalizeEmbeddingsResponse(body)
+}
+
+func (p *Compat) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	return defaultImages.BuildImagesGenerationsRequest(ctx, body)
+}
+
+func (p *Compat) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) {
+	return defaultImages.NormalizeImagesGenerationsResponse(body)
+}
+
+// DefaultEmbeddingCompat returns the shared OpenAI-compatible embeddings adapter.
+func DefaultEmbeddingCompat() providerapi.EmbeddingsProvider {
+	return defaultEmbeddings
+}
+
+// DefaultImagesCompat returns the shared OpenAI-compatible images adapter.
+func DefaultImagesCompat() providerapi.ImagesProvider {
+	return defaultImages
+}

pkg/aiproxy/providers/openai/completions.go

@@ -0,0 +1,63 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"fmt"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+// CompletionsCompat forwards OpenAI legacy completions requests unchanged.
+type CompletionsCompat struct {
+	Patches []PatchFunc
+}
+
+// NewCompletionsCompat returns a new OpenAI-compatible completions adapter.
+func NewCompletionsCompat(patches ...PatchFunc) *CompletionsCompat {
+	return &CompletionsCompat{Patches: patches}
+}
+
+func (p *CompletionsCompat) buildBody(body *jsonutils.JSONDict, upstreamModel string, stream bool) *jsonutils.JSONDict {
+	dup := CloneBodyWithModel(body, upstreamModel)
+	if len(p.Patches) == 0 {
+		return dup
+	}
+	return PatchBody(dup, stream, p.Patches...)
+}
+
+func (p *CompletionsCompat) BuildCompletionsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	dup := p.buildBody(body, ctx.UpstreamModel, stream)
+	return &providerapi.HTTPRequest{
+		Method:  http.MethodPost,
+		URL:     CompletionsURL(ctx.BaseURL),
+		Headers: BearerAuthHeaders(ctx.APIKey),
+		Body:    []byte(dup.String()),
+	}, nil
+}
+
+func (p *CompletionsCompat) NormalizeCompletionsResponse(body []byte) ([]byte, error) {
+	return body, nil
+}
+
+func (p *CompletionsCompat) OpenAICompletionsStreamPassthrough() bool {
+	return true
+}

pkg/aiproxy/providers/openai/doc.go

@@ -0,0 +1 @@
+package openai // import "yunion.io/x/onecloud/pkg/aiproxy/providers/openai"

pkg/aiproxy/providers/openai/embeddings.go

@@ -0,0 +1,49 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"fmt"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+// EmbeddingCompat forwards OpenAI embeddings requests unchanged.
+type EmbeddingCompat struct{}
+
+// NewEmbeddingCompat returns a new OpenAI-compatible embeddings adapter.
+func NewEmbeddingCompat() *EmbeddingCompat {
+	return &EmbeddingCompat{}
+}
+
+func (p *EmbeddingCompat) BuildEmbeddingsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	dup := CloneBodyWithModel(body, ctx.UpstreamModel)
+	return &providerapi.HTTPRequest{
+		Method:  http.MethodPost,
+		URL:     EmbeddingsURL(ctx.BaseURL),
+		Headers: BearerAuthHeaders(ctx.APIKey),
+		Body:    []byte(dup.String()),
+	}, nil
+}
+
+func (p *EmbeddingCompat) NormalizeEmbeddingsResponse(body []byte) ([]byte, error) {
+	return body, nil
+}

pkg/aiproxy/providers/openai/images.go

@@ -0,0 +1,49 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"fmt"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+// ImagesCompat forwards OpenAI images/generations requests unchanged.
+type ImagesCompat struct{}
+
+// NewImagesCompat returns a new OpenAI-compatible images adapter.
+func NewImagesCompat() *ImagesCompat {
+	return &ImagesCompat{}
+}
+
+func (p *ImagesCompat) BuildImagesGenerationsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict) (*providerapi.HTTPRequest, error) {
+	if ctx == nil {
+		return nil, fmt.Errorf("nil chat context")
+	}
+	dup := CloneBodyWithModel(body, ctx.UpstreamModel)
+	return &providerapi.HTTPRequest{
+		Method:  http.MethodPost,
+		URL:     ImagesGenerationsURL(ctx.BaseURL),
+		Headers: BearerAuthHeaders(ctx.APIKey),
+		Body:    []byte(dup.String()),
+	}, nil
+}
+
+func (p *ImagesCompat) NormalizeImagesGenerationsResponse(body []byte) ([]byte, error) {
+	return body, nil
+}

pkg/aiproxy/providers/openai/schema.go

@@ -0,0 +1,414 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+)
+
+// Message is one OpenAI chat message entry.
+type Message struct {
+	Role       string          `json:"role"`
+	Content    json.RawMessage `json:"content"`
+	ToolCalls  []ToolCall      `json:"tool_calls,omitempty"`
+	ToolCallID string          `json:"tool_call_id,omitempty"`
+	Name       string          `json:"name,omitempty"`
+}
+
+// ImageItem is one generated image in an OpenAI images response.
+type ImageItem struct {
+	URL           string
+	B64           string
+	RevisedPrompt string
+}
+
+// MessageTextContent extracts plain text from an OpenAI message content field.
+func MessageTextContent(raw json.RawMessage) string {
+	if len(raw) == 0 {
+		return ""
+	}
+	var s string
+	if err := json.Unmarshal(raw, &s); err == nil {
+		return s
+	}
+	var parts []struct {
+		Type string `json:"type"`
+		Text string `json:"text"`
+	}
+	if err := json.Unmarshal(raw, &parts); err == nil {
+		var b strings.Builder
+		for _, p := range parts {
+			if p.Type == "text" && p.Text != "" {
+				if b.Len() > 0 {
+					b.WriteString("\n")
+				}
+				b.WriteString(p.Text)
+			}
+		}
+		return b.String()
+	}
+	return string(raw)
+}
+
+// ParseMessages splits OpenAI chat messages and hoists system prompts.
+func ParseMessages(body *jsonutils.JSONDict) ([]Message, string, error) {
+	if body == nil {
+		return nil, "", fmt.Errorf("nil request body")
+	}
+	arr, err := body.Get("messages")
+	if err != nil {
+		return nil, "", fmt.Errorf("missing messages")
+	}
+	raw := []byte(arr.String())
+	var msgs []Message
+	if err := json.Unmarshal(raw, &msgs); err != nil {
+		return nil, "", fmt.Errorf("invalid messages: %w", err)
+	}
+	var system strings.Builder
+	out := make([]Message, 0, len(msgs))
+	for i := range msgs {
+		role := strings.ToLower(strings.TrimSpace(msgs[i].Role))
+		switch role {
+		case "system":
+			text := MessageTextContent(msgs[i].Content)
+			if text != "" {
+				if system.Len() > 0 {
+					system.WriteString("\n\n")
+				}
+				system.WriteString(text)
+			}
+		case "user", "assistant", "tool":
+			out = append(out, msgs[i])
+		default:
+			out = append(out, msgs[i])
+		}
+	}
+	return out, system.String(), nil
+}
+
+// IntParam reads the first positive int param from an OpenAI JSON body.
+func IntParam(body *jsonutils.JSONDict, keys ...string) (int, bool) {
+	for _, k := range keys {
+		if v, err := body.Int(k); err == nil && v > 0 {
+			return int(v), true
+		}
+	}
+	return 0, false
+}
+
+// FloatParam reads a float param from an OpenAI JSON body.
+func FloatParam(body *jsonutils.JSONDict, key string) (float64, bool) {
+	if v, err := body.Float(key); err == nil {
+		return v, true
+	}
+	return 0, false
+}
+
+// CloneBodyWithModel clones the request body and sets the upstream model id.
+func CloneBodyWithModel(body *jsonutils.JSONDict, upstreamModel string) *jsonutils.JSONDict {
+	dup := jsonutils.NewDict()
+	if body != nil {
+		dup = body.Copy()
+	}
+	dup.Set("model", jsonutils.NewString(upstreamModel))
+	return dup
+}
+
+// ChatCompletionsURL builds an OpenAI-compatible chat completions endpoint.
+func ChatCompletionsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/chat/completions") {
+		return base
+	}
+	if strings.HasSuffix(base, "/v2") {
+		return base + "/chat/completions"
+	}
+	if strings.HasSuffix(base, "/v1") {
+		return base + "/chat/completions"
+	}
+	return base + "/v1/chat/completions"
+}
+
+// CompletionsURL builds an OpenAI-compatible legacy completions endpoint.
+func CompletionsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/completions") {
+		return base
+	}
+	if strings.HasSuffix(base, "/v2") {
+		return base + "/completions"
+	}
+	if strings.HasSuffix(base, "/v1") {
+		return base + "/completions"
+	}
+	return base + "/v1/completions"
+}
+
+// EmbeddingsURL builds an OpenAI-compatible embeddings endpoint.
+func EmbeddingsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/embeddings") {
+		return base
+	}
+	if strings.HasSuffix(base, "/v2") {
+		return base + "/embeddings"
+	}
+	if strings.HasSuffix(base, "/v1") {
+		return base + "/embeddings"
+	}
+	return base + "/v1/embeddings"
+}
+
+// ImagesGenerationsURL builds an OpenAI-compatible images/generations endpoint.
+func ImagesGenerationsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/images/generations") {
+		return base
+	}
+	if strings.HasSuffix(base, "/v2") {
+		return base + "/images/generations"
+	}
+	if strings.HasSuffix(base, "/v1") {
+		return base + "/images/generations"
+	}
+	return base + "/v1/images/generations"
+}
+
+// BearerAuthHeaders returns standard OpenAI bearer auth headers.
+func BearerAuthHeaders(apiKey string) map[string]string {
+	return map[string]string{
+		"Authorization": "Bearer " + strings.TrimSpace(apiKey),
+		"Content-Type":  "application/json",
+	}
+}
+
+// JoinURL joins a base URL and path segment.
+func JoinURL(base, path string) string {
+	base = strings.TrimRight(strings.TrimSpace(base), "/")
+	path = strings.TrimLeft(strings.TrimSpace(path), "/")
+	if base == "" {
+		return "/" + path
+	}
+	return base + "/" + path
+}
+
+// FinishReasonFromStop maps provider-specific stop reasons to OpenAI finish_reason values.
+func FinishReasonFromStop(stop string) string {
+	switch strings.TrimSpace(stop) {
+	case "end_turn", "stop_sequence", "stop", "STOP":
+		return "stop"
+	case "max_tokens", "length":
+		return "length"
+	case "tool_use":
+		return "tool_calls"
+	default:
+		if stop == "" {
+			return "stop"
+		}
+		return stop
+	}
+}
+
+// NewChatCompletion builds an OpenAI chat.completion response object.
+func NewChatCompletion(model, id, content, finishReason string, promptTokens, completionTokens int) map[string]interface{} {
+	total := promptTokens + completionTokens
+	return map[string]interface{}{
+		"id":      id,
+		"object":  "chat.completion",
+		"created": time.Now().Unix(),
+		"model":   model,
+		"choices": []map[string]interface{}{
+			{
+				"index": 0,
+				"message": map[string]interface{}{
+					"role":    "assistant",
+					"content": content,
+				},
+				"finish_reason": FinishReasonFromStop(finishReason),
+			},
+		},
+		"usage": map[string]interface{}{
+			"prompt_tokens":     promptTokens,
+			"completion_tokens": completionTokens,
+			"total_tokens":      total,
+		},
+	}
+}
+
+// NewStreamChunk builds an OpenAI chat.completion.chunk object.
+func NewStreamChunk(model, id string, index int, content string, finishReason string) map[string]interface{} {
+	delta := map[string]interface{}{
+		"role": "assistant",
+	}
+	if content != "" {
+		delta["content"] = content
+	}
+	choice := map[string]interface{}{
+		"index": index,
+		"delta": delta,
+	}
+	if finishReason != "" {
+		choice["finish_reason"] = FinishReasonFromStop(finishReason)
+	}
+	return map[string]interface{}{
+		"id":      id,
+		"object":  "chat.completion.chunk",
+		"created": time.Now().Unix(),
+		"model":   model,
+		"choices": []map[string]interface{}{choice},
+	}
+}
+
+// MarshalJSON marshals v to JSON bytes.
+func MarshalJSON(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+// ParseEmbeddingInput extracts string inputs from an OpenAI embeddings body.
+func ParseEmbeddingInput(body *jsonutils.JSONDict) (texts []string, rawInput json.RawMessage, isText bool, err error) {
+	if body == nil {
+		return nil, nil, false, fmt.Errorf("nil request body")
+	}
+	inp, err := body.Get("input")
+	if err != nil {
+		return nil, nil, false, fmt.Errorf("missing input")
+	}
+	rawInput = []byte(inp.String())
+	var s string
+	if err := json.Unmarshal(rawInput, &s); err == nil {
+		return []string{s}, rawInput, true, nil
+	}
+	var arr []string
+	if err := json.Unmarshal(rawInput, &arr); err == nil {
+		return arr, rawInput, true, nil
+	}
+	return nil, rawInput, false, nil
+}
+
+// NewEmbeddingsResponse builds an OpenAI embeddings list response.
+func NewEmbeddingsResponse(model string, vectors [][]float64, promptTokens int) ([]byte, error) {
+	data := make([]map[string]interface{}, len(vectors))
+	for i, v := range vectors {
+		data[i] = map[string]interface{}{
+			"object":    "embedding",
+			"index":     i,
+			"embedding": v,
+		}
+	}
+	if promptTokens <= 0 {
+		promptTokens = 0
+	}
+	return MarshalJSON(map[string]interface{}{
+		"object": "list",
+		"data":   data,
+		"model":  model,
+		"usage": map[string]interface{}{
+			"prompt_tokens": promptTokens,
+			"total_tokens":  promptTokens,
+		},
+	})
+}
+
+// ParseImagePrompt reads the prompt from an OpenAI images/generations body.
+func ParseImagePrompt(body *jsonutils.JSONDict) (string, error) {
+	if body == nil {
+		return "", fmt.Errorf("nil request body")
+	}
+	prompt, err := body.GetString("prompt")
+	if err != nil || strings.TrimSpace(prompt) == "" {
+		return "", fmt.Errorf("missing prompt")
+	}
+	return strings.TrimSpace(prompt), nil
+}
+
+// ImageCount reads n from an OpenAI images/generations body.
+func ImageCount(body *jsonutils.JSONDict) int {
+	if body == nil {
+		return 1
+	}
+	if n, err := body.Int("n"); err == nil && n > 0 {
+		return int(n)
+	}
+	return 1
+}
+
+// ImageSize reads size from an OpenAI images/generations body.
+func ImageSize(body *jsonutils.JSONDict) string {
+	if body == nil {
+		return "1024x1024"
+	}
+	if s, err := body.GetString("size"); err == nil && strings.TrimSpace(s) != "" {
+		return strings.TrimSpace(s)
+	}
+	return "1024x1024"
+}
+
+// SizeToAspectRatio maps OpenAI image size strings to provider aspect ratios.
+func SizeToAspectRatio(size string) string {
+	switch strings.TrimSpace(size) {
+	case "1024x1792", "768x1344", "720x1280":
+		return "9:16"
+	case "1792x1024", "1344x768", "1280x720":
+		return "16:9"
+	case "256x256", "512x512", "1024x1024":
+		return "1:1"
+	default:
+		return "1:1"
+	}
+}
+
+// NewImagesGenerationsResponse builds an OpenAI images/generations response.
+func NewImagesGenerationsResponse(items []ImageItem) ([]byte, error) {
+	data := make([]map[string]interface{}, 0, len(items))
+	for _, item := range items {
+		row := map[string]interface{}{}
+		if item.URL != "" {
+			row["url"] = item.URL
+		}
+		if item.B64 != "" {
+			row["b64_json"] = item.B64
+		}
+		if item.RevisedPrompt != "" {
+			row["revised_prompt"] = item.RevisedPrompt
+		}
+		data = append(data, row)
+	}
+	return MarshalJSON(map[string]interface{}{
+		"created": time.Now().Unix(),
+		"data":    data,
+	})
+}
+
+// PatchFunc mutates an OpenAI request body before forwarding upstream.
+type PatchFunc func(body *jsonutils.JSONDict, stream bool)
+
+// PatchBody clones body and applies optional patches.
+func PatchBody(body *jsonutils.JSONDict, stream bool, patches ...PatchFunc) *jsonutils.JSONDict {
+	dup := jsonutils.NewDict()
+	if body != nil {
+		dup = body.Copy()
+	}
+	for _, patch := range patches {
+		if patch != nil {
+			patch(dup, stream)
+		}
+	}
+	return dup
+}

pkg/aiproxy/providers/openai/tools.go

@@ -0,0 +1,528 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+)
+
+// ToolCall is one OpenAI assistant tool invocation.
+type ToolCall struct {
+	Index    int          `json:"index,omitempty"`
+	ID       string       `json:"id,omitempty"`
+	Type     string       `json:"type,omitempty"`
+	Function ToolFunction `json:"function"`
+}
+
+// ToolFunction is the function payload inside a tool call.
+type ToolFunction struct {
+	Name      string `json:"name"`
+	Arguments string `json:"arguments"`
+}
+
+// ToolDefinition describes one OpenAI function tool.
+type ToolDefinition struct {
+	Type     string          `json:"type"`
+	Function ToolFunctionDef `json:"function"`
+}
+
+// ToolFunctionDef is the function schema in an OpenAI tools array.
+type ToolFunctionDef struct {
+	Name        string          `json:"name"`
+	Description string          `json:"description,omitempty"`
+	Parameters  json.RawMessage `json:"parameters,omitempty"`
+}
+
+// AssistantMessage is a normalized assistant response for OpenAI chat.completion.
+type AssistantMessage struct {
+	Content   string
+	ToolCalls []ToolCall
+}
+
+// ExtractTools reads tools and tool_choice from an OpenAI chat body.
+func ExtractTools(body *jsonutils.JSONDict) ([]ToolDefinition, json.RawMessage, error) {
+	if body == nil {
+		return nil, nil, nil
+	}
+	toolsRaw, err := body.Get("tools")
+	if err != nil {
+		return nil, nil, nil
+	}
+	var tools []ToolDefinition
+	if err := json.Unmarshal([]byte(toolsRaw.String()), &tools); err != nil {
+		return nil, nil, fmt.Errorf("invalid tools: %w", err)
+	}
+	var toolChoice json.RawMessage
+	if tc, err := body.Get("tool_choice"); err == nil {
+		toolChoice = []byte(tc.String())
+	}
+	return tools, toolChoice, nil
+}
+
+// ToolsToAnthropic converts OpenAI tools to Anthropic tools.
+func ToolsToAnthropic(tools []ToolDefinition) []map[string]interface{} {
+	out := make([]map[string]interface{}, 0, len(tools))
+	for _, t := range tools {
+		if strings.TrimSpace(t.Type) != "" && t.Type != "function" {
+			continue
+		}
+		name := strings.TrimSpace(t.Function.Name)
+		if name == "" {
+			continue
+		}
+		item := map[string]interface{}{
+			"name": name,
+		}
+		if desc := strings.TrimSpace(t.Function.Description); desc != "" {
+			item["description"] = desc
+		}
+		if len(t.Function.Parameters) > 0 && string(t.Function.Parameters) != "null" {
+			var schema interface{}
+			if json.Unmarshal(t.Function.Parameters, &schema) == nil {
+				item["input_schema"] = schema
+			}
+		}
+		out = append(out, item)
+	}
+	return out
+}
+
+// ToolChoiceToAnthropic converts OpenAI tool_choice to Anthropic tool_choice.
+func ToolChoiceToAnthropic(raw json.RawMessage) interface{} {
+	if len(raw) == 0 {
+		return nil
+	}
+	var s string
+	if json.Unmarshal(raw, &s) == nil {
+		switch strings.ToLower(strings.TrimSpace(s)) {
+		case "", "auto":
+			return map[string]interface{}{"type": "auto"}
+		case "none":
+			return map[string]interface{}{"type": "none"}
+		case "required":
+			return map[string]interface{}{"type": "any"}
+		}
+	}
+	var obj struct {
+		Type     string `json:"type"`
+		Function struct {
+			Name string `json:"name"`
+		} `json:"function"`
+	}
+	if json.Unmarshal(raw, &obj) == nil {
+		if strings.EqualFold(obj.Type, "function") && strings.TrimSpace(obj.Function.Name) != "" {
+			return map[string]interface{}{
+				"type": "tool",
+				"name": strings.TrimSpace(obj.Function.Name),
+			}
+		}
+	}
+	return nil
+}
+
+// MessagesToAnthropic converts OpenAI messages to Anthropic message objects.
+func MessagesToAnthropic(msgs []Message) ([]map[string]interface{}, error) {
+	out := make([]map[string]interface{}, 0, len(msgs))
+	for _, m := range msgs {
+		role := strings.ToLower(strings.TrimSpace(m.Role))
+		switch role {
+		case "assistant":
+			blocks := assistantContentToAnthropic(m)
+			if len(blocks) == 0 {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role":    "assistant",
+				"content": blocks,
+			})
+		case "tool":
+			text := MessageTextContent(m.Content)
+			if m.ToolCallID == "" && text == "" {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role": "user",
+				"content": []map[string]interface{}{
+					{
+						"type":        "tool_result",
+						"tool_use_id": m.ToolCallID,
+						"content":     text,
+					},
+				},
+			})
+		case "user":
+			text := MessageTextContent(m.Content)
+			if text == "" {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role": "user",
+				"content": []map[string]interface{}{
+					{"type": "text", "text": text},
+				},
+			})
+		default:
+			text := MessageTextContent(m.Content)
+			if text == "" {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role": role,
+				"content": []map[string]interface{}{
+					{"type": "text", "text": text},
+				},
+			})
+		}
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("no convertible messages")
+	}
+	return out, nil
+}
+
+func assistantContentToAnthropic(m Message) []map[string]interface{} {
+	blocks := make([]map[string]interface{}, 0, 1+len(m.ToolCalls))
+	if text := MessageTextContent(m.Content); text != "" {
+		blocks = append(blocks, map[string]interface{}{
+			"type": "text",
+			"text": text,
+		})
+	}
+	for _, tc := range m.ToolCalls {
+		if strings.TrimSpace(tc.Function.Name) == "" {
+			continue
+		}
+		input := map[string]interface{}{}
+		args := strings.TrimSpace(tc.Function.Arguments)
+		if args != "" {
+			_ = json.Unmarshal([]byte(args), &input)
+		}
+		id := strings.TrimSpace(tc.ID)
+		if id == "" {
+			id = "toolu_" + strings.TrimSpace(tc.Function.Name)
+		}
+		blocks = append(blocks, map[string]interface{}{
+			"type":  "tool_use",
+			"id":    id,
+			"name":  strings.TrimSpace(tc.Function.Name),
+			"input": input,
+		})
+	}
+	return blocks
+}
+
+// AnthropicBlock is one Anthropic message content block.
+type AnthropicBlock struct {
+	Type  string                 `json:"type"`
+	Text  string                 `json:"text"`
+	ID    string                 `json:"id"`
+	Name  string                 `json:"name"`
+	Input map[string]interface{} `json:"input"`
+}
+
+// AnthropicBlocksToAssistant converts Anthropic content blocks to OpenAI assistant message fields.
+func AnthropicBlocksToAssistant(blocks []AnthropicBlock) AssistantMessage {
+	var out AssistantMessage
+	for _, b := range blocks {
+		switch b.Type {
+		case "text":
+			out.Content += b.Text
+		case "tool_use":
+			args, _ := json.Marshal(b.Input)
+			id := strings.TrimSpace(b.ID)
+			if id == "" {
+				id = "call_" + strings.TrimSpace(b.Name)
+			}
+			out.ToolCalls = append(out.ToolCalls, ToolCall{
+				ID:   id,
+				Type: "function",
+				Function: ToolFunction{
+					Name:      strings.TrimSpace(b.Name),
+					Arguments: string(args),
+				},
+			})
+		}
+	}
+	return out
+}
+
+// ToolsToGemini converts OpenAI tools to Gemini functionDeclarations.
+func ToolsToGemini(tools []ToolDefinition) []map[string]interface{} {
+	decls := make([]map[string]interface{}, 0, len(tools))
+	for _, t := range tools {
+		if t.Type != "" && t.Type != "function" {
+			continue
+		}
+		name := strings.TrimSpace(t.Function.Name)
+		if name == "" {
+			continue
+		}
+		decl := map[string]interface{}{
+			"name": name,
+		}
+		if desc := strings.TrimSpace(t.Function.Description); desc != "" {
+			decl["description"] = desc
+		}
+		if len(t.Function.Parameters) > 0 && string(t.Function.Parameters) != "null" {
+			var params interface{}
+			if json.Unmarshal(t.Function.Parameters, &params) == nil {
+				decl["parameters"] = params
+			}
+		}
+		decls = append(decls, decl)
+	}
+	if len(decls) == 0 {
+		return nil
+	}
+	return []map[string]interface{}{{"functionDeclarations": decls}}
+}
+
+// MessagesToGemini converts OpenAI messages to Gemini contents entries.
+func MessagesToGemini(msgs []Message) ([]map[string]interface{}, error) {
+	out := make([]map[string]interface{}, 0, len(msgs))
+	for _, m := range msgs {
+		role := strings.ToLower(strings.TrimSpace(m.Role))
+		switch role {
+		case "assistant":
+			parts := assistantPartsToGemini(m)
+			if len(parts) == 0 {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role":  "model",
+				"parts": parts,
+			})
+		case "tool":
+			name := strings.TrimSpace(m.Name)
+			if name == "" {
+				name = "tool"
+			}
+			resp := toolResultToGeminiResponse(MessageTextContent(m.Content))
+			out = append(out, map[string]interface{}{
+				"role": "user",
+				"parts": []map[string]interface{}{
+					{
+						"functionResponse": map[string]interface{}{
+							"name":     name,
+							"response": resp,
+						},
+					},
+				},
+			})
+		case "user":
+			text := MessageTextContent(m.Content)
+			if text == "" {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role": "user",
+				"parts": []map[string]interface{}{
+					{"text": text},
+				},
+			})
+		default:
+			text := MessageTextContent(m.Content)
+			if text == "" {
+				continue
+			}
+			out = append(out, map[string]interface{}{
+				"role": "user",
+				"parts": []map[string]interface{}{
+					{"text": text},
+				},
+			})
+		}
+	}
+	if len(out) == 0 {
+		return nil, fmt.Errorf("no convertible messages")
+	}
+	return out, nil
+}
+
+func assistantPartsToGemini(m Message) []map[string]interface{} {
+	parts := make([]map[string]interface{}, 0, 1+len(m.ToolCalls))
+	if text := MessageTextContent(m.Content); text != "" {
+		parts = append(parts, map[string]interface{}{"text": text})
+	}
+	for _, tc := range m.ToolCalls {
+		name := strings.TrimSpace(tc.Function.Name)
+		if name == "" {
+			continue
+		}
+		args := map[string]interface{}{}
+		if raw := strings.TrimSpace(tc.Function.Arguments); raw != "" {
+			_ = json.Unmarshal([]byte(raw), &args)
+		}
+		parts = append(parts, map[string]interface{}{
+			"functionCall": map[string]interface{}{
+				"name": name,
+				"args": args,
+			},
+		})
+	}
+	return parts
+}
+
+func toolResultToGeminiResponse(content string) map[string]interface{} {
+	content = strings.TrimSpace(content)
+	if content == "" {
+		return map[string]interface{}{}
+	}
+	var obj map[string]interface{}
+	if json.Unmarshal([]byte(content), &obj) == nil {
+		return obj
+	}
+	return map[string]interface{}{"output": content}
+}
+
+type geminiPart struct {
+	Text         string `json:"text"`
+	FunctionCall *struct {
+		Name string                 `json:"name"`
+		Args map[string]interface{} `json:"args"`
+	} `json:"functionCall"`
+}
+
+// GeminiPart is one Gemini content part in a candidate response.
+type GeminiPart = geminiPart
+
+// GeminiPartsToAssistant converts Gemini candidate parts to OpenAI assistant fields.
+func GeminiPartsToAssistant(parts []geminiPart) AssistantMessage {
+	var out AssistantMessage
+	for _, p := range parts {
+		if p.Text != "" {
+			out.Content += p.Text
+		}
+		if p.FunctionCall != nil && strings.TrimSpace(p.FunctionCall.Name) != "" {
+			args, _ := json.Marshal(p.FunctionCall.Args)
+			out.ToolCalls = append(out.ToolCalls, ToolCall{
+				ID:   "call_" + strings.TrimSpace(p.FunctionCall.Name),
+				Type: "function",
+				Function: ToolFunction{
+					Name:      strings.TrimSpace(p.FunctionCall.Name),
+					Arguments: string(args),
+				},
+			})
+		}
+	}
+	return out
+}
+
+// NewChatCompletionWithTools builds an OpenAI chat.completion including tool_calls.
+func NewChatCompletionWithTools(model, id string, msg AssistantMessage, finishReason string, promptTokens, completionTokens int) map[string]interface{} {
+	message := map[string]interface{}{
+		"role": "assistant",
+	}
+	if msg.Content != "" {
+		message["content"] = msg.Content
+	} else if len(msg.ToolCalls) > 0 {
+		message["content"] = nil
+	} else {
+		message["content"] = ""
+	}
+	if len(msg.ToolCalls) > 0 {
+		calls := make([]map[string]interface{}, len(msg.ToolCalls))
+		for i, tc := range msg.ToolCalls {
+			typ := tc.Type
+			if typ == "" {
+				typ = "function"
+			}
+			calls[i] = map[string]interface{}{
+				"id":   tc.ID,
+				"type": typ,
+				"function": map[string]interface{}{
+					"name":      tc.Function.Name,
+					"arguments": tc.Function.Arguments,
+				},
+			}
+		}
+		message["tool_calls"] = calls
+	}
+	total := promptTokens + completionTokens
+	reason := FinishReasonFromStop(finishReason)
+	if len(msg.ToolCalls) > 0 && reason == "stop" {
+		reason = "tool_calls"
+	}
+	return map[string]interface{}{
+		"id":      id,
+		"object":  "chat.completion",
+		"created": jsonNowUnix(),
+		"model":   model,
+		"choices": []map[string]interface{}{
+			{
+				"index":         0,
+				"message":       message,
+				"finish_reason": reason,
+			},
+		},
+		"usage": map[string]interface{}{
+			"prompt_tokens":     promptTokens,
+			"completion_tokens": completionTokens,
+			"total_tokens":      total,
+		},
+	}
+}
+
+// NewStreamChunkToolDelta builds an OpenAI stream chunk with tool_calls delta.
+func NewStreamChunkToolDelta(model, id string, index int, tc ToolCall, finishReason string) map[string]interface{} {
+	delta := map[string]interface{}{
+		"role": "assistant",
+	}
+	call := map[string]interface{}{
+		"index": index,
+	}
+	if tc.ID != "" {
+		call["id"] = tc.ID
+	}
+	typ := tc.Type
+	if typ == "" {
+		typ = "function"
+	}
+	call["type"] = typ
+	fn := map[string]interface{}{}
+	if tc.Function.Name != "" {
+		fn["name"] = tc.Function.Name
+	}
+	if tc.Function.Arguments != "" {
+		fn["arguments"] = tc.Function.Arguments
+	}
+	if len(fn) > 0 {
+		call["function"] = fn
+	}
+	delta["tool_calls"] = []map[string]interface{}{call}
+	choice := map[string]interface{}{
+		"index": 0,
+		"delta": delta,
+	}
+	if finishReason != "" {
+		choice["finish_reason"] = FinishReasonFromStop(finishReason)
+	}
+	return map[string]interface{}{
+		"id":      id,
+		"object":  "chat.completion.chunk",
+		"created": jsonNowUnix(),
+		"model":   model,
+		"choices": []map[string]interface{}{choice},
+	}
+}
+
+func jsonNowUnix() int64 {
+	return time.Now().Unix()
+}

pkg/aiproxy/providers/openai/tools_test.go

@@ -0,0 +1,136 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package openai
+
+import (
+	"encoding/json"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestToolsToAnthropic(t *testing.T) {
+	tools := []ToolDefinition{{
+		Type: "function",
+		Function: ToolFunctionDef{
+			Name:        "get_weather",
+			Description: "Get weather",
+			Parameters:  json.RawMessage(`{"type":"object","properties":{"city":{"type":"string"}}}`),
+		},
+	}}
+	out := ToolsToAnthropic(tools)
+	if len(out) != 1 {
+		t.Fatalf("expected 1 tool, got %d", len(out))
+	}
+	if out[0]["name"] != "get_weather" {
+		t.Fatalf("unexpected name: %v", out[0]["name"])
+	}
+}
+
+func TestMessagesToAnthropicToolRoundTrip(t *testing.T) {
+	msgs := []Message{
+		{Role: "user", Content: json.RawMessage(`"hello"`)},
+		{
+			Role: "assistant",
+			ToolCalls: []ToolCall{{
+				ID:   "call_1",
+				Type: "function",
+				Function: ToolFunction{
+					Name:      "get_weather",
+					Arguments: `{"city":"Boston"}`,
+				},
+			}},
+		},
+		{Role: "tool", ToolCallID: "call_1", Content: json.RawMessage(`"72F"`)},
+	}
+	anthropicMsgs, err := MessagesToAnthropic(msgs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(anthropicMsgs) != 3 {
+		t.Fatalf("expected 3 messages, got %d", len(anthropicMsgs))
+	}
+	blocks, ok := anthropicMsgs[1]["content"].([]map[string]interface{})
+	if !ok || len(blocks) != 1 || blocks[0]["type"] != "tool_use" {
+		t.Fatalf("expected assistant tool_use block, got %#v", anthropicMsgs[1])
+	}
+}
+
+func TestAnthropicBlocksToAssistant(t *testing.T) {
+	msg := AnthropicBlocksToAssistant([]AnthropicBlock{
+		{Type: "text", Text: "Checking"},
+		{Type: "tool_use", ID: "toolu_1", Name: "get_weather", Input: map[string]interface{}{"city": "Boston"}},
+	})
+	if msg.Content != "Checking" {
+		t.Fatalf("unexpected content: %q", msg.Content)
+	}
+	if len(msg.ToolCalls) != 1 || msg.ToolCalls[0].Function.Name != "get_weather" {
+		t.Fatalf("unexpected tool calls: %#v", msg.ToolCalls)
+	}
+}
+
+func TestMessagesToGemini(t *testing.T) {
+	msgs := []Message{
+		{Role: "user", Content: json.RawMessage(`"hi"`)},
+		{
+			Role: "assistant",
+			ToolCalls: []ToolCall{{
+				Function: ToolFunction{Name: "fn", Arguments: `{"a":1}`},
+			}},
+		},
+		{Role: "tool", Name: "fn", Content: json.RawMessage(`{"result":"ok"}`)},
+	}
+	contents, err := MessagesToGemini(msgs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(contents) != 3 {
+		t.Fatalf("expected 3 contents, got %d", len(contents))
+	}
+}
+
+func TestExtractTools(t *testing.T) {
+	body, _ := jsonutils.Parse([]byte(`{
+		"tools":[{"type":"function","function":{"name":"fn","parameters":{"type":"object"}}}],
+		"tool_choice":"auto"
+	}`))
+	tools, choice, err := ExtractTools(body.(*jsonutils.JSONDict))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(tools) != 1 || tools[0].Function.Name != "fn" {
+		t.Fatalf("unexpected tools: %#v", tools)
+	}
+	if string(choice) != `"auto"` {
+		t.Fatalf("unexpected tool_choice: %s", choice)
+	}
+}
+
+func TestNewChatCompletionWithTools(t *testing.T) {
+	out := NewChatCompletionWithTools("m", "id", AssistantMessage{
+		ToolCalls: []ToolCall{{
+			ID: "call_1", Type: "function",
+			Function: ToolFunction{Name: "fn", Arguments: `{}`},
+		}},
+	}, "tool_use", 1, 2)
+	choices := out["choices"].([]map[string]interface{})
+	msg := choices[0]["message"].(map[string]interface{})
+	if msg["tool_calls"] == nil {
+		t.Fatal("expected tool_calls in message")
+	}
+	if choices[0]["finish_reason"] != "tool_calls" {
+		t.Fatalf("expected finish_reason tool_calls, got %v", choices[0]["finish_reason"])
+	}
+}

pkg/aiproxy/providers/providers_test.go

@@ -0,0 +1,196 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"encoding/json"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestAliyunProviderEnableThinkingPatch(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("qwen-turbo"), "model")
+	body.Add(jsonutils.NewArray(jsonutils.NewDict()), "messages")
+
+	p := Get("aliyun")
+	req, err := p.BuildUpstreamRequest(&ChatContext{
+		ProviderKey:   "aliyun",
+		BaseURL:       "https://dashscope.aliyuncs.com/compatible-mode",
+		APIKey:        "sk-test",
+		UpstreamModel: "qwen-turbo",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var wire map[string]interface{}
+	if err := json.Unmarshal(req.Body, &wire); err != nil {
+		t.Fatal(err)
+	}
+	if v, ok := wire["enable_thinking"].(bool); !ok || v {
+		t.Fatalf("expected enable_thinking=false, got %#v", wire["enable_thinking"])
+	}
+}
+
+func TestAnthropicProviderBuildRequest(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("claude-3-5-sonnet"), "model")
+	body.Add(jsonutils.NewInt(1024), "max_tokens")
+	sysMsg := jsonutils.NewDict()
+	sysMsg.Set("role", jsonutils.NewString("system"))
+	sysMsg.Set("content", jsonutils.NewString("You are helpful."))
+	userMsg := jsonutils.NewDict()
+	userMsg.Set("role", jsonutils.NewString("user"))
+	userMsg.Set("content", jsonutils.NewString("Hi"))
+	msgs := jsonutils.NewArray(sysMsg, userMsg)
+	body.Add(msgs, "messages")
+
+	p := Get("anthropic")
+	req, err := p.BuildUpstreamRequest(&ChatContext{
+		ProviderKey:   "anthropic",
+		BaseURL:       "https://api.anthropic.com",
+		APIKey:        "sk-ant",
+		UpstreamModel: "claude-3-5-sonnet-20241022",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://api.anthropic.com/v1/messages" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+	if req.Headers["x-api-key"] != "sk-ant" {
+		t.Fatalf("missing x-api-key header")
+	}
+	var wire map[string]interface{}
+	if err := json.Unmarshal(req.Body, &wire); err != nil {
+		t.Fatal(err)
+	}
+	if wire["system"] != "You are helpful." {
+		t.Fatalf("expected system prompt, got %#v", wire["system"])
+	}
+}
+
+func TestAnthropicNormalizeResponse(t *testing.T) {
+	p := Get("anthropic")
+	raw := []byte(`{
+		"id":"msg_1",
+		"model":"claude-3-5-sonnet-20241022",
+		"content":[{"type":"text","text":"Hello"}],
+		"stop_reason":"end_turn",
+		"usage":{"input_tokens":3,"output_tokens":1}
+	}`)
+	out, err := p.NormalizeResponse(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		t.Fatal(err)
+	}
+	if resp["object"] != "chat.completion" {
+		t.Fatalf("unexpected object: %#v", resp["object"])
+	}
+	choices := resp["choices"].([]interface{})
+	msg := choices[0].(map[string]interface{})["message"].(map[string]interface{})
+	if msg["content"] != "Hello" {
+		t.Fatalf("unexpected content: %#v", msg["content"])
+	}
+}
+
+func TestAnthropicToolCalling(t *testing.T) {
+	body := jsonutils.NewDict()
+	userMsg := jsonutils.NewDict()
+	userMsg.Set("role", jsonutils.NewString("user"))
+	userMsg.Set("content", jsonutils.NewString("Weather in Boston?"))
+	msgs := jsonutils.NewArray(userMsg)
+	body.Add(msgs, "messages")
+	tool := jsonutils.NewDict()
+	tool.Set("type", jsonutils.NewString("function"))
+	fn := jsonutils.NewDict()
+	fn.Set("name", jsonutils.NewString("get_weather"))
+	fn.Set("parameters", jsonutils.NewDict())
+	tool.Set("function", fn)
+	body.Add(jsonutils.NewArray(tool), "tools")
+
+	p := Get("anthropic")
+	req, err := p.BuildUpstreamRequest(&ChatContext{
+		BaseURL:       "https://api.anthropic.com",
+		APIKey:        "sk-ant",
+		UpstreamModel: "claude-3-5-sonnet-20241022",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var wire map[string]interface{}
+	if err := json.Unmarshal(req.Body, &wire); err != nil {
+		t.Fatal(err)
+	}
+	tools, ok := wire["tools"].([]interface{})
+	if !ok || len(tools) != 1 {
+		t.Fatalf("expected tools in request, got %#v", wire["tools"])
+	}
+
+	raw := []byte(`{
+		"id":"msg_2",
+		"model":"claude-3-5-sonnet-20241022",
+		"content":[{"type":"tool_use","id":"toolu_1","name":"get_weather","input":{"city":"Boston"}}],
+		"stop_reason":"tool_use",
+		"usage":{"input_tokens":10,"output_tokens":5}
+	}`)
+	out, err := p.NormalizeResponse(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var resp map[string]interface{}
+	if err := json.Unmarshal(out, &resp); err != nil {
+		t.Fatal(err)
+	}
+	choices := resp["choices"].([]interface{})
+	choice := choices[0].(map[string]interface{})
+	if choice["finish_reason"] != "tool_calls" {
+		t.Fatalf("expected finish_reason tool_calls, got %#v", choice["finish_reason"])
+	}
+	msg := choice["message"].(map[string]interface{})
+	if msg["tool_calls"] == nil {
+		t.Fatal("expected tool_calls in normalized response")
+	}
+}
+
+func TestOpenAICompatPassthrough(t *testing.T) {
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("gpt-4o"), "model")
+	body.Add(jsonutils.NewArray(jsonutils.NewDict()), "messages")
+
+	p := Get("openai")
+	req, err := p.BuildUpstreamRequest(&ChatContext{
+		BaseURL:       "https://api.openai.com",
+		APIKey:        "sk-test",
+		UpstreamModel: "gpt-4o-mini",
+	}, body, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "https://api.openai.com/v1/chat/completions" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+}
+
+func TestRegistryFallback(t *testing.T) {
+	p := Get("unknown-provider-key")
+	if !p.OpenAIStreamPassthrough() {
+		t.Fatal("unknown provider should use openai-compatible passthrough")
+	}
+}

pkg/aiproxy/providers/registry.go

@@ -0,0 +1,105 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	"strings"
+	"sync"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/aliyun"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/anthropic"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/azure"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/baidu"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/cohere"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/gemini"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/vllm"
+)
+
+var (
+	registryMu sync.RWMutex
+	registry   = map[string]providerapi.Provider{}
+	defaultP   providerapi.Provider
+)
+
+var openAICompatKeys = []string{
+	"openai",
+	"groq",
+	"mistral",
+	"cerebras",
+	"perplexity",
+	"openrouter",
+	"fireworks",
+	"nebius",
+	"xai",
+	"parasail",
+	"sgl",
+	"huggingface",
+	"ollama",
+	"xiaomi",
+}
+
+func init() {
+	defaultP = openai.NewCompat("")
+	register(defaultP)
+	for _, key := range openAICompatKeys {
+		register(openai.NewCompat(key))
+	}
+	register(cohere.New())
+	register(aliyun.New())
+	register(baidu.New())
+	register(anthropic.New())
+	register(gemini.New())
+	register(azure.New())
+	register(vllm.New())
+}
+
+// Register adds or replaces a provider implementation for its Key().
+func Register(p Provider) {
+	if p == nil {
+		return
+	}
+	register(p)
+}
+
+func register(p providerapi.Provider) {
+	registryMu.Lock()
+	defer registryMu.Unlock()
+	k := normalizeKey(p.Key())
+	if k == "" {
+		registry[""] = p
+		return
+	}
+	registry[k] = p
+}
+
+// Get returns the provider for providerKey, or the default OpenAI-compatible passthrough.
+func Get(providerKey string) Provider {
+	registryMu.RLock()
+	defer registryMu.RUnlock()
+	k := normalizeKey(providerKey)
+	if p, ok := registry[k]; ok {
+		return p
+	}
+	if defaultP != nil {
+		return defaultP
+	}
+	return openai.NewCompat("")
+}
+
+func normalizeKey(k string) string {
+	return strings.ToLower(strings.TrimSpace(k))
+}

pkg/aiproxy/providers/stream.go

@@ -0,0 +1,25 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import api "yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+
+// OpenAIStreamPassthrough reports whether upstream SSE chunks are already OpenAI-compatible.
+func OpenAIStreamPassthrough(prov Provider, ctx *ChatContext) bool {
+	if p, ok := prov.(api.ContextualStreamPassthrough); ok {
+		return p.OpenAIStreamPassthroughForContext(ctx)
+	}
+	return prov.OpenAIStreamPassthrough()
+}

pkg/aiproxy/providers/types.go

@@ -0,0 +1,32 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package providers
+
+import (
+	api "yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+type (
+	ChatContext         = api.ChatContext
+	HTTPRequest         = api.HTTPRequest
+	StreamChunk         = api.StreamChunk
+	StreamState         = api.StreamState
+	Provider            = api.Provider
+	EmbeddingsProvider  = api.EmbeddingsProvider
+	ImagesProvider      = api.ImagesProvider
+	CompletionsProvider = api.CompletionsProvider
+)
+
+type ContextualStreamPassthrough = api.ContextualStreamPassthrough

pkg/aiproxy/providers/vllm/doc.go

@@ -0,0 +1,2 @@
+// Package vllm adapts OpenAI-compatible requests to vLLM OpenAI API servers.
+package vllm // import "yunion.io/x/onecloud/pkg/aiproxy/providers/vllm"

pkg/aiproxy/providers/vllm/vllm.go

@@ -0,0 +1,57 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vllm
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+	"yunion.io/x/onecloud/pkg/aiproxy/providers/openai"
+)
+
+type provider struct {
+	*openai.Compat
+	completions *openai.CompletionsCompat
+}
+
+func patchVLLMRequest(body *jsonutils.JSONDict, stream bool) {
+	if body == nil {
+		return
+	}
+	if !stream {
+		body.Remove("stream_options")
+	}
+}
+
+// New returns the vLLM OpenAI-compatible provider adapter.
+func New() providerapi.Provider {
+	patches := []openai.PatchFunc{patchVLLMRequest}
+	return &provider{
+		Compat:      openai.NewCompat("vllm", patches...),
+		completions: openai.NewCompletionsCompat(patches...),
+	}
+}
+
+func (p *provider) BuildCompletionsRequest(ctx *providerapi.ChatContext, body *jsonutils.JSONDict, stream bool) (*providerapi.HTTPRequest, error) {
+	return p.completions.BuildCompletionsRequest(ctx, body, stream)
+}
+
+func (p *provider) NormalizeCompletionsResponse(body []byte) ([]byte, error) {
+	return p.completions.NormalizeCompletionsResponse(body)
+}
+
+func (p *provider) OpenAICompletionsStreamPassthrough() bool {
+	return p.completions.OpenAICompletionsStreamPassthrough()
+}

pkg/aiproxy/providers/vllm/vllm_test.go

@@ -0,0 +1,77 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vllm
+
+import (
+	"strings"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/providerapi"
+)
+
+func TestVLLMProviderBuildCompletionsRequest(t *testing.T) {
+	p := New()
+	cp, ok := p.(providerapi.CompletionsProvider)
+	if !ok {
+		t.Fatal("vllm provider should implement CompletionsProvider")
+	}
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString("hello"), "prompt")
+	streamOpts := jsonutils.NewDict()
+	streamOpts.Add(jsonutils.JSONTrue, "include_usage")
+	body.Add(streamOpts, "stream_options")
+
+	req, err := cp.BuildCompletionsRequest(&providerapi.ChatContext{
+		BaseURL:       "http://127.0.0.1:8000",
+		UpstreamModel: "Qwen/Qwen2.5-7B-Instruct",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "http://127.0.0.1:8000/v1/completions" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+	if strings.Contains(string(req.Body), "stream_options") {
+		t.Fatalf("stream_options should be stripped for non-stream requests: %s", req.Body)
+	}
+}
+
+func TestVLLMProviderBuildChatCompletionsRequest(t *testing.T) {
+	p := New()
+	body := jsonutils.NewDict()
+	msg := jsonutils.NewDict()
+	msg.Add(jsonutils.NewString("user"), "role")
+	msg.Add(jsonutils.NewString("hi"), "content")
+	body.Add(jsonutils.NewArray(msg), "messages")
+	streamOpts := jsonutils.NewDict()
+	streamOpts.Add(jsonutils.JSONTrue, "include_usage")
+	body.Add(streamOpts, "stream_options")
+
+	req, err := p.BuildUpstreamRequest(&providerapi.ChatContext{
+		BaseURL:       "http://127.0.0.1:8000",
+		UpstreamModel: "Qwen/Qwen2.5-7B-Instruct",
+	}, body, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if req.URL != "http://127.0.0.1:8000/v1/chat/completions" {
+		t.Fatalf("unexpected url: %s", req.URL)
+	}
+	if strings.Contains(string(req.Body), "stream_options") {
+		t.Fatalf("stream_options should be stripped for non-stream requests: %s", req.Body)
+	}
+}

pkg/aiproxy/service/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service // import "yunion.io/x/onecloud/pkg/aiproxy/service"

pkg/aiproxy/service/service.go

@@ -0,0 +1,92 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"os"
+	"time"
+
+	"yunion.io/x/log"
+	_ "yunion.io/x/sqlchemy/backends"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/handlers"
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/options"
+	apolicy "yunion.io/x/onecloud/pkg/aiproxy/policy"
+	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/cachesync"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+)
+
+func StartService() {
+	opts := &options.Options
+	commonOpts := &opts.CommonOptions
+	dbOpts := &opts.DBOptions
+	baseOpts := &opts.BaseOptions
+	common_options.ParseOptions(opts, os.Args, "aiproxy.conf", api.SERVICE_TYPE)
+	apolicy.Init()
+	if err := models.InitLocalProxyNodeId(opts, opts.IsSlaveNode); err != nil {
+		log.Fatalf("init local proxy node id: %v", err)
+	}
+
+	app_common.InitAuth(commonOpts, func() {
+		log.Infof("Auth complete!!")
+	})
+	common_options.StartOptionManager(opts, opts.ConfigSyncPeriodSeconds, api.SERVICE_TYPE, api.SERVICE_VERSION, options.OnOptionsChange)
+
+	app := app_common.InitApp(&opts.BaseOptions, false)
+
+	cloudcommon.InitDB(dbOpts)
+	handlers.InitHandlers(app, opts.IsSlaveNode)
+
+	if opts.IsSlaveNode {
+		if !db.CheckSync(false, dbOpts.EnableDBChecksumTables, dbOpts.DBChecksumSkipInit) {
+			log.Fatalf("database schema not in sync!")
+		}
+		if dbOpts.ExitAfterDBInit {
+			log.Infof("Exiting after db initialization ...")
+			os.Exit(0)
+		}
+		db.AppDBInit(app)
+		startSlaveNodeRegisterLoop(opts)
+	} else {
+		db.EnsureAppSyncDB(app, dbOpts, models.InitDB)
+	}
+	defer cloudcommon.CloseDB()
+
+	if !opts.IsSlaveNode {
+		err := taskman.TaskManager.InitializeData()
+		if err != nil {
+			log.Fatalf("TaskManager.InitializeData fail %s", err)
+		}
+
+		cachesync.StartTenantCacheSync(opts.TenantCacheExpireSeconds)
+
+		cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount, opts.TimeZone)
+		cron.AddJobAtIntervalsWithStartRun("TaskCleanupJob", time.Duration(options.Options.TaskArchiveIntervalMinutes)*time.Minute, taskman.TaskManager.TaskCleanupJob, true)
+
+		cron.Start()
+		defer cron.Stop()
+	}
+
+	app_common.ServeForeverWithCleanup(app, baseOpts, func() {
+		cloudcommon.CloseDB()
+	})
+}

pkg/aiproxy/service/slave_register.go

@@ -0,0 +1,67 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"context"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/httputils"
+
+	"yunion.io/x/onecloud/pkg/aiproxy/models"
+	"yunion.io/x/onecloud/pkg/aiproxy/options"
+	"yunion.io/x/onecloud/pkg/apis/identity"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	apmodules "yunion.io/x/onecloud/pkg/mcclient/modules/aiproxy"
+)
+
+func startSlaveNodeRegisterLoop(opts *options.SAiProxyOptions) {
+	interval := time.Duration(opts.NodeHeartbeatIntervalSeconds) * time.Second
+	if interval <= 0 {
+		interval = 60 * time.Second
+	}
+	address, err := models.AdvertiseAddressFromOptions(opts)
+	if err != nil {
+		log.Fatalf("invalid standby advertise address: %v", err)
+	}
+
+	go func() {
+		ctx := context.Background()
+		register := func() {
+			session := auth.GetAdminSessionWithPublic(ctx, opts.Region)
+			master, err := session.GetServiceURL(apmodules.AiProxyNodes.ServiceType(), identity.EndpointInterfacePublic, httputils.POST)
+			if err != nil {
+				log.Errorf("aiproxy standby resolve primary public endpoint failed: %v", err)
+				return
+			}
+			params := jsonutils.Marshal(map[string]interface{}{
+				"address": address,
+			})
+			if _, err := apmodules.AiProxyNodes.PerformClassAction(session, "register", params); err != nil {
+				log.Errorf("aiproxy standby register with primary %s failed: %v", master, err)
+				return
+			}
+			log.Debugf("aiproxy standby registered with primary %s as %s", master, address)
+		}
+		register()
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+		for range ticker.C {
+			register()
+		}
+	}()
+}

pkg/aiproxy/upstream/doc.go

@@ -0,0 +1 @@
+package upstream // import "yunion.io/x/onecloud/pkg/aiproxy/upstream"

pkg/aiproxy/upstream/openai_compat.go

@@ -0,0 +1,281 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package upstream
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Request is one upstream chat call (OpenAI-compatible by default, or provider-native when URL/headers are set).
+type Request struct {
+	BaseURL string
+	URL     string
+	APIKey  string
+	Headers map[string]string
+	Body    []byte
+}
+
+// Response is a non-streaming upstream response body.
+type Response struct {
+	StatusCode int
+	Body       []byte
+}
+
+// StreamChunk is one SSE event payload (bytes after "data: ").
+type StreamChunk struct {
+	Data []byte
+	Done bool
+}
+
+// RawSSEEvent is one parsed server-sent event line group from an upstream.
+type RawSSEEvent struct {
+	Event string
+	Data  []byte
+}
+
+// Error carries upstream HTTP status and optional JSON error body.
+type Error struct {
+	StatusCode int
+	Message    string
+	Body       []byte
+}
+
+func (e *Error) Error() string {
+	if e == nil {
+		return ""
+	}
+	if e.Message != "" {
+		return e.Message
+	}
+	if len(e.Body) > 0 {
+		return string(e.Body)
+	}
+	return fmt.Sprintf("upstream HTTP %d", e.StatusCode)
+}
+
+// ChatCompletionsURL builds the chat completions endpoint from a provider base URL.
+// BaseURL is the origin + optional path prefix (e.g. https://dashscope.aliyuncs.com/compatible-mode).
+func ChatCompletionsURL(baseURL string) string {
+	base := strings.TrimRight(strings.TrimSpace(baseURL), "/")
+	if strings.HasSuffix(base, "/v1") {
+		return base + "/chat/completions"
+	}
+	return base + "/v1/chat/completions"
+}
+
+var (
+	httpClient     *http.Client
+	httpClientOnce sync.Once
+)
+
+func sharedHTTPClient() *http.Client {
+	httpClientOnce.Do(func() {
+		httpClient = &http.Client{
+			Transport: &http.Transport{
+				MaxIdleConns:        256,
+				MaxIdleConnsPerHost: 64,
+				IdleConnTimeout:     90 * time.Second,
+			},
+		}
+	})
+	return httpClient
+}
+
+func requestURL(req *Request) string {
+	if req == nil {
+		return ""
+	}
+	if u := strings.TrimSpace(req.URL); u != "" {
+		return u
+	}
+	return ChatCompletionsURL(req.BaseURL)
+}
+
+func newUpstreamRequest(ctx context.Context, req *Request) (*http.Request, error) {
+	if req == nil {
+		return nil, fmt.Errorf("nil upstream request")
+	}
+	url := requestURL(req)
+	apiKey := strings.TrimSpace(req.APIKey)
+	if url == "" {
+		return nil, fmt.Errorf("empty upstream URL")
+	}
+	if apiKey == "" && len(req.Headers) == 0 {
+		return nil, fmt.Errorf("empty API key")
+	}
+	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(req.Body))
+	if err != nil {
+		return nil, err
+	}
+	httpReq.Header.Set("Content-Type", "application/json")
+	for k, v := range req.Headers {
+		httpReq.Header.Set(k, v)
+	}
+	if apiKey != "" && httpReq.Header.Get("Authorization") == "" && httpReq.Header.Get("x-api-key") == "" && httpReq.Header.Get("api-key") == "" && httpReq.Header.Get("x-goog-api-key") == "" {
+		httpReq.Header.Set("Authorization", "Bearer "+apiKey)
+	}
+	return httpReq, nil
+}
+
+func readResponseBody(resp *http.Response, maxBytes int64) ([]byte, error) {
+	defer resp.Body.Close()
+	if maxBytes <= 0 {
+		maxBytes = 32 << 20
+	}
+	return io.ReadAll(io.LimitReader(resp.Body, maxBytes))
+}
+
+func errorFromResponse(resp *http.Response, body []byte) *Error {
+	status := resp.StatusCode
+	msg := strings.TrimSpace(resp.Status)
+	if len(body) > 0 {
+		var wrap struct {
+			Error *struct {
+				Message string `json:"message"`
+			} `json:"error"`
+		}
+		if err := json.Unmarshal(body, &wrap); err == nil && wrap.Error != nil && wrap.Error.Message != "" {
+			msg = wrap.Error.Message
+		}
+	}
+	return &Error{StatusCode: status, Message: msg, Body: body}
+}
+
+// ChatCompletion performs a non-streaming chat completions request.
+func ChatCompletion(ctx context.Context, req *Request) (*Response, *Error) {
+	httpReq, err := newUpstreamRequest(ctx, req)
+	if err != nil {
+		return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
+	}
+	resp, err := sharedHTTPClient().Do(httpReq)
+	if err != nil {
+		return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
+	}
+	body, err := readResponseBody(resp, 32<<20)
+	if err != nil {
+		return nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return nil, errorFromResponse(resp, body)
+	}
+	return &Response{StatusCode: resp.StatusCode, Body: body}, nil
+}
+
+// ChatCompletionStream opens a streaming chat completions request and returns SSE data chunks.
+func ChatCompletionStream(ctx context.Context, req *Request) (<-chan StreamChunk, *Error) {
+	_, resp, uerr := openChatCompletionStream(ctx, req)
+	if uerr != nil {
+		return nil, uerr
+	}
+	out := make(chan StreamChunk, 16)
+	go func() {
+		defer close(out)
+		defer resp.Body.Close()
+		for evt := range readSSE(resp.Body) {
+			if evt.Done {
+				out <- StreamChunk{Done: true}
+				return
+			}
+			out <- StreamChunk{Data: evt.Data}
+		}
+		if ctx.Err() != nil {
+			return
+		}
+	}()
+	return out, nil
+}
+
+// ChatCompletionStreamRaw opens a streaming request and returns raw SSE events (event + data lines).
+func ChatCompletionStreamRaw(ctx context.Context, req *Request) (<-chan RawSSEEvent, *Error) {
+	_, resp, uerr := openChatCompletionStream(ctx, req)
+	if uerr != nil {
+		return nil, uerr
+	}
+	out := make(chan RawSSEEvent, 16)
+	go func() {
+		defer close(out)
+		defer resp.Body.Close()
+		for evt := range readSSE(resp.Body) {
+			if evt.Done {
+				return
+			}
+			out <- RawSSEEvent{Event: evt.Event, Data: evt.Data}
+		}
+	}()
+	return out, nil
+}
+
+type sseFrame struct {
+	Event string
+	Data  []byte
+	Done  bool
+}
+
+func openChatCompletionStream(ctx context.Context, req *Request) (*http.Request, *http.Response, *Error) {
+	httpReq, err := newUpstreamRequest(ctx, req)
+	if err != nil {
+		return nil, nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
+	}
+	httpReq.Header.Set("Accept", "text/event-stream")
+	resp, err := sharedHTTPClient().Do(httpReq)
+	if err != nil {
+		return nil, nil, &Error{StatusCode: http.StatusBadGateway, Message: err.Error()}
+	}
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		body, _ := readResponseBody(resp, 1<<20)
+		return nil, nil, errorFromResponse(resp, body)
+	}
+	return httpReq, resp, nil
+}
+
+func readSSE(r io.Reader) <-chan sseFrame {
+	out := make(chan sseFrame, 16)
+	go func() {
+		defer close(out)
+		sc := bufio.NewScanner(r)
+		sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
+		var pendingEvent string
+		for sc.Scan() {
+			line := strings.TrimSpace(sc.Text())
+			if line == "" {
+				continue
+			}
+			if strings.HasPrefix(line, "event:") {
+				pendingEvent = strings.TrimSpace(strings.TrimPrefix(line, "event:"))
+				continue
+			}
+			if !strings.HasPrefix(line, "data:") {
+				continue
+			}
+			payload := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
+			if payload == "[DONE]" {
+				out <- sseFrame{Done: true}
+				return
+			}
+			out <- sseFrame{Event: pendingEvent, Data: []byte(payload)}
+			pendingEvent = ""
+		}
+	}()
+	return out
+}

pkg/apis/aiproxy/ai_key.go

@@ -0,0 +1,91 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"encoding/json"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+// SAiKeyRouting constrains which client request models may use an ai_key, and relative priority.
+// Matching uses the JSON body "model" string (same as ai_routing.model_pattern: exact match, case-insensitive,
+// or prefix* glob when the pattern ends with "*").
+//
+// AllowedModelKeys: when non-empty, the requested model must match at least one entry.
+// When empty, any model is allowed unless blocked by BlockedModelKeys.
+//
+// BlockedModelKeys: requested model must not match any entry (evaluated after allow-list pass).
+//
+// Weight in routing is legacy; prefer the ai_key.weight column. When ai_key.weight is unset (0), routing.weight is used.
+type SAiKeyRouting struct {
+	AllowedModelKeys []string `json:"allowed_model_keys,omitempty"`
+	BlockedModelKeys []string `json:"blocked_model_keys,omitempty"`
+	Weight           int      `json:"weight,omitempty"`
+}
+
+// String implements gotypes.ISerializable for sqlchemy JSON/compound columns.
+func (r *SAiKeyRouting) String() string {
+	if r == nil {
+		return "{}"
+	}
+	b, err := json.Marshal(r)
+	if err != nil {
+		return "{}"
+	}
+	return string(b)
+}
+
+// IsZero implements gotypes.ISerializable.
+func (r *SAiKeyRouting) IsZero() bool {
+	if r == nil {
+		return true
+	}
+	return len(r.AllowedModelKeys) == 0 && len(r.BlockedModelKeys) == 0 && r.Weight == 0
+}
+
+type AiKeyListInput struct {
+	apis.EnabledStatusStandaloneResourceListInput
+
+	AiProviderId string `json:"ai_provider_id"`
+}
+
+type AiKeyCreateInput struct {
+	apis.EnabledStatusStandaloneResourceCreateInput
+
+	AiProviderId string         `json:"ai_provider_id"`
+	Secret       string         `json:"secret"`
+	Weight       int            `json:"weight"`
+	Routing      *SAiKeyRouting `json:"routing"`
+}
+
+type AiKeyUpdateInput struct {
+	apis.EnabledStatusStandaloneResourceBaseUpdateInput
+
+	AiProviderId string         `json:"ai_provider_id"`
+	Secret       string         `json:"secret"`
+	Weight       int            `json:"weight,omitzero"`
+	Routing      *SAiKeyRouting `json:"routing"`
+	Enabled      *bool          `json:"enabled"`
+}
+
+type AiKeyDetails struct {
+	apis.EnabledStatusStandaloneResourceDetails
+
+	AiProviderId   string         `json:"ai_provider_id"`
+	AiProviderName string         `json:"ai_provider_name"`
+	Weight         int            `json:"weight"`
+	Routing        *SAiKeyRouting `json:"routing"`
+}

pkg/apis/aiproxy/ai_model.go

@@ -0,0 +1,49 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type AiModelListInput struct {
+	apis.EnabledStatusStandaloneResourceListInput
+
+	AiProviderId string `json:"ai_provider_id"`
+	ModelKey     string `json:"model_key"`
+}
+
+type AiModelCreateInput struct {
+	apis.EnabledStatusStandaloneResourceCreateInput
+
+	AiProviderId string `json:"ai_provider_id"`
+	ModelKey     string `json:"model_key"`
+}
+
+type AiModelUpdateInput struct {
+	apis.EnabledStatusStandaloneResourceBaseUpdateInput
+
+	AiProviderId string `json:"ai_provider_id"`
+	ModelKey     string `json:"model_key"`
+	Enabled      *bool  `json:"enabled"`
+}
+
+type AiModelDetails struct {
+	apis.EnabledStatusStandaloneResourceDetails
+
+	AiProviderId   string `json:"ai_provider_id"`
+	AiProviderName string `json:"ai_provider_name"`
+	ModelKey       string `json:"model_key"`
+}

pkg/apis/aiproxy/ai_provider.go

@@ -0,0 +1,92 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"encoding/json"
+	"strings"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+// SAiProviderConfig holds JSON-serialized provider connectivity settings for an ai_provider row.
+type SAiProviderConfig struct {
+	BaseURL string `json:"base_url,omitempty"`
+	APIKey  string `json:"api_key,omitempty"`
+}
+
+// ResolvedBaseURL returns config.base_url.
+func (c *SAiProviderConfig) ResolvedBaseURL() string {
+	if c == nil {
+		return ""
+	}
+	return strings.TrimSpace(c.BaseURL)
+}
+
+// ResolvedAPIKey returns config.api_key.
+func (c *SAiProviderConfig) ResolvedAPIKey() string {
+	if c == nil {
+		return ""
+	}
+	return strings.TrimSpace(c.APIKey)
+}
+
+// String implements gotypes.ISerializable for sqlchemy JSON/compound columns.
+func (c *SAiProviderConfig) String() string {
+	if c == nil {
+		return "{}"
+	}
+	b, err := json.Marshal(c)
+	if err != nil {
+		return "{}"
+	}
+	return string(b)
+}
+
+// IsZero implements gotypes.ISerializable.
+func (c *SAiProviderConfig) IsZero() bool {
+	if c == nil {
+		return true
+	}
+	return c.ResolvedBaseURL() == "" && c.ResolvedAPIKey() == ""
+}
+
+type AiProviderListInput struct {
+	apis.EnabledStatusStandaloneResourceListInput
+
+	ProviderKey string `json:"provider_key"`
+}
+
+type AiProviderCreateInput struct {
+	apis.EnabledStatusStandaloneResourceCreateInput
+
+	ProviderKey string             `json:"provider_key"`
+	Config      *SAiProviderConfig `json:"config"`
+}
+
+type AiProviderUpdateInput struct {
+	apis.EnabledStatusStandaloneResourceBaseUpdateInput
+
+	ProviderKey string             `json:"provider_key"`
+	Config      *SAiProviderConfig `json:"config"`
+	Enabled     *bool              `json:"enabled"`
+}
+
+type AiProviderDetails struct {
+	apis.EnabledStatusStandaloneResourceDetails
+
+	ProviderKey string             `json:"provider_key"`
+	Config      *SAiProviderConfig `json:"config"`
+}

pkg/apis/aiproxy/ai_proxy_node.go

@@ -0,0 +1,65 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"time"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type AiProxyNodeListInput struct {
+	apis.EnabledStatusStandaloneResourceListInput
+
+	Address string `json:"address"`
+	Domain  string `json:"domain"`
+}
+
+type AiProxyNodeCreateInput struct {
+	apis.EnabledStatusStandaloneResourceCreateInput
+
+	Address   string `json:"address"`
+	Domain    string `json:"domain"`
+	HbTimeout int    `json:"hb_timeout"`
+}
+
+type AiProxyNodeUpdateInput struct {
+	apis.EnabledStatusStandaloneResourceBaseUpdateInput
+
+	Address   string `json:"address"`
+	Domain    string `json:"domain"`
+	HbTimeout int    `json:"hb_timeout"`
+	Enabled   *bool  `json:"enabled"`
+}
+
+type AiProxyNodeDetails struct {
+	apis.EnabledStatusStandaloneResourceDetails
+
+	Address   string    `json:"address"`
+	Domain    string    `json:"domain"`
+	LastSeen  time.Time `json:"last_seen"`
+	HbTimeout int       `json:"hb_timeout"`
+	IsActive  bool      `json:"is_active"`
+}
+
+// AiProxyNodeRegisterInput is sent by standby instances to the primary on startup and heartbeat.
+type AiProxyNodeRegisterInput struct {
+	Address   string `json:"address"`
+	HbTimeout int    `json:"hb_timeout"`
+}
+
+type AiProxyNodeRegisterOutput struct {
+	Id string `json:"id"`
+}

pkg/apis/aiproxy/ai_routing.go

@@ -0,0 +1,72 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type AiRoutingListInput struct {
+	apis.SharableVirtualResourceListInput
+	apis.EnabledResourceBaseListInput
+
+	ModelPattern  string `json:"model_pattern"`
+	AiProxyNodeId string `json:"ai_proxy_node_id"`
+}
+
+// AiRoutingModelItem is one catalog model binding when creating ai_routing.
+// Priority orders models within the routing (lower = higher priority). Weight is an alias for Priority.
+type AiRoutingModelItem struct {
+	AiProviderId string `json:"ai_provider_id"`
+	AiModelId    string `json:"ai_model_id"`
+	Priority     int    `json:"priority,omitempty"`
+	Weight       int    `json:"weight,omitempty"`
+	ModelPattern string `json:"model_pattern,omitempty"`
+	Enabled      *bool  `json:"enabled,omitempty"`
+}
+
+type AiRoutingCreateInput struct {
+	apis.SharableVirtualResourceCreateInput
+	apis.EnabledBaseResourceCreateInput
+
+	Priority      int                  `json:"priority"`
+	ModelPattern  string               `json:"model_pattern"`
+	AiProxyNodeId string               `json:"ai_proxy_node_id"`
+	Models        []AiRoutingModelItem `json:"models"`
+}
+
+type AiRoutingUpdateInput struct {
+	apis.SharableVirtualResourceBaseUpdateInput
+
+	Priority      int    `json:"priority"`
+	ModelPattern  string `json:"model_pattern"`
+	AiProxyNodeId string `json:"ai_proxy_node_id"`
+	Enabled       *bool  `json:"enabled"`
+}
+
+type AiRoutingDetails struct {
+	apis.SharableVirtualResourceDetails
+
+	Priority      int                     `json:"priority"`
+	ModelPattern  string                  `json:"model_pattern"`
+	AiProxyNodeId string                  `json:"ai_proxy_node_id"`
+	Enabled       bool                    `json:"enabled"`
+	RoutingModels []AiRoutingModelDetails `json:"routing_models,omitempty"`
+}
+
+// AiRoutingSetModelsInput replaces all ai_routing_models for an ai_routing.
+type AiRoutingSetModelsInput struct {
+	Models []AiRoutingModelItem `json:"models"`
+}

pkg/apis/aiproxy/ai_routing_model.go

@@ -0,0 +1,61 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+type AiRoutingModelListInput struct {
+	apis.StandaloneResourceListInput
+
+	AiRoutingId  string `json:"ai_routing_id"`
+	AiProviderId string `json:"ai_provider_id"`
+	AiModelId    string `json:"ai_model_id"`
+	Enabled      *bool  `json:"enabled"`
+}
+
+type AiRoutingModelCreateInput struct {
+	apis.StandaloneResourceCreateInput
+
+	AiRoutingId  string `json:"ai_routing_id"`
+	AiProviderId string `json:"ai_provider_id"`
+	AiModelId    string `json:"ai_model_id"`
+	Priority     int    `json:"priority"`
+	ModelPattern string `json:"model_pattern"`
+	Enabled      *bool  `json:"enabled"`
+}
+
+type AiRoutingModelUpdateInput struct {
+	apis.StandaloneResourceBaseUpdateInput
+
+	AiRoutingId  string `json:"ai_routing_id,omitempty"`
+	AiProviderId string `json:"ai_provider_id,omitempty"`
+	AiModelId    string `json:"ai_model_id,omitempty"`
+	Priority     int    `json:"priority,omitzero"`
+	ModelPattern string `json:"model_pattern,omitempty"`
+	Enabled      *bool  `json:"enabled,omitempty"`
+}
+
+type AiRoutingModelDetails struct {
+	apis.StandaloneResourceDetails
+
+	AiRoutingId  string `json:"ai_routing_id"`
+	AiProviderId string `json:"ai_provider_id"`
+	AiModelId    string `json:"ai_model_id"`
+	Priority     int    `json:"priority"`
+	ModelPattern string `json:"model_pattern"`
+	Enabled      bool   `json:"enabled"`
+}

pkg/apis/aiproxy/ai_virtual_key.go

@@ -0,0 +1,96 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"encoding/json"
+
+	"yunion.io/x/pkg/tristate"
+
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+// SAiVirtualKeyLimits constrains which catalog providers a virtual key may route to,
+// caps client max_tokens, and configures request rate (approximate per-minute token bucket).
+//
+// AllowedAiProviderIds: when non-empty, the resolved ai_provider must match one entry by id or name (case-insensitive for name).
+// When empty, any provider allowed by routing applies.
+//
+// MaxTokensPerRequest: when > 0, caps the JSON body max_tokens (missing max_tokens is set to this value).
+//
+// RequestsPerMinute: when > 0, enforces an approximate per-minute limit per virtual key row (in-process; multi-replica deployments need external limiting).
+type SAiVirtualKeyLimits struct {
+	AllowedAiProviderIds []string `json:"allowed_ai_provider_ids,omitempty"`
+	MaxTokensPerRequest  int      `json:"max_tokens_per_request,omitempty"`
+	RequestsPerMinute    int      `json:"requests_per_minute,omitempty"`
+}
+
+// String implements gotypes.ISerializable for sqlchemy JSON columns.
+func (l *SAiVirtualKeyLimits) String() string {
+	if l == nil {
+		return "{}"
+	}
+	b, err := json.Marshal(l)
+	if err != nil {
+		return "{}"
+	}
+	return string(b)
+}
+
+// IsZero implements gotypes.ISerializable.
+func (l *SAiVirtualKeyLimits) IsZero() bool {
+	if l == nil {
+		return true
+	}
+	return len(l.AllowedAiProviderIds) == 0 && l.MaxTokensPerRequest == 0 && l.RequestsPerMinute == 0
+}
+
+type AiVirtualKeyListInput struct {
+	apis.VirtualResourceListInput
+	apis.EnabledResourceBaseListInput
+
+	VirtualKey string `json:"virtual_key"`
+	UserId     string `json:"user_id"`
+}
+
+type AiVirtualKeyCreateInput struct {
+	apis.VirtualResourceCreateInput
+
+	// OwnerId is the owning user; defaults to the creating user when empty.
+	OwnerId string `json:"owner_id"`
+	// VirtualKey is optional; when empty a unique sk- prefixed key is generated.
+	VirtualKey string               `json:"virtual_key"`
+	Limits     *SAiVirtualKeyLimits `json:"limits"`
+	Enabled    tristate.TriState    `json:"enabled"`
+}
+
+type AiVirtualKeyUpdateInput struct {
+	apis.VirtualResourceBaseUpdateInput
+
+	OwnerId    string               `json:"owner_id"`
+	VirtualKey string               `json:"virtual_key"`
+	Limits     *SAiVirtualKeyLimits `json:"limits"`
+	Enabled    tristate.TriState    `json:"enabled"`
+}
+
+type AiVirtualKeyDetails struct {
+	apis.VirtualResourceDetails
+
+	OwnerId    string               `json:"owner_id"`
+	OwnerName  string               `json:"owner_name"`
+	VirtualKey string               `json:"virtual_key"`
+	Limits     *SAiVirtualKeyLimits `json:"limits"`
+	Enabled    bool                 `json:"enabled"`
+}

pkg/apis/aiproxy/consts.go

@@ -0,0 +1,24 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy
+
+import (
+	"yunion.io/x/onecloud/pkg/apis"
+)
+
+const (
+	SERVICE_TYPE    = apis.SERVICE_TYPE_AIPROXY
+	SERVICE_VERSION = ""
+)

pkg/apis/aiproxy/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aiproxy // import "yunion.io/x/onecloud/pkg/apis/aiproxy"