Feature/llm route for aiproxy (#25044)

* feat(aiproxy): support routing model in aiproxy

* feat(llm): add llm-router

pkg/aiproxy/models/ai_routing_models.go

@@ -15,11 +15,17 @@
 package models
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
+	"encoding/json"
 	stderrors "errors"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
 	"strings"
+	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
@@ -315,7 +321,7 @@ func fetchEnabledAiRoutingModels(routingId string) ([]SAiRoutingModel, error) {
 }
 
 // pickAiRoutingModel selects provider/model from ai_routing_models by request model name.
-func pickAiRoutingModel(ctx context.Context, userCred mcclient.TokenCredential, routing *SAiRouting, reqModel string) (providerId, modelId string, err error) {
+func pickAiRoutingModel(ctx context.Context, userCred mcclient.TokenCredential, routing *SAiRouting, reqModel string, body *jsonutils.JSONDict) (providerId, modelId string, err error) {
 	if routing == nil {
 		return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing")
 	}
@@ -326,12 +332,186 @@ func pickAiRoutingModel(ctx context.Context, userCred mcclient.TokenCredential,
 	if len(entries) == 0 {
 		return "", "", errors.Wrap(httperrors.ErrInvalidStatus, "ai_routing has no ai_routing_models")
 	}
+	var modelsById map[string]*SAiModel
+	if routing.RouterEnabled {
+		modelIds := make([]string, 0, len(entries))
+		for i := range entries {
+			modelIds = append(modelIds, entries[i].AiModelId)
+		}
+		modelsById, err = fetchEnabledAiModelsByIds(modelIds)
+		if err != nil {
+			return "", "", err
+		}
+	}
+	selected, err := pickAiRoutingModelFromEntries(ctx, routing, reqModel, body, entries, modelsById)
+	if err != nil {
+		return "", "", err
+	}
+	return selected.AiProviderId, selected.AiModelId, nil
+}
+
+type aiRoutingModelCandidate struct {
+	entry     *SAiRoutingModel
+	modelName string
+}
+
+func pickAiRoutingModelFromEntries(
+	ctx context.Context,
+	routing *SAiRouting,
+	reqModel string,
+	body *jsonutils.JSONDict,
+	entries []SAiRoutingModel,
+	modelsById map[string]*SAiModel,
+) (*SAiRoutingModel, error) {
+	allEntries := make([]*SAiRoutingModel, 0, len(entries))
+	for i := range entries {
+		allEntries = append(allEntries, &entries[i])
+	}
+	if len(allEntries) == 0 {
+		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "ai_routing has no ai_routing_models")
+	}
+
+	if routing != nil && routing.RouterEnabled {
+		candidates := buildAiRoutingModelCandidates(allEntries, modelsById)
+		if len(candidates) == 0 {
+			return routerFallbackPick(routing, allEntries[0], errors.Wrap(httperrors.ErrInvalidStatus, "router has no candidate models"))
+		}
+
+		candidateNames := make([]string, 0, len(candidates))
+		for i := range candidates {
+			candidateNames = append(candidateNames, candidates[i].modelName)
+		}
+		selected, err := callAiRoutingRouter(ctx, routing, body, candidateNames)
+		if err != nil {
+			return routerFallbackPick(routing, allEntries[0], err)
+		}
+		selected = strings.TrimSpace(selected)
+		for i := range candidates {
+			if strings.EqualFold(candidates[i].modelName, selected) {
+				return candidates[i].entry, nil
+			}
+		}
+		return routerFallbackPick(
+			routing,
+			allEntries[0],
+			errors.Wrapf(httperrors.ErrInvalidStatus, "router selected model %q outside candidates", selected),
+		)
+	}
+
+	matches := make([]*SAiRoutingModel, 0, len(entries))
 	for i := range entries {
 		e := &entries[i]
 		if !modelPatternMatches(e.ModelPattern, reqModel) {
 			continue
 		}
-		return e.AiProviderId, e.AiModelId, nil
+		matches = append(matches, e)
 	}
-	return "", "", errors.Wrapf(httperrors.ErrNotFound, "no ai_routing_model matched request model %q", reqModel)
+	if len(matches) == 0 {
+		return nil, errors.Wrapf(httperrors.ErrNotFound, "no ai_routing_model matched request model %q", reqModel)
+	}
+	return matches[0], nil
+}
+
+func buildAiRoutingModelCandidates(entries []*SAiRoutingModel, modelsById map[string]*SAiModel) []aiRoutingModelCandidate {
+	candidates := make([]aiRoutingModelCandidate, 0, len(entries))
+	for _, entry := range entries {
+		if entry == nil {
+			continue
+		}
+		name := clientFacingModelID(entry, modelsById[entry.AiModelId])
+		if name == "" {
+			continue
+		}
+		candidates = append(candidates, aiRoutingModelCandidate{
+			entry:     entry,
+			modelName: name,
+		})
+	}
+	return candidates
+}
+
+func routerFallbackPick(routing *SAiRouting, first *SAiRoutingModel, err error) (*SAiRoutingModel, error) {
+	if routing == nil || strings.TrimSpace(routing.RouterFallbackPolicy) == "" || strings.EqualFold(routing.RouterFallbackPolicy, api.AiRoutingRouterFallbackPriority) {
+		return first, nil
+	}
+	return nil, err
+}
+
+func callAiRoutingRouter(ctx context.Context, routing *SAiRouting, body *jsonutils.JSONDict, candidates []string) (string, error) {
+	if routing == nil {
+		return "", errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing")
+	}
+	endpoint, err := aiRoutingRouterEndpoint(routing.RouterUrl, routing.RouterRoutePath)
+	if err != nil {
+		return "", err
+	}
+	payload := jsonutils.NewDict()
+	if body != nil {
+		if model, _ := body.GetString("model"); model != "" {
+			payload.Set("model", jsonutils.NewString(model))
+		}
+		if messages, err := body.Get("messages"); err == nil {
+			payload.Set("messages", messages)
+		}
+	}
+	payload.Set("candidates", jsonutils.Marshal(candidates))
+
+	timeout := routing.RouterTimeoutSeconds
+	if timeout <= 0 {
+		timeout = api.AiRoutingRouterDefaultTimeoutSeconds
+	}
+	reqCtx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
+	defer cancel()
+	req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, endpoint, bytes.NewReader([]byte(payload.String())))
+	if err != nil {
+		return "", errors.Wrap(err, "new router request")
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "call ai routing router")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		data, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+		return "", errors.Errorf("router status %d: %s", resp.StatusCode, strings.TrimSpace(string(data)))
+	}
+	var out struct {
+		Model string `json:"model"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
+		return "", errors.Wrap(err, "decode router response")
+	}
+	if strings.TrimSpace(out.Model) == "" {
+		return "", errors.Wrap(httperrors.ErrInvalidStatus, "router response missing model")
+	}
+	return out.Model, nil
+}
+
+func aiRoutingRouterEndpoint(routerUrl, routePath string) (string, error) {
+	routerUrl = strings.TrimSpace(routerUrl)
+	if routerUrl == "" {
+		return "", errors.Wrap(httperrors.ErrInvalidStatus, "router_url is required when router is enabled")
+	}
+	u, err := url.Parse(routerUrl)
+	if err != nil {
+		return "", errors.Wrap(err, "parse router_url")
+	}
+	if u.Scheme == "" || u.Host == "" {
+		return "", errors.Wrap(httperrors.ErrInputParameter, "router_url must be absolute")
+	}
+	routePath = strings.TrimSpace(routePath)
+	if routePath == "" {
+		routePath = api.AiRoutingRouterDefaultRoutePath
+	}
+	if !strings.HasPrefix(routePath, "/") {
+		routePath = "/" + routePath
+	}
+	basePath := strings.TrimRight(u.Path, "/")
+	cleanRoutePath := strings.TrimRight(routePath, "/")
+	if basePath == cleanRoutePath {
+		return u.String(), nil
+	}
+	u.Path = basePath + routePath
+	return u.String(), nil
 }

pkg/aiproxy/models/ai_routings.go

@@ -26,6 +26,7 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/aiproxy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
 )
@@ -40,6 +41,13 @@ type SAiRouting struct {
 	ModelPattern string `width:"256" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"`
 	// AiProxyNodeId optionally binds the rule to one aiproxy instance (ai_proxy_node id).
 	AiProxyNodeId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
+
+	// Router fields optionally call an external model router before selecting an ai_routing_model.
+	RouterEnabled        bool   `default:"false" nullable:"false" list:"user" create:"optional" update:"user"`
+	RouterUrl            string `width:"512" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
+	RouterRoutePath      string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
+	RouterTimeoutSeconds int    `default:"0" nullable:"false" list:"user" create:"optional" update:"user"`
+	RouterFallbackPolicy string `width:"32" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"`
 }
 
 type SAiRoutingManager struct {
@@ -77,6 +85,9 @@ func (manager *SAiRoutingManager) ListItemFilter(
 	if v := strings.TrimSpace(query.AiProxyNodeId); v != "" {
 		q = q.Equals("ai_proxy_node_id", v)
 	}
+	if query.RouterEnabled != nil {
+		q = q.Equals("router_enabled", *query.RouterEnabled)
+	}
 	q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred, query.EnabledResourceBaseListInput)
 	if err != nil {
 		return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter")
@@ -96,8 +107,18 @@ func (manager *SAiRoutingManager) FetchCustomizeColumns(
 	sharableRows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
 	routingIds := make([]string, len(objs))
 	for i := range objs {
+		routing := objs[i].(*SAiRouting)
 		rows[i].SharableVirtualResourceDetails = sharableRows[i]
-		routingIds[i] = objs[i].(*SAiRouting).Id
+		rows[i].Priority = routing.Priority
+		rows[i].ModelPattern = routing.ModelPattern
+		rows[i].AiProxyNodeId = routing.AiProxyNodeId
+		rows[i].RouterEnabled = routing.RouterEnabled
+		rows[i].RouterUrl = routing.RouterUrl
+		rows[i].RouterRoutePath = routing.RouterRoutePath
+		rows[i].RouterTimeoutSeconds = routing.RouterTimeoutSeconds
+		rows[i].RouterFallbackPolicy = routing.RouterFallbackPolicy
+		rows[i].Enabled = routing.GetEnabled()
+		routingIds[i] = routing.Id
 	}
 	if fields == nil || fields.Contains("routing_models") {
 		for i, rid := range routingIds {
@@ -132,6 +153,80 @@ func (routing *SAiRouting) PerformEnable(ctx context.Context, userCred mcclient.
 	return nil, nil
 }
 
+func normalizeAiRoutingRouterFallbackPolicy(policy string) (string, error) {
+	policy = strings.ToLower(strings.TrimSpace(policy))
+	switch policy {
+	case "":
+		return api.AiRoutingRouterFallbackPriority, nil
+	case api.AiRoutingRouterFallbackPriority, api.AiRoutingRouterFallbackFailClosed:
+		return policy, nil
+	default:
+		return "", errors.Wrapf(httperrors.ErrInputParameter, "unsupported router_fallback_policy %q", policy)
+	}
+}
+
+func normalizeAiRoutingRouterRoutePath(routePath string) string {
+	routePath = strings.TrimSpace(routePath)
+	if routePath == "" {
+		return api.AiRoutingRouterDefaultRoutePath
+	}
+	if !strings.HasPrefix(routePath, "/") {
+		return "/" + routePath
+	}
+	return routePath
+}
+
+func normalizeAiRoutingRouterTimeoutSeconds(timeout int) int {
+	if timeout <= 0 {
+		return api.AiRoutingRouterDefaultTimeoutSeconds
+	}
+	return timeout
+}
+
+func normalizeAiRoutingRouterCreate(input *api.AiRoutingCreateInput) error {
+	input.RouterUrl = strings.TrimSpace(input.RouterUrl)
+	input.RouterRoutePath = normalizeAiRoutingRouterRoutePath(input.RouterRoutePath)
+	input.RouterTimeoutSeconds = normalizeAiRoutingRouterTimeoutSeconds(input.RouterTimeoutSeconds)
+	policy, err := normalizeAiRoutingRouterFallbackPolicy(input.RouterFallbackPolicy)
+	if err != nil {
+		return err
+	}
+	input.RouterFallbackPolicy = policy
+	if input.RouterEnabled && input.RouterUrl == "" {
+		return errors.Wrap(httperrors.ErrInputParameter, "router_url is required when router_enabled is true")
+	}
+	return nil
+}
+
+func normalizeAiRoutingRouterUpdate(routing *SAiRouting, query jsonutils.JSONObject, input *api.AiRoutingUpdateInput) error {
+	effectiveEnabled := routing.RouterEnabled
+	if input.RouterEnabled != nil {
+		effectiveEnabled = *input.RouterEnabled
+	}
+	effectiveUrl := strings.TrimSpace(routing.RouterUrl)
+	if query.Contains("router_url") {
+		input.RouterUrl = strings.TrimSpace(input.RouterUrl)
+		effectiveUrl = input.RouterUrl
+	}
+	if query.Contains("router_route_path") {
+		input.RouterRoutePath = normalizeAiRoutingRouterRoutePath(input.RouterRoutePath)
+	}
+	if query.Contains("router_timeout_seconds") {
+		input.RouterTimeoutSeconds = normalizeAiRoutingRouterTimeoutSeconds(input.RouterTimeoutSeconds)
+	}
+	if query.Contains("router_fallback_policy") {
+		policy, err := normalizeAiRoutingRouterFallbackPolicy(input.RouterFallbackPolicy)
+		if err != nil {
+			return err
+		}
+		input.RouterFallbackPolicy = policy
+	}
+	if effectiveEnabled && effectiveUrl == "" {
+		return errors.Wrap(httperrors.ErrInputParameter, "router_url is required when router_enabled is true")
+	}
+	return nil
+}
+
 func (routing *SAiRouting) PerformDisable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformDisableInput) (jsonutils.JSONObject, error) {
 	if err := db.EnabledPerformEnable(routing, ctx, userCred, false); err != nil {
 		return nil, errors.Wrap(err, "EnabledPerformEnable")
@@ -158,6 +253,9 @@ func (routing *SAiRouting) ValidateUpdateData(
 	} else if query.Contains("ai_proxy_node_id") {
 		input.AiProxyNodeId = ""
 	}
+	if err := normalizeAiRoutingRouterUpdate(routing, query, input); err != nil {
+		return input, err
+	}
 	return input, nil
 }
 
@@ -184,6 +282,9 @@ func (manager *SAiRoutingManager) ValidateCreateData(
 	if err != nil {
 		return input, err
 	}
+	if err := normalizeAiRoutingRouterCreate(&input); err != nil {
+		return input, err
+	}
 
 	if input.Enabled == nil && input.Disabled == nil {
 		input.SetEnabled()

pkg/aiproxy/models/catalog_seed.go

@@ -125,7 +125,7 @@ func catalogProviderId(providerKey string) string {
 }
 
 func catalogProviderExists(providerId string) (bool, error) {
-	cnt, err := AiProviderManager.Query().Equals("id", providerId).CountWithError()
+	cnt, err := AiProviderManager.RawQuery().Equals("id", providerId).CountWithError()
 	if err != nil {
 		return false, errors.Wrap(err, "count catalog ai_provider")
 	}
@@ -133,7 +133,7 @@ func catalogProviderExists(providerId string) (bool, error) {
 }
 
 func catalogModelExists(modelId string) (bool, error) {
-	cnt, err := AiModelManager.Query().Equals("id", modelId).CountWithError()
+	cnt, err := AiModelManager.RawQuery().Equals("id", modelId).CountWithError()
 	if err != nil {
 		return false, errors.Wrap(err, "count catalog ai_model")
 	}

pkg/aiproxy/models/chat_upstream.go

@@ -146,11 +146,12 @@ func resolveCatalogModelFromRouting(
 	vk *SAiVirtualKey,
 	routing *SAiRouting,
 	reqModel string,
+	body *jsonutils.JSONDict,
 ) (*resolvedCatalogModel, error) {
 	if routing == nil {
 		return nil, errors.Wrap(httperrors.ErrInvalidStatus, "nil ai_routing")
 	}
-	providerId, modelId, err := pickAiRoutingModel(ctx, userCred, routing, reqModel)
+	providerId, modelId, err := pickAiRoutingModel(ctx, userCred, routing, reqModel, body)
 	if err != nil {
 		return nil, err
 	}
@@ -209,7 +210,7 @@ func ResolveChatUpstream(ctx context.Context, userCred mcclient.TokenCredential,
 		return nil, errors.Wrap(httperrors.ErrNotFound, "no ai_routing matched for virtual key project on this aiproxy node")
 	}
 
-	resolved, err := resolveCatalogModelFromRouting(ctx, userCred, vk, routing, reqModel)
+	resolved, err := resolveCatalogModelFromRouting(ctx, userCred, vk, routing, reqModel, body)
 	if err != nil {
 		return nil, err
 	}

pkg/apis/aiproxy/ai_routing.go

@@ -18,12 +18,21 @@ import (
 	"yunion.io/x/onecloud/pkg/apis"
 )
 
+const (
+	AiRoutingRouterDefaultRoutePath      = "/v1/route"
+	AiRoutingRouterDefaultTimeoutSeconds = 3
+
+	AiRoutingRouterFallbackPriority   = "priority"
+	AiRoutingRouterFallbackFailClosed = "fail_closed"
+)
+
 type AiRoutingListInput struct {
 	apis.SharableVirtualResourceListInput
 	apis.EnabledResourceBaseListInput
 
 	ModelPattern  string `json:"model_pattern"`
 	AiProxyNodeId string `json:"ai_proxy_node_id"`
+	RouterEnabled *bool  `json:"router_enabled"`
 }
 
 // AiRoutingModelItem is one catalog model binding when creating ai_routing.
@@ -41,29 +50,44 @@ type AiRoutingCreateInput struct {
 	apis.SharableVirtualResourceCreateInput
 	apis.EnabledBaseResourceCreateInput
 
-	Priority      int                  `json:"priority"`
-	ModelPattern  string               `json:"model_pattern"`
-	AiProxyNodeId string               `json:"ai_proxy_node_id"`
-	Models        []AiRoutingModelItem `json:"models"`
+	Priority             int                  `json:"priority"`
+	ModelPattern         string               `json:"model_pattern"`
+	AiProxyNodeId        string               `json:"ai_proxy_node_id"`
+	RouterEnabled        bool                 `json:"router_enabled"`
+	RouterUrl            string               `json:"router_url"`
+	RouterRoutePath      string               `json:"router_route_path"`
+	RouterTimeoutSeconds int                  `json:"router_timeout_seconds"`
+	RouterFallbackPolicy string               `json:"router_fallback_policy"`
+	Models               []AiRoutingModelItem `json:"models"`
 }
 
 type AiRoutingUpdateInput struct {
 	apis.SharableVirtualResourceBaseUpdateInput
 
-	Priority      int    `json:"priority"`
-	ModelPattern  string `json:"model_pattern"`
-	AiProxyNodeId string `json:"ai_proxy_node_id"`
-	Enabled       *bool  `json:"enabled"`
+	Priority             int    `json:"priority"`
+	ModelPattern         string `json:"model_pattern"`
+	AiProxyNodeId        string `json:"ai_proxy_node_id"`
+	RouterEnabled        *bool  `json:"router_enabled"`
+	RouterUrl            string `json:"router_url"`
+	RouterRoutePath      string `json:"router_route_path"`
+	RouterTimeoutSeconds int    `json:"router_timeout_seconds"`
+	RouterFallbackPolicy string `json:"router_fallback_policy"`
+	Enabled              *bool  `json:"enabled"`
 }
 
 type AiRoutingDetails struct {
 	apis.SharableVirtualResourceDetails
 
-	Priority      int                     `json:"priority"`
-	ModelPattern  string                  `json:"model_pattern"`
-	AiProxyNodeId string                  `json:"ai_proxy_node_id"`
-	Enabled       bool                    `json:"enabled"`
-	RoutingModels []AiRoutingModelDetails `json:"routing_models,omitempty"`
+	Priority             int                     `json:"priority"`
+	ModelPattern         string                  `json:"model_pattern"`
+	AiProxyNodeId        string                  `json:"ai_proxy_node_id"`
+	RouterEnabled        bool                    `json:"router_enabled"`
+	RouterUrl            string                  `json:"router_url"`
+	RouterRoutePath      string                  `json:"router_route_path"`
+	RouterTimeoutSeconds int                     `json:"router_timeout_seconds"`
+	RouterFallbackPolicy string                  `json:"router_fallback_policy"`
+	Enabled              bool                    `json:"enabled"`
+	RoutingModels        []AiRoutingModelDetails `json:"routing_models,omitempty"`
 }
 
 // AiRoutingSetModelsInput replaces all ai_routing_models for an ai_routing.

pkg/apis/llm/image.go

@@ -16,6 +16,7 @@ const (
 	LLM_IMAGE_TYPE_COMFYUI      LLMImageType = "comfyui"
 	LLM_IMAGE_TYPE_OPENCLAW     LLMImageType = "openclaw"
 	LLM_IMAGE_TYPE_HERMES_AGENT LLMImageType = "hermes-agent"
+	LLM_IMAGE_TYPE_LLM_ROUTER   LLMImageType = "llm-router"
 	LLM_IMAGE_TYPE_DESKTOP      LLMImageType = "desktop"
 )
 
@@ -28,6 +29,7 @@ var (
 		string(LLM_IMAGE_TYPE_COMFYUI),
 		string(LLM_IMAGE_TYPE_OPENCLAW),
 		string(LLM_IMAGE_TYPE_HERMES_AGENT),
+		string(LLM_IMAGE_TYPE_LLM_ROUTER),
 		string(LLM_IMAGE_TYPE_DESKTOP),
 	)
 )

pkg/apis/llm/llm_container.go

@@ -16,6 +16,7 @@ const (
 	LLM_CONTAINER_COMFYUI      LLMContainerType = "comfyui"
 	LLM_CONTAINER_OPENCLAW     LLMContainerType = "openclaw"
 	LLM_CONTAINER_HERMES_AGENT LLMContainerType = "hermes-agent"
+	LLM_CONTAINER_LLM_ROUTER   LLMContainerType = "llm-router"
 	LLM_CONTAINER_DESKTOP      LLMContainerType = "desktop"
 )
 
@@ -28,6 +29,7 @@ var (
 		string(LLM_CONTAINER_COMFYUI),
 		string(LLM_CONTAINER_OPENCLAW),
 		string(LLM_CONTAINER_HERMES_AGENT),
+		string(LLM_CONTAINER_LLM_ROUTER),
 		string(LLM_CONTAINER_DESKTOP),
 	)
 	LLM_INSTANT_MODEL_TYPES = sets.NewString(

pkg/apis/llm/llm_router.go

@@ -0,0 +1,12 @@
+package llm
+
+const (
+	LLM_ROUTER                     = "llm-router"
+	LLM_ROUTER_EXEC_PATH           = "llm-router-runtime"
+	LLM_ROUTER_DEFAULT_RUNTIME     = "llmrouter-lib"
+	LLM_ROUTER_DEFAULT_MODEL_DIR   = "/models"
+	LLM_ROUTER_DEFAULT_ROUTE_PATH  = "/v1/route"
+	LLM_ROUTER_DEFAULT_HEALTH_PATH = "/health"
+	LLM_ROUTER_DEFAULT_PORT        = 8000
+	LLM_ROUTER_CACHE_DIR           = "/root/.cache"
+)

pkg/apis/llm/llm_spec.go

@@ -30,6 +30,7 @@ type LLMSpec struct {
 	ComfyUI     *LLMSpecComfyUI     `json:"comfyui,omitempty"`
 	OpenClaw    *LLMSpecOpenClaw    `json:"openclaw,omitempty"`
 	HermesAgent *LLMSpecHermesAgent `json:"hermes_agent,omitempty"`
+	LLMRouter   *LLMSpecLLMRouter   `json:"llm_router,omitempty"`
 }
 
 func (s *LLMSpec) String() string {
@@ -40,7 +41,7 @@ func (s *LLMSpec) IsZero() bool {
 	if s == nil {
 		return true
 	}
-	return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil
+	return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil && s.LLMRouter == nil
 }
 
 // LLMSpecOllama holds type-specific fields for ollama SKUs.
@@ -195,6 +196,43 @@ func (s *LLMSpecHermesAgent) IsZero() bool {
 	return s.LLMId == "" && s.LLMUrl == "" && s.Model == "" && s.ApiKey == "" && s.ContextLength == 0
 }
 
+type LLMRouterEnv struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+type LLMRouterArg struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+type LLMSpecLLMRouter struct {
+	Runtime              string                 `json:"runtime"`
+	RouterMethod         string                 `json:"router_method"`
+	ConfigPath           string                 `json:"config_path,omitempty"`
+	ModelDir             string                 `json:"model_dir,omitempty"`
+	RoutePath            string                 `json:"route_path,omitempty"`
+	HealthPath           string                 `json:"health_path,omitempty"`
+	CandidateMappingPath string                 `json:"candidate_mapping_path,omitempty"`
+	CustomizedEnvs       []*LLMRouterEnv        `json:"customized_envs,omitempty"`
+	CustomizedArgs       []*LLMRouterArg        `json:"customized_args,omitempty"`
+	Extra                map[string]interface{} `json:"extra,omitempty"`
+}
+
+func (s *LLMSpecLLMRouter) String() string {
+	return jsonutils.Marshal(s).String()
+}
+
+func (s *LLMSpecLLMRouter) IsZero() bool {
+	if s == nil {
+		return true
+	}
+	return s.Runtime == "" && s.RouterMethod == "" && s.ConfigPath == "" &&
+		s.ModelDir == "" && s.RoutePath == "" && s.HealthPath == "" &&
+		s.CandidateMappingPath == "" && len(s.CustomizedEnvs) == 0 &&
+		len(s.CustomizedArgs) == 0 && len(s.Extra) == 0
+}
+
 func init() {
 	gotypes.RegisterSerializable(reflect.TypeOf(new(LLMSpec)), func() gotypes.ISerializable {
 		return new(LLMSpec)

pkg/llm/drivers/llm_container/llm_router.go

@@ -0,0 +1,308 @@
+package llm_container
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	commonapi "yunion.io/x/onecloud/pkg/apis"
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/llm/models"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func init() {
+	models.RegisterLLMContainerDriver(newLLMRouter())
+}
+
+type llmRouter struct {
+	baseDriver
+}
+
+func newLLMRouter() models.ILLMContainerDriver {
+	return &llmRouter{baseDriver: newBaseDriver(api.LLM_CONTAINER_LLM_ROUTER)}
+}
+
+func (r *llmRouter) GetSpec(sku *models.SLLMSku) interface{} {
+	if sku == nil || sku.LLMType != string(api.LLM_CONTAINER_LLM_ROUTER) || sku.LLMSpec == nil || sku.LLMSpec.LLMRouter == nil {
+		return nil
+	}
+	return sku.LLMSpec.LLMRouter
+}
+
+func copyLLMRouterSpec(in *api.LLMSpecLLMRouter) *api.LLMSpecLLMRouter {
+	if in == nil {
+		return nil
+	}
+	out := *in
+	out.Runtime = strings.TrimSpace(out.Runtime)
+	out.RouterMethod = strings.TrimSpace(out.RouterMethod)
+	out.ConfigPath = strings.TrimSpace(out.ConfigPath)
+	out.ModelDir = strings.TrimSpace(out.ModelDir)
+	out.RoutePath = strings.TrimSpace(out.RoutePath)
+	out.HealthPath = strings.TrimSpace(out.HealthPath)
+	out.CandidateMappingPath = strings.TrimSpace(out.CandidateMappingPath)
+	if in.CustomizedEnvs != nil {
+		out.CustomizedEnvs = append([]*api.LLMRouterEnv(nil), in.CustomizedEnvs...)
+	}
+	if in.CustomizedArgs != nil {
+		out.CustomizedArgs = append([]*api.LLMRouterArg(nil), in.CustomizedArgs...)
+	}
+	if in.Extra != nil {
+		out.Extra = map[string]interface{}{}
+		for k, v := range in.Extra {
+			out.Extra[k] = v
+		}
+	}
+	return &out
+}
+
+func mergeLLMRouterSpecs(base, override *api.LLMSpecLLMRouter) *api.LLMSpecLLMRouter {
+	if base == nil && override == nil {
+		return nil
+	}
+	out := copyLLMRouterSpec(base)
+	if out == nil {
+		out = &api.LLMSpecLLMRouter{}
+	}
+	ov := copyLLMRouterSpec(override)
+	if ov == nil {
+		return out
+	}
+	if ov.Runtime != "" {
+		out.Runtime = ov.Runtime
+	}
+	if ov.RouterMethod != "" {
+		out.RouterMethod = ov.RouterMethod
+	}
+	if ov.ConfigPath != "" {
+		out.ConfigPath = ov.ConfigPath
+	}
+	if ov.ModelDir != "" {
+		out.ModelDir = ov.ModelDir
+	}
+	if ov.RoutePath != "" {
+		out.RoutePath = ov.RoutePath
+	}
+	if ov.HealthPath != "" {
+		out.HealthPath = ov.HealthPath
+	}
+	if ov.CandidateMappingPath != "" {
+		out.CandidateMappingPath = ov.CandidateMappingPath
+	}
+	if ov.CustomizedEnvs != nil {
+		out.CustomizedEnvs = ov.CustomizedEnvs
+	}
+	if ov.CustomizedArgs != nil {
+		out.CustomizedArgs = ov.CustomizedArgs
+	}
+	if ov.Extra != nil {
+		if out.Extra == nil {
+			out.Extra = map[string]interface{}{}
+		}
+		for k, v := range ov.Extra {
+			out.Extra[k] = v
+		}
+	}
+	return out
+}
+
+func (r *llmRouter) normalizeSpec(in *api.LLMSpecLLMRouter) (*api.LLMSpecLLMRouter, error) {
+	out := copyLLMRouterSpec(in)
+	if out == nil {
+		out = &api.LLMSpecLLMRouter{}
+	}
+	out.Runtime = api.LLM_ROUTER_DEFAULT_RUNTIME
+	if out.ModelDir == "" {
+		out.ModelDir = api.LLM_ROUTER_DEFAULT_MODEL_DIR
+	}
+	if out.RoutePath == "" {
+		out.RoutePath = api.LLM_ROUTER_DEFAULT_ROUTE_PATH
+	}
+	if out.HealthPath == "" {
+		out.HealthPath = api.LLM_ROUTER_DEFAULT_HEALTH_PATH
+	}
+	if out.RouterMethod == "" {
+		return out, errors.Wrap(httperrors.ErrMissingParameter, "llm_router.router_method is required")
+	}
+	return out, nil
+}
+
+func (r *llmRouter) GetEffectiveSpec(llm *models.SLLM, sku *models.SLLMSku) interface{} {
+	var skuSpec *api.LLMSpecLLMRouter
+	if spec := r.GetSpec(sku); spec != nil {
+		skuSpec = spec.(*api.LLMSpecLLMRouter)
+	}
+	var llmSpec *api.LLMSpecLLMRouter
+	if llm != nil && llm.LLMSpec != nil && llm.LLMSpec.LLMRouter != nil {
+		llmSpec = llm.LLMSpec.LLMRouter
+	}
+	out, err := r.normalizeSpec(mergeLLMRouterSpecs(skuSpec, llmSpec))
+	if err != nil {
+		return mergeLLMRouterSpecs(skuSpec, llmSpec)
+	}
+	return out
+}
+
+func (r *llmRouter) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) {
+	var err error
+	input, err = r.baseDriver.ValidateLLMSkuCreateData(ctx, userCred, input)
+	if err != nil {
+		return nil, err
+	}
+	spec, err := r.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec)
+	if err != nil {
+		return nil, err
+	}
+	input.LLMSpec = spec
+	return input, nil
+}
+
+func (r *llmRouter) ValidateLLMSkuUpdateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSkuUpdateInput) (*api.LLMSkuUpdateInput, error) {
+	var err error
+	input, err = r.baseDriver.ValidateLLMSkuUpdateData(ctx, userCred, sku, input)
+	if err != nil {
+		return nil, err
+	}
+	if input.LLMSpec != nil {
+		input.LLMSpec, err = r.ValidateLLMUpdateSpec(ctx, userCred, nil, input.LLMSpec)
+	}
+	return input, err
+}
+
+func (r *llmRouter) ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSpec) (*api.LLMSpec, error) {
+	var skuSpec *api.LLMSpecLLMRouter
+	if sku != nil && sku.LLMSpec != nil {
+		skuSpec = sku.LLMSpec.LLMRouter
+	}
+	var inputSpec *api.LLMSpecLLMRouter
+	if input != nil {
+		inputSpec = input.LLMRouter
+	}
+	spec, err := r.normalizeSpec(mergeLLMRouterSpecs(skuSpec, inputSpec))
+	if err != nil {
+		return input, err
+	}
+	return &api.LLMSpec{LLMRouter: spec}, nil
+}
+
+func (r *llmRouter) ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *api.LLMSpec) (*api.LLMSpec, error) {
+	if input == nil || input.LLMRouter == nil {
+		return input, nil
+	}
+	spec, err := r.normalizeSpec(input.LLMRouter)
+	if err != nil {
+		return input, err
+	}
+	return &api.LLMSpec{LLMRouter: spec}, nil
+}
+
+type llmRouterRuntimeArg struct {
+	Key   string
+	Value string
+}
+
+func (a llmRouterRuntimeArg) String() string {
+	key := strings.TrimSpace(a.Key)
+	if key == "" {
+		return ""
+	}
+	key = strings.TrimPrefix(key, "--")
+	if strings.TrimSpace(a.Value) == "" {
+		return "--" + key
+	}
+	return fmt.Sprintf("--%s %s", key, strings.TrimSpace(a.Value))
+}
+
+func llmRouterCustomizedArgsToRuntime(args []*api.LLMRouterArg) []llmRouterRuntimeArg {
+	out := make([]llmRouterRuntimeArg, 0, len(args))
+	for _, arg := range args {
+		if arg == nil || strings.TrimSpace(arg.Key) == "" {
+			continue
+		}
+		out = append(out, llmRouterRuntimeArg{Key: arg.Key, Value: arg.Value})
+	}
+	return out
+}
+
+func buildLLMRouterEntrypointScript(spec *api.LLMSpecLLMRouter) string {
+	parts := []string{
+		api.LLM_ROUTER_EXEC_PATH,
+		"--host 0.0.0.0",
+		fmt.Sprintf("--port %d", api.LLM_ROUTER_DEFAULT_PORT),
+	}
+	return strings.Join(parts, " ")
+}
+
+func (r *llmRouter) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput {
+	effSpec := &api.LLMSpecLLMRouter{RouterMethod: "routerdc"}
+	if eff := r.GetEffectiveSpec(llm, sku); eff != nil {
+		effSpec = eff.(*api.LLMSpecLLMRouter)
+	}
+	envs := []*commonapi.ContainerKeyValue{
+		models.NewEnv("LLM_ROUTER_RUNTIME", effSpec.Runtime),
+		models.NewEnv("LLM_ROUTER_METHOD", effSpec.RouterMethod),
+		models.NewEnv("LLM_ROUTER_MODEL_DIR", effSpec.ModelDir),
+		models.NewEnv("LLM_ROUTER_ROUTE_PATH", effSpec.RoutePath),
+		models.NewEnv("LLM_ROUTER_HEALTH_PATH", effSpec.HealthPath),
+	}
+	if effSpec.ConfigPath != "" {
+		envs = append(envs, models.NewEnv("LLM_ROUTER_CONFIG", effSpec.ConfigPath))
+	}
+	if effSpec.CandidateMappingPath != "" {
+		envs = append(envs, models.NewEnv("LLM_ROUTER_CANDIDATE_MAPPING", effSpec.CandidateMappingPath))
+	}
+	for _, env := range effSpec.CustomizedEnvs {
+		if env != nil && strings.TrimSpace(env.Key) != "" {
+			envs = append(envs, models.NewEnv(strings.TrimSpace(env.Key), env.Value))
+		}
+	}
+	spec := computeapi.ContainerSpec{
+		ContainerSpec: commonapi.ContainerSpec{
+			Image:             image.ToContainerImage(),
+			ImageCredentialId: image.CredentialId,
+			Command:           []string{"/bin/sh", "-c"},
+			Args:              []string{buildLLMRouterEntrypointScript(effSpec)},
+			EnableLxcfs:       true,
+			AlwaysRestart:     true,
+			Envs:              envs,
+		},
+	}
+	appendContainerIsolatedDevices(&spec, llm, sku, devices)
+	diskIndex := 0
+	spec.VolumeMounts = append(spec.VolumeMounts,
+		&commonapi.ContainerVolumeMount{
+			Disk: &commonapi.ContainerVolumeMountDisk{
+				SubDirectory: api.LLM_ROUTER,
+				Index:        &diskIndex,
+			},
+			Type:      commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
+			MountPath: effSpec.ModelDir,
+			ReadOnly:  false,
+		},
+		&commonapi.ContainerVolumeMount{
+			Disk: &commonapi.ContainerVolumeMountDisk{
+				SubDirectory: "cache",
+				Index:        &diskIndex,
+			},
+			Type:      commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK,
+			MountPath: api.LLM_ROUTER_CACHE_DIR,
+			ReadOnly:  false,
+		},
+	)
+	return &computeapi.PodContainerCreateInput{ContainerSpec: spec}
+}
+
+func (r *llmRouter) GetContainerSpecs(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) []*computeapi.PodContainerCreateInput {
+	return []*computeapi.PodContainerCreateInput{
+		r.GetContainerSpec(ctx, llm, image, sku, props, devices, diskId),
+	}
+}
+
+func (r *llmRouter) GetLLMAccessUrlInfo(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *models.LLMAccessInfoInput) (*api.LLMAccessUrlInfo, error) {
+	return models.GetLLMAccessUrlInfo(ctx, userCred, llm, input, "http", api.LLM_ROUTER_DEFAULT_PORT)
+}

pkg/llm/models/llm.go

@@ -986,6 +986,10 @@ func (llm *SLLM) GetLLMAccessUrlInfo(ctx context.Context, userCred mcclient.Toke
 	return llm.GetLLMContainerDriver().GetLLMAccessUrlInfo(ctx, userCred, llm, input)
 }
 
+func (llm *SLLM) GetDetailsUrl(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*api.LLMAccessUrlInfo, error) {
+	return llm.GetLLMAccessUrlInfo(ctx, userCred, query)
+}
+
 func GetLLMAccessUrlInfo(ctx context.Context, userCred mcclient.TokenCredential, llm *SLLM, input *LLMAccessInfoInput, protocol string, defaultPort int) (*api.LLMAccessUrlInfo, error) {
 	port := defaultPort
 	accessUrl := input.ServerIp

pkg/mcclient/options/aiproxy/resources.go

@@ -283,6 +283,7 @@ type AiRoutingListOptions struct {
 
 	ModelPattern  string `json:"model_pattern"`
 	AiProxyNodeId string `json:"ai_proxy_node_id"`
+	RouterEnabled *bool  `help:"filter by router enabled flag" json:"router_enabled"`
 	Enabled       *bool  `help:"filter by enabled flag" json:"enabled"`
 }
 
@@ -297,12 +298,17 @@ type AiRoutingShowOptions struct {
 type AiRoutingCreateOptions struct {
 	apis.SharableVirtualResourceCreateInput
 
-	Priority      int    `json:"priority,omitzero"`
-	ModelPattern  string `json:"model_pattern,omitempty"`
-	AiProxyNodeId string `json:"ai_proxy_node_id,omitempty"`
-	Models        string `help:"routing models JSON array: [{ai_provider_id,ai_model_id,priority|weight,model_pattern?}]" json:"-"`
-	Enabled       *bool  `help:"turn on enabled flag" json:"enabled,omitempty"`
-	Disabled      *bool  `help:"turn off enabled flag" json:"disabled,omitempty"`
+	Priority             int    `json:"priority,omitzero"`
+	ModelPattern         string `json:"model_pattern,omitempty"`
+	AiProxyNodeId        string `json:"ai_proxy_node_id,omitempty"`
+	RouterEnabled        *bool  `help:"enable external model router" json:"router_enabled,omitempty"`
+	RouterUrl            string `help:"external router base URL, e.g. http://192.168.6.60:20000" json:"router_url,omitempty"`
+	RouterRoutePath      string `help:"external router route path, default /v1/route" json:"router_route_path,omitempty"`
+	RouterTimeoutSeconds int    `help:"external router timeout seconds, default 3" json:"router_timeout_seconds,omitzero"`
+	RouterFallbackPolicy string `help:"router failure policy: priority or fail_closed" json:"router_fallback_policy,omitempty"`
+	Models               string `help:"routing models JSON array: [{ai_provider_id,ai_model_id,priority|weight,model_pattern?}]" json:"-"`
+	Enabled              *bool  `help:"turn on enabled flag" json:"enabled,omitempty"`
+	Disabled             *bool  `help:"turn off enabled flag" json:"disabled,omitempty"`
 }
 
 func (o *AiRoutingCreateOptions) Params() (jsonutils.JSONObject, error) {
@@ -316,11 +322,16 @@ func (o *AiRoutingCreateOptions) Params() (jsonutils.JSONObject, error) {
 type AiRoutingUpdateOptions struct {
 	apis.SharableVirtualResourceBaseUpdateInput
 
-	ID            string `help:"ID or name" json:"-"`
-	Priority      int    `json:"priority,omitzero"`
-	ModelPattern  string `json:"model_pattern,omitempty"`
-	AiProxyNodeId string `json:"ai_proxy_node_id,omitempty"`
-	Enabled       *bool  `json:"enabled,omitempty"`
+	ID                   string `help:"ID or name" json:"-"`
+	Priority             int    `json:"priority,omitzero"`
+	ModelPattern         string `json:"model_pattern,omitempty"`
+	AiProxyNodeId        string `json:"ai_proxy_node_id,omitempty"`
+	RouterEnabled        *bool  `help:"enable or disable external model router" json:"router_enabled,omitempty"`
+	RouterUrl            string `help:"external router base URL" json:"router_url,omitempty"`
+	RouterRoutePath      string `help:"external router route path" json:"router_route_path,omitempty"`
+	RouterTimeoutSeconds int    `help:"external router timeout seconds" json:"router_timeout_seconds,omitzero"`
+	RouterFallbackPolicy string `help:"router failure policy: priority or fail_closed" json:"router_fallback_policy,omitempty"`
+	Enabled              *bool  `json:"enabled,omitempty"`
 }
 
 func (o *AiRoutingUpdateOptions) GetId() string {

pkg/mcclient/options/llm/image.go

@@ -18,7 +18,7 @@ func (o *LLMImageShowOptions) Params() (jsonutils.JSONObject, error) {
 type LLMImageListOptions struct {
 	options.BaseListOptions
 
-	LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|desktop" help:"filter by llm type"`
+	LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop" help:"filter by llm type"`
 }
 
 func (o *LLMImageListOptions) Params() (jsonutils.JSONObject, error) {
@@ -30,7 +30,7 @@ type LLMImageCreateOptions struct {
 	IMAGE_NAME   string `json:"image_name"`
 	IMAGE_LABEL  string `json:"image_label"`
 	CredentialId string `json:"credential_id"`
-	LLM_TYPE     string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, desktop or dify"`
+	LLM_TYPE     string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop or dify"`
 }
 
 func (o *LLMImageCreateOptions) Params() (jsonutils.JSONObject, error) {
@@ -44,7 +44,7 @@ type LLMImageUpdateOptions struct {
 	ImageName    string `json:"image_name"`
 	ImageLabel   string `json:"image_label"`
 	CredentialId string `json:"credential_id"`
-	LlmType      string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, desktop or dify"`
+	LlmType      string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|llm-router|desktop" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop or dify"`
 	AppName      string `json:"app_name" help:"desktop application identifier (LinuxServer image id), e.g. firefox, chromium, steam, webtop-ubuntu-xfce"`
 }
 

pkg/mcclient/options/llm/llm_deployment.go

@@ -47,7 +47,7 @@ type LLMDeploymentCreateOptions struct {
 	// Mode B/C: SKU spec (all sku-* flags collapse into sku_spec)
 	SkuName          string   `help:"SKU name (default: <deploy-name>-sku)" json:"-"`
 	SkuLLMImageId    string   `help:"container image id (for SkuSpec)" json:"-"`
-	SkuLLMType       string   `help:"container llm type" choices:"ollama|vllm|comfyui|sglang|dify" json:"-"`
+	SkuLLMType       string   `help:"container llm type" choices:"ollama|vllm|comfyui|sglang|dify|llm-router" json:"-"`
 	SkuCpu           int      `help:"SKU CPU cores" json:"-"`
 	SkuMemory        int      `help:"SKU memory MB" json:"-"`
 	SkuDiskSize      int      `help:"SKU disk size MB" json:"-"`

pkg/mcclient/options/llm/llm_sku.go

@@ -13,7 +13,7 @@ import (
 type LLMSkuListOptions struct {
 	options.BaseListOptions
 
-	LLMType    string `json:"llm_type" choices:"ollama|vllm|sglang|dify|comfyui|openclaw|hermes-agent|desktop"`
+	LLMType    string `json:"llm_type" choices:"ollama|vllm|sglang|dify|comfyui|openclaw|hermes-agent|llm-router|desktop"`
 	Source     string `json:"source" help:"filter by source (huggingface, model_scope, local_path)"`
 	Categories string `json:"categories" help:"filter by category (llm, embedding, image, ...)"`
 }
@@ -36,7 +36,7 @@ type LLMSkuCreateOptions struct {
 	MountedModels []string `help:"mounted models, <model_id> e.g. qwen2:0.5b-dup" json:"mounted_models"`
 
 	LLM_IMAGE_ID string `json:"llm_image_id"`
-	LLM_TYPE     string `json:"llm_type" choices:"ollama|vllm|sglang|comfyui|hermes-agent|desktop"`
+	LLM_TYPE     string `json:"llm_type" choices:"ollama|vllm|sglang|comfyui|hermes-agent|llm-router|desktop"`
 
 	// Model source
 	Source              string `help:"model source: huggingface, model_scope, local_path" json:"source"`
@@ -67,6 +67,14 @@ type LLMSkuCreateOptions struct {
 	HermesModel         string `token:"hermes-model" help:"model name for Hermes" json:"-"`
 	HermesApiKey        string `token:"hermes-api-key" help:"API key for Hermes custom provider; defaults to EMPTY when omitted" json:"-"`
 	HermesContextLength int    `token:"hermes-context-length" help:"Hermes model.context_length" json:"-"`
+
+	RouterMethod               string   `token:"router-method" help:"LLM router method, e.g. routerdc" json:"-"`
+	RouterConfigPath           string   `token:"router-config-path" help:"LLM router config path inside container" json:"-"`
+	RouterModelDir             string   `token:"router-model-dir" help:"LLM router model directory inside container" json:"-"`
+	RouterRoutePath            string   `token:"router-route-path" help:"LLM router route API path, e.g. /v1/route" json:"-"`
+	RouterHealthPath           string   `token:"router-health-path" help:"LLM router health API path, e.g. /health" json:"-"`
+	RouterCandidateMappingPath string   `token:"router-candidate-mapping-path" help:"LLM router candidate mapping path inside container" json:"-"`
+	RouterEnv                  []string `token:"router-env" help:"LLM router env in format key=value; repeatable" json:"-"`
 }
 
 func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) {
@@ -126,10 +134,49 @@ func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) {
 			spec := &api.LLMSpec{HermesAgent: hermesSpec}
 			dict.Set("llm_spec", jsonutils.Marshal(spec))
 		}
+	case string(api.LLM_CONTAINER_LLM_ROUTER):
+		routerSpec, err := newLLMRouterSpecFromArgs(o)
+		if err != nil {
+			return nil, err
+		}
+		dict.Set("llm_spec", jsonutils.Marshal(&api.LLMSpec{LLMRouter: routerSpec}))
 	}
 	return dict, nil
 }
 
+func newLLMRouterSpecFromArgs(o *LLMSkuCreateOptions) (*api.LLMSpecLLMRouter, error) {
+	method := strings.TrimSpace(o.RouterMethod)
+	if method == "" {
+		return nil, errors.Error("--router-method is required when llm_type=llm-router")
+	}
+	envs, err := newLLMRouterEnvsFromArgs(o.RouterEnv)
+	if err != nil {
+		return nil, err
+	}
+	return &api.LLMSpecLLMRouter{
+		RouterMethod:         method,
+		ConfigPath:           strings.TrimSpace(o.RouterConfigPath),
+		ModelDir:             strings.TrimSpace(o.RouterModelDir),
+		RoutePath:            strings.TrimSpace(o.RouterRoutePath),
+		HealthPath:           strings.TrimSpace(o.RouterHealthPath),
+		CandidateMappingPath: strings.TrimSpace(o.RouterCandidateMappingPath),
+		CustomizedEnvs:       envs,
+	}, nil
+}
+
+func newLLMRouterEnvsFromArgs(args []string) ([]*api.LLMRouterEnv, error) {
+	envs := make([]*api.LLMRouterEnv, 0, len(args))
+	for _, arg := range args {
+		key, val, ok := strings.Cut(arg, "=")
+		key = strings.TrimSpace(key)
+		if !ok || key == "" {
+			return nil, errors.Errorf("invalid --router-env %q, expected key=value", arg)
+		}
+		envs = append(envs, &api.LLMRouterEnv{Key: key, Value: strings.TrimSpace(val)})
+	}
+	return envs, nil
+}
+
 func (o *LLMSkuCreateOptions) buildModelSpec() (*api.InstantModelImportInput, error) {
 	if o.ModelTag == "" {
 		return nil, errors.Error("--model-tag is required for model spec")