diff --git a/cmd/climc/shell/llm/llm_router_agent.go b/cmd/climc/shell/llm/llm_router_agent.go new file mode 100644 index 0000000000..2f6ae3e8f0 --- /dev/null +++ b/cmd/climc/shell/llm/llm_router_agent.go @@ -0,0 +1,66 @@ +package llm + +import ( + "fmt" + "io" + "net/http" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/cmd/climc/shell" + "yunion.io/x/onecloud/pkg/mcclient" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/llm" + options "yunion.io/x/onecloud/pkg/mcclient/options/llm" +) + +func init() { + cmd := shell.NewResourceCmd(&modules.LLMRouterAgent) + cmd.List(new(options.LLMRouterAgentListOptions)) + cmd.Show(new(options.LLMRouterAgentShowOptions)) + cmd.Create(new(options.LLMRouterAgentCreateOptions)) + cmd.Update(new(options.LLMRouterAgentUpdateOptions)) + cmd.Delete(new(options.LLMRouterAgentDeleteOptions)) + shell.R(&options.LLMRouterAgentRouteOptions{}, "llm-router-agent-route", "Route llm_router_agent", routeLLMRouterAgent) +} + +func routeLLMRouterAgent(s *mcclient.ClientSession, args *options.LLMRouterAgentRouteOptions) error { + id := args.ID + if id != "default" { + var err error + id, err = modules.LLMRouterAgent.GetId(s, args.ID, nil) + if err != nil { + return err + } + } + bodyJSON, err := args.Params() + if err != nil { + return err + } + headers := http.Header{} + headers.Set("Content-Type", "application/json") + resp, err := s.RawVersionRequest( + modules.LLMRouterAgent.ServiceType(), + modules.LLMRouterAgent.EndpointType(), + "POST", + fmt.Sprintf("/llm_router_agents/%s/route", id), + headers, + strings.NewReader(bodyJSON.String()), + ) + if err != nil { + return err + } + defer resp.Body.Close() + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("Error: %s %s", resp.Status, string(respBody)) + } + obj, err := jsonutils.Parse(respBody) + if err != nil { + return err + } + shell.PrintObject(obj) + return nil +} diff --git a/pkg/aiproxy/models/ai_routing_models.go b/pkg/aiproxy/models/ai_routing_models.go index 650745e268..66c5450111 100644 --- a/pkg/aiproxy/models/ai_routing_models.go +++ b/pkg/aiproxy/models/ai_routing_models.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" + "yunion.io/x/pkg/util/httputils" "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/aiproxy" @@ -629,7 +630,7 @@ func callAiRoutingRouter(ctx context.Context, routing *SAiRouting, routerModel s return nil, errors.Wrap(err, "new router request") } req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := httputils.GetTimeoutClient(time.Duration(timeout) * time.Second).Do(req) if err != nil { return nil, errors.Wrap(err, "call ai routing router") } diff --git a/pkg/apis/llm/image.go b/pkg/apis/llm/image.go index 1221935b36..48b349d588 100644 --- a/pkg/apis/llm/image.go +++ b/pkg/apis/llm/image.go @@ -16,7 +16,6 @@ const ( LLM_IMAGE_TYPE_COMFYUI LLMImageType = "comfyui" LLM_IMAGE_TYPE_OPENCLAW LLMImageType = "openclaw" LLM_IMAGE_TYPE_HERMES_AGENT LLMImageType = "hermes-agent" - LLM_IMAGE_TYPE_LLM_ROUTER LLMImageType = "llm-router" LLM_IMAGE_TYPE_DESKTOP LLMImageType = "desktop" LLM_IMAGE_TYPE_BENCHMARK LLMImageType = "benchmark" ) @@ -30,7 +29,6 @@ var ( string(LLM_IMAGE_TYPE_COMFYUI), string(LLM_IMAGE_TYPE_OPENCLAW), string(LLM_IMAGE_TYPE_HERMES_AGENT), - string(LLM_IMAGE_TYPE_LLM_ROUTER), string(LLM_IMAGE_TYPE_DESKTOP), string(LLM_IMAGE_TYPE_BENCHMARK), ) diff --git a/pkg/apis/llm/llm_const.go b/pkg/apis/llm/llm_const.go index df93dd41a2..16647edef3 100644 --- a/pkg/apis/llm/llm_const.go +++ b/pkg/apis/llm/llm_const.go @@ -4,6 +4,8 @@ const ( STATUS_READY = "ready" ) +const LLM_DEFAULT_CONTEXT_TOKENS = 8192 + const ( /* 未知 */ LLM_STATUS_UNKNOWN = "unknown" diff --git a/pkg/apis/llm/llm_container.go b/pkg/apis/llm/llm_container.go index b8937b2f43..425f234bbf 100644 --- a/pkg/apis/llm/llm_container.go +++ b/pkg/apis/llm/llm_container.go @@ -16,7 +16,6 @@ const ( LLM_CONTAINER_COMFYUI LLMContainerType = "comfyui" LLM_CONTAINER_OPENCLAW LLMContainerType = "openclaw" LLM_CONTAINER_HERMES_AGENT LLMContainerType = "hermes-agent" - LLM_CONTAINER_LLM_ROUTER LLMContainerType = "llm-router" LLM_CONTAINER_DESKTOP LLMContainerType = "desktop" ) @@ -29,7 +28,6 @@ var ( string(LLM_CONTAINER_COMFYUI), string(LLM_CONTAINER_OPENCLAW), string(LLM_CONTAINER_HERMES_AGENT), - string(LLM_CONTAINER_LLM_ROUTER), string(LLM_CONTAINER_DESKTOP), ) LLM_INSTANT_MODEL_TYPES = sets.NewString( diff --git a/pkg/apis/llm/llm_router.go b/pkg/apis/llm/llm_router.go deleted file mode 100644 index 6920ae10a1..0000000000 --- a/pkg/apis/llm/llm_router.go +++ /dev/null @@ -1,12 +0,0 @@ -package llm - -const ( - LLM_ROUTER = "llm-router" - LLM_ROUTER_EXEC_PATH = "llm-router-runtime" - LLM_ROUTER_DEFAULT_RUNTIME = "llmrouter-lib" - LLM_ROUTER_DEFAULT_MODEL_DIR = "/models" - LLM_ROUTER_DEFAULT_ROUTE_PATH = "/v1/route" - LLM_ROUTER_DEFAULT_HEALTH_PATH = "/health" - LLM_ROUTER_DEFAULT_PORT = 8000 - LLM_ROUTER_CACHE_DIR = "/root/.cache" -) diff --git a/pkg/apis/llm/llm_router_agent.go b/pkg/apis/llm/llm_router_agent.go new file mode 100644 index 0000000000..b42da74959 --- /dev/null +++ b/pkg/apis/llm/llm_router_agent.go @@ -0,0 +1,98 @@ +package llm + +import "yunion.io/x/onecloud/pkg/apis" + +const ( + LLM_ROUTER_ROUTE_SIMPLE = "simple" + LLM_ROUTER_ROUTE_COMPLEX = "complex" + + LLM_ROUTER_DEFAULT_ROUTE = LLM_ROUTER_ROUTE_COMPLEX + LLM_ROUTER_DEFAULT_MAX_PROMPT_CHARS = LLM_DEFAULT_CONTEXT_TOKENS * 3 / 4 + LLM_ROUTER_DEFAULT_MAX_DECISION_TOKENS = 64 + LLM_ROUTER_DEFAULT_SIMPLE_DEFINITION = "适合翻译、摘要、短问答和格式转换。" + LLM_ROUTER_DEFAULT_COMPLEX_DEFINITION = "适合代码、数学、长上下文、复杂推理和日志分析。" +) + +type LLMRouterAgentListInput struct { + apis.SharableVirtualResourceListInput + + LLMDriver string `json:"llm_driver"` +} + +type LLMRouterAgentCreateInput struct { + apis.SharableVirtualResourceCreateInput + + LLMId string `json:"llm_id" help:"LLM instance ID; if set, llm_url is resolved from it"` + LLMUrl string `json:"llm_url" help:"decision model OpenAI-compatible base URL"` + LLMDriver string `json:"llm_driver" help:"decision model driver, default openai"` + Model string `json:"model" help:"decision model name"` + ApiKey string `json:"api_key" help:"decision model API key"` + DefaultRoute string `json:"default_route" help:"fallback route, default complex"` + MaxPromptChars int `json:"max_prompt_chars" help:"request char limit; 0 auto-detects /v1/models context, longer requests use complex"` + MaxDecisionTokens int `json:"max_decision_tokens" help:"decision answer token hint"` + CandidateMapping map[string][]string `json:"candidate_mapping" help:"route to candidate model names"` + SimpleDefinition string `json:"simple_definition" help:"definition of simple requests"` + ComplexDefinition string `json:"complex_definition" help:"definition of complex requests"` + SimpleExamples []string `json:"simple_examples" help:"examples routed to simple"` + ComplexExamples []string `json:"complex_examples" help:"examples routed to complex"` +} + +type LLMRouterAgentUpdateInput struct { + apis.SharableVirtualResourceBaseUpdateInput + + LLMId *string `json:"llm_id,omitempty" help:"LLM instance ID; if set, llm_url is resolved from it"` + LLMUrl *string `json:"llm_url,omitempty" help:"decision model OpenAI-compatible base URL"` + LLMDriver *string `json:"llm_driver,omitempty" help:"decision model driver"` + Model *string `json:"model,omitempty" help:"decision model name"` + ApiKey *string `json:"api_key,omitempty" help:"decision model API key"` + DefaultRoute *string `json:"default_route,omitempty" help:"fallback route"` + MaxPromptChars *int `json:"max_prompt_chars,omitempty" help:"request char limit; 0 recomputes from /v1/models context"` + MaxDecisionTokens *int `json:"max_decision_tokens,omitempty" help:"decision answer token hint"` + CandidateMapping *map[string][]string `json:"candidate_mapping,omitempty" help:"route to candidate model names"` + SimpleDefinition *string `json:"simple_definition,omitempty" help:"definition of simple requests"` + ComplexDefinition *string `json:"complex_definition,omitempty" help:"definition of complex requests"` + SimpleExamples *[]string `json:"simple_examples,omitempty" help:"replace examples routed to simple"` + ComplexExamples *[]string `json:"complex_examples,omitempty" help:"replace examples routed to complex"` +} + +type LLMRouterAgentDetails struct { + apis.SharableVirtualResourceDetails + + LLMId string `json:"llm_id"` + LLMName string `json:"llm_name"` +} + +type LLMRouterPromptConfig struct { + SimpleDefinition string `json:"simple_definition"` + ComplexDefinition string `json:"complex_definition"` + SimpleExamples []string `json:"simple_examples"` + ComplexExamples []string `json:"complex_examples"` +} + +type LLMRouterExpandExamplesInput struct { + SimpleDefinition *string `json:"simple_definition,omitempty"` + ComplexDefinition *string `json:"complex_definition,omitempty"` + SimpleExamples *[]string `json:"simple_examples,omitempty"` + ComplexExamples *[]string `json:"complex_examples,omitempty"` +} + +type LLMRouterMessage struct { + Role string `json:"role"` + Content interface{} `json:"content"` +} + +type LLMRouterRouteRequest struct { + Model string `json:"model"` + Messages []LLMRouterMessage `json:"messages"` + Candidates []string `json:"candidates"` +} + +type LLMRouterRouteResponse struct { + Model string `json:"model"` +} + +type LLMRouterDecision struct { + Route string `json:"route"` + Confidence float64 `json:"confidence"` + Reason string `json:"reason,omitempty"` +} diff --git a/pkg/apis/llm/llm_spec.go b/pkg/apis/llm/llm_spec.go index 0d2224e9a9..f22d88038d 100644 --- a/pkg/apis/llm/llm_spec.go +++ b/pkg/apis/llm/llm_spec.go @@ -30,7 +30,6 @@ type LLMSpec struct { ComfyUI *LLMSpecComfyUI `json:"comfyui,omitempty"` OpenClaw *LLMSpecOpenClaw `json:"openclaw,omitempty"` HermesAgent *LLMSpecHermesAgent `json:"hermes_agent,omitempty"` - LLMRouter *LLMSpecLLMRouter `json:"llm_router,omitempty"` } func (s *LLMSpec) String() string { @@ -41,7 +40,7 @@ func (s *LLMSpec) IsZero() bool { if s == nil { return true } - return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil && s.LLMRouter == nil + return s.Ollama == nil && s.Vllm == nil && s.SGLang == nil && s.Dify == nil && s.ComfyUI == nil && s.OpenClaw == nil && s.HermesAgent == nil } // LLMSpecOllama holds type-specific fields for ollama SKUs. @@ -196,43 +195,6 @@ func (s *LLMSpecHermesAgent) IsZero() bool { return s.LLMId == "" && s.LLMUrl == "" && s.Model == "" && s.ApiKey == "" && s.ContextLength == 0 } -type LLMRouterEnv struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type LLMRouterArg struct { - Key string `json:"key"` - Value string `json:"value"` -} - -type LLMSpecLLMRouter struct { - Runtime string `json:"runtime"` - RouterMethod string `json:"router_method"` - ConfigPath string `json:"config_path,omitempty"` - ModelDir string `json:"model_dir,omitempty"` - RoutePath string `json:"route_path,omitempty"` - HealthPath string `json:"health_path,omitempty"` - CandidateMappingPath string `json:"candidate_mapping_path,omitempty"` - CustomizedEnvs []*LLMRouterEnv `json:"customized_envs,omitempty"` - CustomizedArgs []*LLMRouterArg `json:"customized_args,omitempty"` - Extra map[string]interface{} `json:"extra,omitempty"` -} - -func (s *LLMSpecLLMRouter) String() string { - return jsonutils.Marshal(s).String() -} - -func (s *LLMSpecLLMRouter) IsZero() bool { - if s == nil { - return true - } - return s.Runtime == "" && s.RouterMethod == "" && s.ConfigPath == "" && - s.ModelDir == "" && s.RoutePath == "" && s.HealthPath == "" && - s.CandidateMappingPath == "" && len(s.CustomizedEnvs) == 0 && - len(s.CustomizedArgs) == 0 && len(s.Extra) == 0 -} - func init() { gotypes.RegisterSerializable(reflect.TypeOf(new(LLMSpec)), func() gotypes.ISerializable { return new(LLMSpec) diff --git a/pkg/llm/drivers/llm_container/llm_router.go b/pkg/llm/drivers/llm_container/llm_router.go deleted file mode 100644 index 07f8ce1ee5..0000000000 --- a/pkg/llm/drivers/llm_container/llm_router.go +++ /dev/null @@ -1,308 +0,0 @@ -package llm_container - -import ( - "context" - "fmt" - "strings" - - "yunion.io/x/pkg/errors" - - commonapi "yunion.io/x/onecloud/pkg/apis" - computeapi "yunion.io/x/onecloud/pkg/apis/compute" - api "yunion.io/x/onecloud/pkg/apis/llm" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/llm/models" - "yunion.io/x/onecloud/pkg/mcclient" -) - -func init() { - models.RegisterLLMContainerDriver(newLLMRouter()) -} - -type llmRouter struct { - baseDriver -} - -func newLLMRouter() models.ILLMContainerDriver { - return &llmRouter{baseDriver: newBaseDriver(api.LLM_CONTAINER_LLM_ROUTER)} -} - -func (r *llmRouter) GetSpec(sku *models.SLLMSku) interface{} { - if sku == nil || sku.LLMType != string(api.LLM_CONTAINER_LLM_ROUTER) || sku.LLMSpec == nil || sku.LLMSpec.LLMRouter == nil { - return nil - } - return sku.LLMSpec.LLMRouter -} - -func copyLLMRouterSpec(in *api.LLMSpecLLMRouter) *api.LLMSpecLLMRouter { - if in == nil { - return nil - } - out := *in - out.Runtime = strings.TrimSpace(out.Runtime) - out.RouterMethod = strings.TrimSpace(out.RouterMethod) - out.ConfigPath = strings.TrimSpace(out.ConfigPath) - out.ModelDir = strings.TrimSpace(out.ModelDir) - out.RoutePath = strings.TrimSpace(out.RoutePath) - out.HealthPath = strings.TrimSpace(out.HealthPath) - out.CandidateMappingPath = strings.TrimSpace(out.CandidateMappingPath) - if in.CustomizedEnvs != nil { - out.CustomizedEnvs = append([]*api.LLMRouterEnv(nil), in.CustomizedEnvs...) - } - if in.CustomizedArgs != nil { - out.CustomizedArgs = append([]*api.LLMRouterArg(nil), in.CustomizedArgs...) - } - if in.Extra != nil { - out.Extra = map[string]interface{}{} - for k, v := range in.Extra { - out.Extra[k] = v - } - } - return &out -} - -func mergeLLMRouterSpecs(base, override *api.LLMSpecLLMRouter) *api.LLMSpecLLMRouter { - if base == nil && override == nil { - return nil - } - out := copyLLMRouterSpec(base) - if out == nil { - out = &api.LLMSpecLLMRouter{} - } - ov := copyLLMRouterSpec(override) - if ov == nil { - return out - } - if ov.Runtime != "" { - out.Runtime = ov.Runtime - } - if ov.RouterMethod != "" { - out.RouterMethod = ov.RouterMethod - } - if ov.ConfigPath != "" { - out.ConfigPath = ov.ConfigPath - } - if ov.ModelDir != "" { - out.ModelDir = ov.ModelDir - } - if ov.RoutePath != "" { - out.RoutePath = ov.RoutePath - } - if ov.HealthPath != "" { - out.HealthPath = ov.HealthPath - } - if ov.CandidateMappingPath != "" { - out.CandidateMappingPath = ov.CandidateMappingPath - } - if ov.CustomizedEnvs != nil { - out.CustomizedEnvs = ov.CustomizedEnvs - } - if ov.CustomizedArgs != nil { - out.CustomizedArgs = ov.CustomizedArgs - } - if ov.Extra != nil { - if out.Extra == nil { - out.Extra = map[string]interface{}{} - } - for k, v := range ov.Extra { - out.Extra[k] = v - } - } - return out -} - -func (r *llmRouter) normalizeSpec(in *api.LLMSpecLLMRouter) (*api.LLMSpecLLMRouter, error) { - out := copyLLMRouterSpec(in) - if out == nil { - out = &api.LLMSpecLLMRouter{} - } - out.Runtime = api.LLM_ROUTER_DEFAULT_RUNTIME - if out.ModelDir == "" { - out.ModelDir = api.LLM_ROUTER_DEFAULT_MODEL_DIR - } - if out.RoutePath == "" { - out.RoutePath = api.LLM_ROUTER_DEFAULT_ROUTE_PATH - } - if out.HealthPath == "" { - out.HealthPath = api.LLM_ROUTER_DEFAULT_HEALTH_PATH - } - if out.RouterMethod == "" { - return out, errors.Wrap(httperrors.ErrMissingParameter, "llm_router.router_method is required") - } - return out, nil -} - -func (r *llmRouter) GetEffectiveSpec(llm *models.SLLM, sku *models.SLLMSku) interface{} { - var skuSpec *api.LLMSpecLLMRouter - if spec := r.GetSpec(sku); spec != nil { - skuSpec = spec.(*api.LLMSpecLLMRouter) - } - var llmSpec *api.LLMSpecLLMRouter - if llm != nil && llm.LLMSpec != nil && llm.LLMSpec.LLMRouter != nil { - llmSpec = llm.LLMSpec.LLMRouter - } - out, err := r.normalizeSpec(mergeLLMRouterSpecs(skuSpec, llmSpec)) - if err != nil { - return mergeLLMRouterSpecs(skuSpec, llmSpec) - } - return out -} - -func (r *llmRouter) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.TokenCredential, input *api.LLMSkuCreateInput) (*api.LLMSkuCreateInput, error) { - var err error - input, err = r.baseDriver.ValidateLLMSkuCreateData(ctx, userCred, input) - if err != nil { - return nil, err - } - spec, err := r.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec) - if err != nil { - return nil, err - } - input.LLMSpec = spec - return input, nil -} - -func (r *llmRouter) ValidateLLMSkuUpdateData(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSkuUpdateInput) (*api.LLMSkuUpdateInput, error) { - var err error - input, err = r.baseDriver.ValidateLLMSkuUpdateData(ctx, userCred, sku, input) - if err != nil { - return nil, err - } - if input.LLMSpec != nil { - input.LLMSpec, err = r.ValidateLLMUpdateSpec(ctx, userCred, nil, input.LLMSpec) - } - return input, err -} - -func (r *llmRouter) ValidateLLMCreateSpec(ctx context.Context, userCred mcclient.TokenCredential, sku *models.SLLMSku, input *api.LLMSpec) (*api.LLMSpec, error) { - var skuSpec *api.LLMSpecLLMRouter - if sku != nil && sku.LLMSpec != nil { - skuSpec = sku.LLMSpec.LLMRouter - } - var inputSpec *api.LLMSpecLLMRouter - if input != nil { - inputSpec = input.LLMRouter - } - spec, err := r.normalizeSpec(mergeLLMRouterSpecs(skuSpec, inputSpec)) - if err != nil { - return input, err - } - return &api.LLMSpec{LLMRouter: spec}, nil -} - -func (r *llmRouter) ValidateLLMUpdateSpec(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *api.LLMSpec) (*api.LLMSpec, error) { - if input == nil || input.LLMRouter == nil { - return input, nil - } - spec, err := r.normalizeSpec(input.LLMRouter) - if err != nil { - return input, err - } - return &api.LLMSpec{LLMRouter: spec}, nil -} - -type llmRouterRuntimeArg struct { - Key string - Value string -} - -func (a llmRouterRuntimeArg) String() string { - key := strings.TrimSpace(a.Key) - if key == "" { - return "" - } - key = strings.TrimPrefix(key, "--") - if strings.TrimSpace(a.Value) == "" { - return "--" + key - } - return fmt.Sprintf("--%s %s", key, strings.TrimSpace(a.Value)) -} - -func llmRouterCustomizedArgsToRuntime(args []*api.LLMRouterArg) []llmRouterRuntimeArg { - out := make([]llmRouterRuntimeArg, 0, len(args)) - for _, arg := range args { - if arg == nil || strings.TrimSpace(arg.Key) == "" { - continue - } - out = append(out, llmRouterRuntimeArg{Key: arg.Key, Value: arg.Value}) - } - return out -} - -func buildLLMRouterEntrypointScript(spec *api.LLMSpecLLMRouter) string { - parts := []string{ - api.LLM_ROUTER_EXEC_PATH, - "--host 0.0.0.0", - fmt.Sprintf("--port %d", api.LLM_ROUTER_DEFAULT_PORT), - } - return strings.Join(parts, " ") -} - -func (r *llmRouter) GetContainerSpec(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) *computeapi.PodContainerCreateInput { - effSpec := &api.LLMSpecLLMRouter{RouterMethod: "routerdc"} - if eff := r.GetEffectiveSpec(llm, sku); eff != nil { - effSpec = eff.(*api.LLMSpecLLMRouter) - } - envs := []*commonapi.ContainerKeyValue{ - models.NewEnv("LLM_ROUTER_RUNTIME", effSpec.Runtime), - models.NewEnv("LLM_ROUTER_METHOD", effSpec.RouterMethod), - models.NewEnv("LLM_ROUTER_MODEL_DIR", effSpec.ModelDir), - models.NewEnv("LLM_ROUTER_ROUTE_PATH", effSpec.RoutePath), - models.NewEnv("LLM_ROUTER_HEALTH_PATH", effSpec.HealthPath), - } - if effSpec.ConfigPath != "" { - envs = append(envs, models.NewEnv("LLM_ROUTER_CONFIG", effSpec.ConfigPath)) - } - if effSpec.CandidateMappingPath != "" { - envs = append(envs, models.NewEnv("LLM_ROUTER_CANDIDATE_MAPPING", effSpec.CandidateMappingPath)) - } - for _, env := range effSpec.CustomizedEnvs { - if env != nil && strings.TrimSpace(env.Key) != "" { - envs = append(envs, models.NewEnv(strings.TrimSpace(env.Key), env.Value)) - } - } - spec := computeapi.ContainerSpec{ - ContainerSpec: commonapi.ContainerSpec{ - Image: image.ToContainerImage(), - ImageCredentialId: image.CredentialId, - Command: []string{"/bin/sh", "-c"}, - Args: []string{buildLLMRouterEntrypointScript(effSpec)}, - EnableLxcfs: true, - AlwaysRestart: true, - Envs: envs, - }, - } - appendContainerIsolatedDevices(&spec, llm, sku, devices) - diskIndex := 0 - spec.VolumeMounts = append(spec.VolumeMounts, - &commonapi.ContainerVolumeMount{ - Disk: &commonapi.ContainerVolumeMountDisk{ - SubDirectory: api.LLM_ROUTER, - Index: &diskIndex, - }, - Type: commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK, - MountPath: effSpec.ModelDir, - ReadOnly: false, - }, - &commonapi.ContainerVolumeMount{ - Disk: &commonapi.ContainerVolumeMountDisk{ - SubDirectory: "cache", - Index: &diskIndex, - }, - Type: commonapi.CONTAINER_VOLUME_MOUNT_TYPE_DISK, - MountPath: api.LLM_ROUTER_CACHE_DIR, - ReadOnly: false, - }, - ) - return &computeapi.PodContainerCreateInput{ContainerSpec: spec} -} - -func (r *llmRouter) GetContainerSpecs(ctx context.Context, llm *models.SLLM, image *models.SLLMImage, sku *models.SLLMSku, props []string, devices []computeapi.SIsolatedDevice, diskId string) []*computeapi.PodContainerCreateInput { - return []*computeapi.PodContainerCreateInput{ - r.GetContainerSpec(ctx, llm, image, sku, props, devices, diskId), - } -} - -func (r *llmRouter) GetLLMAccessUrlInfo(ctx context.Context, userCred mcclient.TokenCredential, llm *models.SLLM, input *models.LLMAccessInfoInput) (*api.LLMAccessUrlInfo, error) { - return models.GetLLMAccessUrlInfo(ctx, userCred, llm, input, "http", api.LLM_ROUTER_DEFAULT_PORT) -} diff --git a/pkg/llm/models/llm.go b/pkg/llm/models/llm.go index b62fefac1a..1c70520bb2 100644 --- a/pkg/llm/models/llm.go +++ b/pkg/llm/models/llm.go @@ -914,6 +914,13 @@ func (llm *SLLM) ValidateDeleteCondition(ctx context.Context, info jsonutils.JSO if cnt > 0 { return httperrors.NewConflictError("LLM is being used by %d MCPAgents", cnt) } + cnt, err = GetLLMRouterAgentManager().Query().Equals("llm_id", llm.Id).CountWithError() + if err != nil { + return errors.Wrap(err, "GetLLMRouterAgentManager().Query().CountWithError") + } + if cnt > 0 { + return httperrors.NewConflictError("LLM is being used by %d LLMRouterAgents", cnt) + } return nil } diff --git a/pkg/llm/models/llm_container_driver.go b/pkg/llm/models/llm_container_driver.go index 4cfb136b72..0688b0f1de 100644 --- a/pkg/llm/models/llm_container_driver.go +++ b/pkg/llm/models/llm_container_driver.go @@ -53,7 +53,8 @@ func getDriver[K ~string, D any](drvs *drivers, typ K) D { func getDriverWithError[K ~string, D any](drvs *drivers, typ K) (D, error) { drv, err := drvs.GetWithError(string(typ)) if err != nil { - return drv.(D), err + var zero D + return zero, err } return drv.(D), nil } diff --git a/pkg/llm/models/llm_deployment_gpu.go b/pkg/llm/models/llm_deployment_gpu.go index c585f312ad..47429a29fb 100644 --- a/pkg/llm/models/llm_deployment_gpu.go +++ b/pkg/llm/models/llm_deployment_gpu.go @@ -23,11 +23,6 @@ const ( autoGpuMemoryUtilizationMin = 0.05 autoGpuMemoryUtilizationMax = 0.95 - // vLLM derives an unset max-model-len from model config. When auto GPU - // memory utilization is enabled, inject a conservative cap so the heuristic - // VRAM estimate is not paired with an unexpectedly large context. - autoGpuMemoryUtilizationDefaultContextTokens = int64(8192) - sglangAutoGpuMemoryMetadataReserveMB = 512 ) @@ -465,7 +460,7 @@ func buildAutoGpuMemoryUtilizationLLMSpec(sku *SLLMSku, utilization float64) (*a if spec != nil && spec.Vllm != nil && !runtimeHasExplicitTokenLimit(sku) { spec.Vllm.CustomizedArgs = append(spec.Vllm.CustomizedArgs, &api.VllmCustomizedArg{ Key: "max-model-len", - Value: strconv.FormatInt(autoGpuMemoryUtilizationDefaultContextTokens, 10), + Value: strconv.FormatInt(api.LLM_DEFAULT_CONTEXT_TOKENS, 10), }) } } diff --git a/pkg/llm/models/llm_router_agent.go b/pkg/llm/models/llm_router_agent.go new file mode 100644 index 0000000000..e16f36d3ac --- /dev/null +++ b/pkg/llm/models/llm_router_agent.go @@ -0,0 +1,825 @@ +package models + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + seclib "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/stringutils2" +) + +func init() { + GetLLMRouterAgentManager() +} + +var llmRouterAgentManager *SLLMRouterAgentManager + +func GetLLMRouterAgentManager() *SLLMRouterAgentManager { + if llmRouterAgentManager != nil { + return llmRouterAgentManager + } + llmRouterAgentManager = &SLLMRouterAgentManager{ + SSharableVirtualResourceBaseManager: db.NewSharableVirtualResourceBaseManager( + SLLMRouterAgent{}, + "llm_router_agents_tbl", + "llm_router_agent", + "llm_router_agents", + ), + } + llmRouterAgentManager.SetVirtualObject(llmRouterAgentManager) + return llmRouterAgentManager +} + +type SLLMRouterAgentManager struct { + db.SSharableVirtualResourceBaseManager +} + +type SLLMRouterAgent struct { + db.SSharableVirtualResourceBase + + LLMId string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"optional" update:"user"` + LLMUrl string `width:"512" charset:"utf8" nullable:"false" list:"user" create:"required" update:"user"` + LLMDriver string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"optional" update:"user"` + Model string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" update:"user"` + ApiKey string `width:"512" charset:"utf8" nullable:"true" create:"optional" update:"user"` + DefaultRoute string `width:"32" charset:"ascii" nullable:"false" default:"complex" list:"user" create:"optional" update:"user"` + MaxPromptChars int `nullable:"false" default:"6144" list:"user" create:"optional" update:"user"` + MaxDecisionTokens int `nullable:"false" default:"64" list:"user" create:"optional" update:"user"` + CandidateMapping jsonutils.JSONObject `length:"long" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` + SimpleDefinition string `length:"long" charset:"utf8" nullable:"false" list:"user" create:"optional" update:"user"` + ComplexDefinition string `length:"long" charset:"utf8" nullable:"false" list:"user" create:"optional" update:"user"` + SimpleExamples jsonutils.JSONObject `length:"long" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` + ComplexExamples jsonutils.JSONObject `length:"long" charset:"utf8" nullable:"true" list:"user" create:"optional" update:"user"` +} + +func (agent *SLLMRouterAgent) BeforeInsert() { + if len(agent.Id) == 0 { + agent.Id = db.DefaultUUIDGenerator() + } + agent.encryptApiKey() + agent.SSharableVirtualResourceBase.BeforeInsert() +} + +func (agent *SLLMRouterAgent) BeforeUpdate() { + agent.encryptApiKey() +} + +func (agent *SLLMRouterAgent) encryptApiKey() { + if len(agent.ApiKey) == 0 { + return + } + if _, err := seclib.DescryptAESBase64(agent.Id, agent.ApiKey); err == nil { + return + } + sec, err := seclib.EncryptAESBase64(agent.Id, agent.ApiKey) + if err != nil { + log.Errorf("EncryptAESBase64 fail %s", err) + return + } + agent.ApiKey = sec +} + +func (agent *SLLMRouterAgent) GetApiKey() (string, error) { + if len(agent.ApiKey) == 0 { + return "", nil + } + key, err := seclib.DescryptAESBase64(agent.Id, agent.ApiKey) + if err == nil { + return key, nil + } + return "", errors.Wrap(err, "decrypt llm router api key") +} + +func (man *SLLMRouterAgentManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input *api.LLMRouterAgentCreateInput) (*api.LLMRouterAgentCreateInput, error) { + var err error + input.SharableVirtualResourceCreateInput, err = man.SSharableVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, input.SharableVirtualResourceCreateInput) + if err != nil { + return input, errors.Wrap(err, "validate SharableVirtualResourceCreateInput") + } + + if err := resolveRouterAgentLLM(ctx, userCred, query, &input.LLMId, &input.LLMUrl, &input.Model); err != nil { + return input, err + } + input.LLMDriver = normalizeRouterLLMDriver(input.LLMDriver) + if !api.IsLLMClientType(input.LLMDriver) { + return input, errors.Wrapf(httperrors.ErrInputParameter, "llm_driver must be one of: %s, got: %s", api.LLM_CLIENT_TYPES.List(), input.LLMDriver) + } + input.LLMUrl = strings.TrimSpace(input.LLMUrl) + input.Model = strings.TrimSpace(input.Model) + if input.LLMUrl == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "llm_url is required (or provide llm_id to auto-fetch)") + } + if input.Model == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "model is required") + } + input.DefaultRoute = normalizeRouterRoute(input.DefaultRoute) + input.MaxPromptChars, err = resolveRouterMaxPromptChars(ctx, input.MaxPromptChars, input.LLMUrl, input.Model, input.ApiKey) + if err != nil { + return input, err + } + input.MaxDecisionTokens = normalizePositiveInt(input.MaxDecisionTokens, api.LLM_ROUTER_DEFAULT_MAX_DECISION_TOKENS) + input.CandidateMapping = normalizeCandidateMapping(input.CandidateMapping) + promptConfig := normalizeRouterPromptConfig(api.LLMRouterPromptConfig{ + SimpleDefinition: input.SimpleDefinition, + ComplexDefinition: input.ComplexDefinition, + SimpleExamples: input.SimpleExamples, + ComplexExamples: input.ComplexExamples, + }) + input.SimpleDefinition = promptConfig.SimpleDefinition + input.ComplexDefinition = promptConfig.ComplexDefinition + input.SimpleExamples = promptConfig.SimpleExamples + input.ComplexExamples = promptConfig.ComplexExamples + input.Status = api.STATUS_READY + return input, nil +} + +func (agent *SLLMRouterAgent) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMRouterAgentUpdateInput) (api.LLMRouterAgentUpdateInput, error) { + var err error + input.SharableVirtualResourceBaseUpdateInput, err = agent.SSharableVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, input.SharableVirtualResourceBaseUpdateInput) + if err != nil { + return input, errors.Wrap(err, "validate SharableVirtualResourceBaseUpdateInput") + } + + if input.LLMId != nil && strings.TrimSpace(*input.LLMId) != "" { + llmId := strings.TrimSpace(*input.LLMId) + llmUrl := "" + model := "" + if input.LLMUrl != nil { + llmUrl = *input.LLMUrl + } + if input.Model != nil { + model = *input.Model + } + if err := resolveRouterAgentLLM(ctx, userCred, query, &llmId, &llmUrl, &model); err != nil { + return input, err + } + input.LLMId = &llmId + input.LLMUrl = &llmUrl + input.Model = &model + } + if input.LLMDriver != nil { + driver := normalizeRouterLLMDriver(*input.LLMDriver) + if !api.IsLLMClientType(driver) { + return input, errors.Wrapf(httperrors.ErrInputParameter, "llm_driver must be one of: %s, got: %s", api.LLM_CLIENT_TYPES.List(), driver) + } + input.LLMDriver = &driver + } + if input.LLMUrl != nil { + llmUrl := strings.TrimSpace(*input.LLMUrl) + if llmUrl == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "llm_url cannot be empty") + } + input.LLMUrl = &llmUrl + } + if input.Model != nil { + model := strings.TrimSpace(*input.Model) + if model == "" { + return input, errors.Wrap(httperrors.ErrInputParameter, "model cannot be empty") + } + input.Model = &model + } + if input.DefaultRoute != nil { + route := normalizeRouterRoute(*input.DefaultRoute) + input.DefaultRoute = &route + } + if err := agent.normalizeRouterUpdateMaxPromptChars(ctx, &input); err != nil { + return input, err + } + if input.MaxDecisionTokens != nil && *input.MaxDecisionTokens <= 0 { + val := api.LLM_ROUTER_DEFAULT_MAX_DECISION_TOKENS + input.MaxDecisionTokens = &val + } + if input.CandidateMapping != nil { + mapping := normalizeCandidateMapping(*input.CandidateMapping) + input.CandidateMapping = &mapping + } + normalizeRouterPromptUpdate(&input) + return input, nil +} + +func resolveRouterAgentLLM(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, llmId *string, llmUrl *string, model *string) error { + if llmId == nil || strings.TrimSpace(*llmId) == "" { + return nil + } + llmObj, err := GetLLMManager().FetchByIdOrName(ctx, userCred, strings.TrimSpace(*llmId)) + if err != nil { + return errors.Wrapf(err, "fetch LLM by id %s", *llmId) + } + llm := llmObj.(*SLLM) + *llmId = llm.Id + info, err := llm.GetLLMAccessUrlInfo(ctx, userCred, query) + if err != nil { + return errors.Wrapf(err, "get LLM URL from LLM %s", *llmId) + } + *llmUrl = info.LoginUrl + if strings.TrimSpace(*model) != "" { + return nil + } + mdlInfos, err := llm.getProbedInstantModelsExt(ctx, userCred) + if err != nil { + return errors.Wrap(err, "get probed models from LLM instance") + } + if len(mdlInfos) == 0 { + return httperrors.NewBadRequestError("no available models found in LLM instance %s", *llmId) + } + var first api.LLMInternalInstantMdlInfo + for _, mdlInfo := range mdlInfos { + first = mdlInfo + break + } + *model = fmt.Sprintf("%s:%s", first.Name, first.Tag) + return nil +} + +func normalizeRouterLLMDriver(driver string) string { + driver = strings.ToLower(strings.TrimSpace(driver)) + if driver == "" { + return string(api.LLM_CLIENT_OPENAI) + } + return driver +} + +func normalizeRouterRoute(route string) string { + route = strings.ToLower(strings.TrimSpace(route)) + if route == "" { + return api.LLM_ROUTER_DEFAULT_ROUTE + } + return route +} + +func normalizePositiveInt(val int, def int) int { + if val <= 0 { + return def + } + return val +} + +const routerModelsTimeout = 3 * time.Second + +type routerModelsResponse struct { + Data []routerModelEntry `json:"data"` +} + +type routerModelEntry struct { + ID string `json:"id"` + MaxModelLen int `json:"max_model_len"` + ContextLength int `json:"context_length"` +} + +func buildRouterModelsURL(endpoint string) (string, error) { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + return "", errors.Error("llm_url is empty") + } + u, err := url.Parse(endpoint) + if err != nil { + return "", errors.Wrap(err, "parse llm_url") + } + if u.Scheme == "" || u.Host == "" { + return "", errors.Error("llm_url must be absolute") + } + u.RawQuery = "" + u.Fragment = "" + path := strings.TrimRight(u.Path, "/") + switch { + case path == "": + u.Path = "/v1/models" + case strings.HasSuffix(path, "/v1/models"): + u.Path = path + case strings.HasSuffix(path, "/v1"): + u.Path = path + "/models" + default: + u.Path = path + "/v1/models" + } + return u.String(), nil +} + +func fetchRouterContextWindow(ctx context.Context, client *http.Client, endpoint, model, apiKey string) (int, error) { + modelsURL, err := buildRouterModelsURL(endpoint) + if err != nil { + return 0, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, modelsURL, nil) + if err != nil { + return 0, errors.Wrap(err, "create /v1/models request") + } + req.Header.Set("Accept", "application/json") + if apiKey = strings.TrimSpace(apiKey); apiKey != "" { + req.Header.Set("Authorization", "Bearer "+apiKey) + } + resp, err := client.Do(req) + if err != nil { + return 0, errors.Wrap(err, "request /v1/models") + } + defer resp.Body.Close() + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return 0, errors.Errorf("/v1/models returned status %d", resp.StatusCode) + } + var out routerModelsResponse + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&out); err != nil { + return 0, errors.Wrap(err, "decode /v1/models response") + } + model = strings.TrimSpace(model) + for _, item := range out.Data { + if strings.TrimSpace(item.ID) != model { + continue + } + if item.MaxModelLen > 0 { + return item.MaxModelLen, nil + } + if item.ContextLength > 0 { + return item.ContextLength, nil + } + return 0, errors.Errorf("model %q has no valid context length", model) + } + return 0, errors.Errorf("model %q not found in /v1/models", model) +} + +func routerMaxPromptChars(contextTokens int) int { + if contextTokens <= 0 { + contextTokens = api.LLM_DEFAULT_CONTEXT_TOKENS + } + return contextTokens * 3 / 4 +} + +func resolveRouterMaxPromptChars(ctx context.Context, requested int, endpoint, model, apiKey string) (int, error) { + if requested < 0 { + return 0, errors.Wrapf(httperrors.ErrInputParameter, "max_prompt_chars must be >= 0, got %d", requested) + } + if requested > 0 { + return requested, nil + } + contextTokens, err := fetchRouterContextWindow(ctx, &http.Client{Timeout: routerModelsTimeout}, endpoint, model, apiKey) + if err != nil { + log.Warningf("resolve llm router context window for model %q failed: %v; using default %d", model, err, api.LLM_DEFAULT_CONTEXT_TOKENS) + contextTokens = api.LLM_DEFAULT_CONTEXT_TOKENS + } + return routerMaxPromptChars(contextTokens), nil +} + +func (agent *SLLMRouterAgent) normalizeRouterUpdateMaxPromptChars(ctx context.Context, input *api.LLMRouterAgentUpdateInput) error { + if input.MaxPromptChars == nil { + return nil + } + requested := *input.MaxPromptChars + if requested != 0 { + maxPromptChars, err := resolveRouterMaxPromptChars(ctx, requested, "", "", "") + if err != nil { + return err + } + input.MaxPromptChars = &maxPromptChars + return nil + } + + llmURL, model := agent.LLMUrl, agent.Model + if input.LLMUrl != nil { + llmURL = *input.LLMUrl + } + if input.Model != nil { + model = *input.Model + } + apiKey := "" + if input.ApiKey != nil { + apiKey = *input.ApiKey + } else { + storedAPIKey, err := agent.GetApiKey() + if err != nil { + return errors.Wrap(err, "decrypt llm router api key") + } + apiKey = storedAPIKey + } + maxPromptChars, err := resolveRouterMaxPromptChars(ctx, 0, llmURL, model, apiKey) + if err != nil { + return err + } + input.MaxPromptChars = &maxPromptChars + return nil +} + +func normalizeCandidateMapping(in map[string][]string) map[string][]string { + out := make(map[string][]string, len(in)) + for route, models := range in { + route = normalizeRouterRoute(route) + for _, model := range models { + model = strings.TrimSpace(model) + if model != "" { + out[route] = append(out[route], model) + } + } + } + return out +} + +func normalizeRouterPromptConfig(cfg api.LLMRouterPromptConfig) api.LLMRouterPromptConfig { + cfg.SimpleDefinition = normalizeRouterDefinition(cfg.SimpleDefinition, api.LLM_ROUTER_DEFAULT_SIMPLE_DEFINITION) + cfg.ComplexDefinition = normalizeRouterDefinition(cfg.ComplexDefinition, api.LLM_ROUTER_DEFAULT_COMPLEX_DEFINITION) + cfg.SimpleExamples = normalizeRouterExamples(cfg.SimpleExamples) + cfg.ComplexExamples = normalizeRouterExamples(cfg.ComplexExamples) + return cfg +} + +func normalizeRouterDefinition(definition, fallback string) string { + definition = strings.TrimSpace(definition) + if definition == "" { + return fallback + } + return definition +} + +func normalizeRouterExamples(in []string) []string { + out := make([]string, 0, len(in)) + seen := map[string]struct{}{} + for _, example := range in { + example = strings.TrimSpace(example) + if example == "" { + continue + } + if _, ok := seen[example]; ok { + continue + } + seen[example] = struct{}{} + out = append(out, example) + } + return out +} + +func normalizeRouterPromptUpdate(input *api.LLMRouterAgentUpdateInput) { + if input.SimpleDefinition != nil { + definition := normalizeRouterDefinition(*input.SimpleDefinition, api.LLM_ROUTER_DEFAULT_SIMPLE_DEFINITION) + input.SimpleDefinition = &definition + } + if input.ComplexDefinition != nil { + definition := normalizeRouterDefinition(*input.ComplexDefinition, api.LLM_ROUTER_DEFAULT_COMPLEX_DEFINITION) + input.ComplexDefinition = &definition + } + if input.SimpleExamples != nil { + examples := normalizeRouterExamples(*input.SimpleExamples) + input.SimpleExamples = &examples + } + if input.ComplexExamples != nil { + examples := normalizeRouterExamples(*input.ComplexExamples) + input.ComplexExamples = &examples + } +} + +func (man *SLLMRouterAgentManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.LLMRouterAgentListInput) (*sqlchemy.SQuery, error) { + q, err := man.SSharableVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, input.SharableVirtualResourceListInput) + if err != nil { + return nil, errors.Wrap(err, "SSharableVirtualResourceBaseManager.ListItemFilter") + } + if len(input.LLMDriver) > 0 { + q = q.Equals("llm_driver", normalizeRouterLLMDriver(input.LLMDriver)) + } + return q, nil +} + +func (manager *SLLMRouterAgentManager) FetchCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []interface{}, fields stringutils2.SSortedStrings, isList bool) []api.LLMRouterAgentDetails { + rows := make([]api.LLMRouterAgentDetails, len(objs)) + vrows := manager.SSharableVirtualResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) + agents := []SLLMRouterAgent{} + jsonutils.Update(&agents, objs) + + llmIds := make([]string, 0) + for i := range agents { + if agents[i].LLMId != "" { + llmIds = append(llmIds, agents[i].LLMId) + } + } + var llmIdNameMap map[string]string + if len(llmIds) > 0 { + var err error + llmIdNameMap, err = db.FetchIdNameMap2(GetLLMManager(), llmIds) + if err != nil { + log.Errorf("FetchIdNameMap2 for LLMs failed: %v", err) + } + } + for i := range rows { + rows[i].SharableVirtualResourceDetails = vrows[i] + if i < len(agents) { + rows[i].LLMId = agents[i].LLMId + if name, ok := llmIdNameMap[agents[i].LLMId]; ok { + rows[i].LLMName = name + } + } + } + return rows +} + +func (agent *SLLMRouterAgent) PerformRoute(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMRouterRouteRequest) (jsonutils.JSONObject, error) { + out, err := agent.Route(ctx, input) + if err != nil { + return nil, err + } + return jsonutils.Marshal(out), nil +} + +func (agent *SLLMRouterAgent) PerformExpandExamples(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.LLMRouterExpandExamplesInput) (jsonutils.JSONObject, error) { + cfg := agent.promptConfig() + applyRouterPromptDraft(&cfg, input) + cfg = normalizeRouterPromptConfig(cfg) + content, err := callRouterAgentModel(ctx, agent, routerExpandSystemPrompt, buildRouterExpandPrompt(cfg)) + if err != nil { + return nil, errors.Wrap(err, "expand router examples") + } + out, err := parseRouterPromptConfig(content) + if err != nil { + return nil, errors.Wrap(err, "parse expanded router examples") + } + return jsonutils.Marshal(out), nil +} + +func (agent *SLLMRouterAgent) Route(ctx context.Context, req api.LLMRouterRouteRequest) (*api.LLMRouterRouteResponse, error) { + if len(req.Candidates) == 0 { + return nil, errors.Wrap(httperrors.ErrInputParameter, "candidates is required") + } + route, ok := agent.routeByRules(req) + if !ok { + route = agent.routeByDecisionModel(ctx, req) + } + if route == "" { + route = normalizeRouterRoute(agent.DefaultRoute) + } + model := agent.pickCandidate(route, req.Candidates) + if model == "" { + return nil, errors.Wrap(httperrors.ErrInvalidStatus, "no selectable candidate model") + } + return &api.LLMRouterRouteResponse{Model: model}, nil +} + +func (agent *SLLMRouterAgent) routeByRules(req api.LLMRouterRouteRequest) (string, bool) { + prompt := agent.requestText(req) + maxChars := normalizePositiveInt(agent.MaxPromptChars, api.LLM_ROUTER_DEFAULT_MAX_PROMPT_CHARS) + if len([]rune(prompt)) > maxChars { + return api.LLM_ROUTER_ROUTE_COMPLEX, true + } + return "", false +} + +const ( + routerDecisionSystemPrompt = "你是模型分流器。根据给定定义和样例把请求分类为 simple 或 complex。只能输出一个 JSON 对象，不得输出 Markdown、代码块或额外解释。" + routerExpandSystemPrompt = "你是模型分流规则编辑器。规范 simple 和 complex 的定义并补充代表性样例。只能输出指定的 JSON 对象，不得输出 Markdown、代码块或额外解释。" +) + +var callRouterAgentModel = func(ctx context.Context, agent *SLLMRouterAgent, systemPrompt, userPrompt string) (string, error) { + if strings.TrimSpace(agent.LLMUrl) == "" || strings.TrimSpace(agent.Model) == "" { + return "", errors.Wrap(httperrors.ErrInvalidStatus, "router model URL and model are required") + } + driver, err := GetLLMClientDriverWithError(api.LLMClientType(normalizeRouterLLMDriver(agent.LLMDriver))) + if err != nil { + return "", err + } + apiKey, err := agent.GetApiKey() + if err != nil { + return "", err + } + tmp := &SMCPAgent{ + LLMUrl: agent.LLMUrl, + LLMDriver: normalizeRouterLLMDriver(agent.LLMDriver), + Model: agent.Model, + ApiKey: apiKey, + } + timeout := time.Duration(options.Options.MCPAgentTimeout) * time.Second + if timeout <= 0 { + timeout = 30 * time.Second + } + callCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + resp, err := driver.Chat(callCtx, tmp, []ILLMChatMessage{ + driver.NewSystemMessage(systemPrompt), + driver.NewUserMessage(userPrompt), + }, nil) + if err != nil { + return "", err + } + return resp.GetContent(), nil +} + +func (agent *SLLMRouterAgent) routeByDecisionModel(ctx context.Context, req api.LLMRouterRouteRequest) string { + prompt := agent.BuildDecisionPrompt(req) + content, err := callRouterAgentModel(ctx, agent, routerDecisionSystemPrompt, prompt) + if err != nil { + log.Warningf("llm router decision model failed: %v", err) + return normalizeRouterRoute(agent.DefaultRoute) + } + decision, err := parseRouterDecision(content) + if err != nil || decision.Confidence < 0.5 { + if err != nil { + log.Warningf("parse llm router decision failed: %v", err) + } + return normalizeRouterRoute(agent.DefaultRoute) + } + return normalizeRouterRoute(decision.Route) +} + +func (agent *SLLMRouterAgent) BuildDecisionPrompt(req api.LLMRouterRouteRequest) string { + promptConfig := agent.promptConfig() + maxTokens := normalizePositiveInt(agent.MaxDecisionTokens, api.LLM_ROUTER_DEFAULT_MAX_DECISION_TOKENS) + return fmt.Sprintf(`根据以下定义和样例选择 route。 + +simple 定义： +%s + +complex 定义： +%s + +simple 样例： +%s + +complex 样例： +%s + +候选模型：%s +最多输出约 %d tokens。 + +只返回 JSON，不要解释： +{"route":"simple or complex","confidence":0.0,"reason":"short reason"} + +请求内容： +%s`, promptConfig.SimpleDefinition, promptConfig.ComplexDefinition, formatRouterExamples(promptConfig.SimpleExamples), formatRouterExamples(promptConfig.ComplexExamples), strings.Join(req.Candidates, ", "), maxTokens, agent.requestText(req)) +} + +func buildRouterExpandPrompt(cfg api.LLMRouterPromptConfig) string { + return fmt.Sprintf(`请规范以下二分类定义和样例，并补充少量有代表性的样例。保持 simple 和 complex 边界互斥且清晰，不要增加其他 route。 + +simple 定义： +%s + +complex 定义： +%s + +simple 样例： +%s + +complex 样例： +%s + +只返回： +{"simple_definition":"...","complex_definition":"...","simple_examples":["..."],"complex_examples":["..."]}`, + cfg.SimpleDefinition, + cfg.ComplexDefinition, + formatRouterExamples(cfg.SimpleExamples), + formatRouterExamples(cfg.ComplexExamples), + ) +} + +func applyRouterPromptDraft(cfg *api.LLMRouterPromptConfig, input api.LLMRouterExpandExamplesInput) { + if input.SimpleDefinition != nil { + cfg.SimpleDefinition = *input.SimpleDefinition + } + if input.ComplexDefinition != nil { + cfg.ComplexDefinition = *input.ComplexDefinition + } + if input.SimpleExamples != nil { + cfg.SimpleExamples = *input.SimpleExamples + } + if input.ComplexExamples != nil { + cfg.ComplexExamples = *input.ComplexExamples + } +} + +func (agent *SLLMRouterAgent) promptConfig() api.LLMRouterPromptConfig { + cfg := api.LLMRouterPromptConfig{ + SimpleDefinition: agent.SimpleDefinition, + ComplexDefinition: agent.ComplexDefinition, + } + if agent.SimpleExamples != nil { + if err := agent.SimpleExamples.Unmarshal(&cfg.SimpleExamples); err != nil { + log.Warningf("invalid simple_examples for llm router agent %s: %v", agent.Id, err) + } + } + if agent.ComplexExamples != nil { + if err := agent.ComplexExamples.Unmarshal(&cfg.ComplexExamples); err != nil { + log.Warningf("invalid complex_examples for llm router agent %s: %v", agent.Id, err) + } + } + return normalizeRouterPromptConfig(cfg) +} + +func formatRouterExamples(examples []string) string { + if len(examples) == 0 { + return "无" + } + return "- " + strings.Join(examples, "\n- ") +} + +func (agent *SLLMRouterAgent) requestText(req api.LLMRouterRouteRequest) string { + parts := make([]string, 0, len(req.Messages)) + for _, msg := range req.Messages { + switch val := msg.Content.(type) { + case string: + parts = append(parts, val) + default: + parts = append(parts, jsonutils.Marshal(val).String()) + } + } + return strings.Join(parts, "\n") +} + +func (agent *SLLMRouterAgent) candidateMapping() map[string][]string { + ret := map[string][]string{} + if agent.CandidateMapping == nil { + return ret + } + if err := agent.CandidateMapping.Unmarshal(&ret); err != nil { + log.Warningf("invalid llm router candidate_mapping: %v", err) + return map[string][]string{} + } + return normalizeCandidateMapping(ret) +} + +func (agent *SLLMRouterAgent) pickCandidate(route string, candidates []string) string { + mapping := agent.candidateMapping() + if model := pickCandidateFromList(mapping[normalizeRouterRoute(route)], candidates); model != "" { + return model + } + def := normalizeRouterRoute(agent.DefaultRoute) + if def != normalizeRouterRoute(route) { + if model := pickCandidateFromList(mapping[def], candidates); model != "" { + return model + } + } + if model := pickCandidateFromList([]string{route}, candidates); model != "" { + return model + } + return strings.TrimSpace(candidates[0]) +} + +func pickCandidateFromList(models []string, candidates []string) string { + for _, model := range models { + model = strings.TrimSpace(model) + for _, candidate := range candidates { + if strings.EqualFold(strings.TrimSpace(candidate), model) { + return strings.TrimSpace(candidate) + } + } + } + return "" +} + +func parseRouterDecision(content string) (*api.LLMRouterDecision, error) { + content = extractRouterJSON(content) + decision := api.LLMRouterDecision{} + if err := json.Unmarshal([]byte(content), &decision); err != nil { + return nil, err + } + decision.Route = normalizeRouterRoute(decision.Route) + if decision.Route == "" { + return nil, errors.Error("missing route") + } + if decision.Confidence == 0 { + decision.Confidence = 1 + } + return &decision, nil +} + +func parseRouterPromptConfig(content string) (*api.LLMRouterPromptConfig, error) { + var raw struct { + SimpleDefinition *string `json:"simple_definition"` + ComplexDefinition *string `json:"complex_definition"` + SimpleExamples *[]string `json:"simple_examples"` + ComplexExamples *[]string `json:"complex_examples"` + } + if err := json.Unmarshal([]byte(extractRouterJSON(content)), &raw); err != nil { + return nil, err + } + if raw.SimpleDefinition == nil || raw.ComplexDefinition == nil || raw.SimpleExamples == nil || raw.ComplexExamples == nil { + return nil, errors.Error("router expansion response missing required fields") + } + cfg := normalizeRouterPromptConfig(api.LLMRouterPromptConfig{ + SimpleDefinition: *raw.SimpleDefinition, + ComplexDefinition: *raw.ComplexDefinition, + SimpleExamples: *raw.SimpleExamples, + ComplexExamples: *raw.ComplexExamples, + }) + return &cfg, nil +} + +func extractRouterJSON(content string) string { + content = strings.TrimSpace(content) + content = strings.TrimPrefix(content, "```json") + content = strings.TrimPrefix(content, "```") + content = strings.TrimSuffix(content, "```") + content = strings.TrimSpace(content) + if start := strings.Index(content, "{"); start >= 0 { + if end := strings.LastIndex(content, "}"); end > start { + content = content[start : end+1] + } + } + return content +} diff --git a/pkg/llm/service/handler.go b/pkg/llm/service/handler.go index 419af94584..e8aef4eea7 100644 --- a/pkg/llm/service/handler.go +++ b/pkg/llm/service/handler.go @@ -304,6 +304,41 @@ func handleDefaultMcpTools(ctx context.Context, w http.ResponseWriter, r *http.R appsrv.SendJSON(w, result) } +func handleLLMRouterAgentRoute(ctx context.Context, w http.ResponseWriter, r *http.Request) { + params, _, body := appsrv.FetchEnv(ctx, w, r) + id := params[""] + if id == "" { + httperrors.MissingParameterError(ctx, w, "id") + return + } + if body == nil { + body = jsonutils.NewDict() + } + if body.Contains(models.GetLLMRouterAgentManager().Keyword()) { + agentObj, _ := body.Get(models.GetLLMRouterAgentManager().Keyword()) + if agentObj != nil { + body = agentObj + } + } + input := api.LLMRouterRouteRequest{} + if err := body.Unmarshal(&input); err != nil { + httperrors.InvalidInputError(ctx, w, "invalid input: %v", err) + return + } + obj, err := models.GetLLMRouterAgentManager().FetchByIdOrName(ctx, nil, id) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + agent := obj.(*models.SLLMRouterAgent) + out, err := agent.Route(ctx, input) + if err != nil { + httperrors.GeneralServerError(ctx, w, err) + return + } + appsrv.SendStruct(w, out) +} + func InitHandlers(app *appsrv.Application, isSlave bool) { db.InitAllManagers() db.RegistUserCredCacheUpdater() @@ -340,6 +375,8 @@ func InitHandlers(app *appsrv.Application, isSlave bool) { // 默认 MCP 服务器 tools：仅使用 options.MCPServerURL，不依赖 mcp_agent 条目 app.AddHandler2("GET", "/mcp_agents/default-mcp-tools", auth.Authenticate(handleDefaultMcpTools), nil, "default_mcp_tools", nil) + app.AddHandler2("POST", "/llm_router_agents//route", handleLLMRouterAgentRoute, nil, "llm_router_agent_route", nil) + for _, manager := range []db.IModelManager{ taskman.TaskManager, taskman.SubTaskManager, @@ -372,6 +409,7 @@ func InitHandlers(app *appsrv.Application, isSlave bool) { models.GetInstantModelManager(), models.GetLLMInstantModelManager(), models.GetMCPAgentManager(), + models.GetLLMRouterAgentManager(), } { db.RegisterModelManager(manager) handler := db.NewModelHandler(manager) diff --git a/pkg/mcclient/modules/llm/mod_llm_router_agent.go b/pkg/mcclient/modules/llm/mod_llm_router_agent.go new file mode 100644 index 0000000000..b5998990d7 --- /dev/null +++ b/pkg/mcclient/modules/llm/mod_llm_router_agent.go @@ -0,0 +1,24 @@ +package llm + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +type LLMRouterAgentManager struct { + modulebase.ResourceManager +} + +var ( + LLMRouterAgent LLMRouterAgentManager +) + +func init() { + LLMRouterAgent = LLMRouterAgentManager{ + ResourceManager: modules.NewLLMManager("llm_router_agent", "llm_router_agents", + []string{}, + []string{}, + ), + } + modules.Register(&LLMRouterAgent) +} diff --git a/pkg/mcclient/options/llm/image.go b/pkg/mcclient/options/llm/image.go index 4f619554e0..6fa4fb37c2 100644 --- a/pkg/mcclient/options/llm/image.go +++ b/pkg/mcclient/options/llm/image.go @@ -18,7 +18,7 @@ func (o *LLMImageShowOptions) Params() (jsonutils.JSONObject, error) { type LLMImageListOptions struct { options.BaseListOptions - LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop|benchmark" help:"filter by llm type"` + LLMType string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|desktop|benchmark" help:"filter by llm type"` } func (o *LLMImageListOptions) Params() (jsonutils.JSONObject, error) { @@ -30,7 +30,7 @@ type LLMImageCreateOptions struct { IMAGE_NAME string `json:"image_name"` IMAGE_LABEL string `json:"image_label"` CredentialId string `json:"credential_id"` - LLM_TYPE string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|llm-router|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop, benchmark or dify"` + LLM_TYPE string `json:"llm_type" choices:"ollama|dify|comfyui|vllm|sglang|hermes-agent|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, desktop, benchmark or dify"` } func (o *LLMImageCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -44,7 +44,7 @@ type LLMImageUpdateOptions struct { ImageName string `json:"image_name"` ImageLabel string `json:"image_label"` CredentialId string `json:"credential_id"` - LlmType string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|llm-router|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, llm-router, desktop, benchmark or dify"` + LlmType string `json:"llm_type" choices:"ollama|dify|vllm|sglang|comfyui|hermes-agent|desktop|benchmark" help:"llm type: ollama, comfyui, vllm, sglang, hermes-agent, desktop, benchmark or dify"` AppName string `json:"app_name" help:"desktop application identifier (LinuxServer image id), e.g. firefox, chromium, steam, webtop-ubuntu-xfce"` } diff --git a/pkg/mcclient/options/llm/llm_deployment.go b/pkg/mcclient/options/llm/llm_deployment.go index 59a52492ab..c8d129b466 100644 --- a/pkg/mcclient/options/llm/llm_deployment.go +++ b/pkg/mcclient/options/llm/llm_deployment.go @@ -47,7 +47,7 @@ type LLMDeploymentCreateOptions struct { // Mode B/C: SKU spec (all sku-* flags collapse into sku_spec) SkuName string `help:"SKU name (default: -sku)" json:"-"` SkuLLMImageId string `help:"container image id (for SkuSpec)" json:"-"` - SkuLLMType string `help:"container llm type" choices:"ollama|vllm|comfyui|sglang|dify|llm-router" json:"-"` + SkuLLMType string `help:"container llm type" choices:"ollama|vllm|comfyui|sglang|dify" json:"-"` SkuCpu int `help:"SKU CPU cores" json:"-"` SkuMemory int `help:"SKU memory MB" json:"-"` SkuDiskSize int `help:"SKU disk size MB" json:"-"` diff --git a/pkg/mcclient/options/llm/llm_router_agent.go b/pkg/mcclient/options/llm/llm_router_agent.go new file mode 100644 index 0000000000..3cbd2154f8 --- /dev/null +++ b/pkg/mcclient/options/llm/llm_router_agent.go @@ -0,0 +1,203 @@ +package llm + +import ( + "fmt" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type LLMRouterAgentListOptions struct { + options.BaseListOptions + + LLMDriver string `json:"llm_driver" help:"filter by llm driver"` +} + +func (o *LLMRouterAgentListOptions) Params() (jsonutils.JSONObject, error) { + return options.ListStructToParams(o) +} + +type LLMRouterAgentShowOptions struct { + options.BaseShowOptions +} + +func (o *LLMRouterAgentShowOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMRouterAgentCreateOptions struct { + apis.SharableVirtualResourceCreateInput + + LlmId string `help:"LLM instance ID; if set, llm_url is resolved from it" json:"llm_id"` + LLM_URL string `help:"decision model OpenAI-compatible base URL" json:"llm_url"` + LLM_DRIVER string `help:"decision model driver" json:"llm_driver" choices:"ollama|openai"` + MODEL string `help:"decision model name" json:"model"` + API_KEY string `help:"decision model API key" json:"api_key"` + DefaultRoute string `help:"fallback route, default complex" json:"default_route"` + MaxPromptChars int `help:"request char limit; 0 auto-detects /v1/models context, longer requests use complex" json:"max_prompt_chars"` + MaxDecisionTokens int `help:"decision answer token hint" json:"max_decision_tokens"` + CandidateMapping string `help:"candidate mapping JSON, e.g. '{\"simple\":[\"qwen\"],\"complex\":[\"deepseek\"]}'" json:"-"` + SimpleDefinition string `help:"definition of simple requests" json:"simple_definition"` + ComplexDefinition string `help:"definition of complex requests" json:"complex_definition"` + SimpleExamples string `help:"simple examples JSON array" json:"-"` + ComplexExamples string `help:"complex examples JSON array" json:"-"` +} + +func (o *LLMRouterAgentCreateOptions) Params() (jsonutils.JSONObject, error) { + obj := jsonutils.Marshal(o).(*jsonutils.JSONDict) + if o.CandidateMapping != "" { + mapping, err := parseRouterCandidateMapping(o.CandidateMapping) + if err != nil { + return nil, err + } + obj.Set("candidate_mapping", jsonutils.Marshal(mapping)) + } + if err := setRouterExamples(obj, "simple_examples", o.SimpleExamples); err != nil { + return nil, err + } + if err := setRouterExamples(obj, "complex_examples", o.ComplexExamples); err != nil { + return nil, err + } + return obj, nil +} + +type LLMRouterAgentUpdateOptions struct { + apis.SharableVirtualResourceBaseUpdateInput + + ID string + LlmId *string `help:"LLM instance ID; if set, llm_url is resolved from it" json:"llm_id,omitempty"` + LlmUrl *string `help:"decision model OpenAI-compatible base URL" json:"llm_url,omitempty"` + LlmDriver *string `help:"decision model driver" json:"llm_driver,omitempty" choices:"ollama|openai"` + Model *string `help:"decision model name" json:"model,omitempty"` + ApiKey *string `help:"decision model API key" json:"api_key,omitempty"` + DefaultRoute *string `help:"fallback route" json:"default_route,omitempty"` + MaxPromptChars *int `help:"request char limit; 0 recomputes from /v1/models context" json:"max_prompt_chars,omitempty"` + MaxDecisionTokens *int `help:"decision answer token hint" json:"max_decision_tokens,omitempty"` + CandidateMapping string `help:"candidate mapping JSON" json:"-"` + SimpleDefinition *string `help:"definition of simple requests" json:"simple_definition,omitempty"` + ComplexDefinition *string `help:"definition of complex requests" json:"complex_definition,omitempty"` + SimpleExamples string `help:"replace simple examples with JSON array" json:"-"` + ComplexExamples string `help:"replace complex examples with JSON array" json:"-"` +} + +func (o *LLMRouterAgentUpdateOptions) GetId() string { + return o.ID +} + +func (o *LLMRouterAgentUpdateOptions) Params() (jsonutils.JSONObject, error) { + obj := jsonutils.Marshal(o).(*jsonutils.JSONDict) + obj.Remove("id") + if o.CandidateMapping != "" { + mapping, err := parseRouterCandidateMapping(o.CandidateMapping) + if err != nil { + return nil, err + } + obj.Set("candidate_mapping", jsonutils.Marshal(mapping)) + } + if err := setRouterExamples(obj, "simple_examples", o.SimpleExamples); err != nil { + return nil, err + } + if err := setRouterExamples(obj, "complex_examples", o.ComplexExamples); err != nil { + return nil, err + } + return obj, nil +} + +type LLMRouterAgentDeleteOptions struct { + options.BaseIdOptions +} + +func (o *LLMRouterAgentDeleteOptions) GetId() string { + return o.ID +} + +func (o *LLMRouterAgentDeleteOptions) Params() (jsonutils.JSONObject, error) { + return options.StructToParams(o) +} + +type LLMRouterAgentRouteOptions struct { + ID string `help:"llm router agent id or name" json:"-"` + Model string `help:"requested model" json:"model"` + Message []string `help:"user message; repeatable" json:"-"` + Candidates []string `help:"candidate model; repeatable" json:"candidates"` +} + +func (o *LLMRouterAgentRouteOptions) GetId() string { + return o.ID +} + +func (o *LLMRouterAgentRouteOptions) Params() (jsonutils.JSONObject, error) { + messages := make([]api.LLMRouterMessage, 0, len(o.Message)) + for _, msg := range o.Message { + messages = append(messages, api.LLMRouterMessage{Role: "user", Content: msg}) + } + input := api.LLMRouterRouteRequest{ + Model: o.Model, + Messages: messages, + Candidates: o.Candidates, + } + return jsonutils.Marshal(input), nil +} + +type LLMRouterAgentExpandExamplesOptions struct { + ID string `help:"llm router agent id or name" json:"-"` + SimpleDefinition *string `help:"draft definition of simple requests" json:"simple_definition,omitempty"` + ComplexDefinition *string `help:"draft definition of complex requests" json:"complex_definition,omitempty"` + SimpleExamples string `help:"draft simple examples JSON array" json:"-"` + ComplexExamples string `help:"draft complex examples JSON array" json:"-"` +} + +func (o *LLMRouterAgentExpandExamplesOptions) GetId() string { + return o.ID +} + +func (o *LLMRouterAgentExpandExamplesOptions) Params() (jsonutils.JSONObject, error) { + obj := jsonutils.Marshal(o).(*jsonutils.JSONDict) + obj.Remove("id") + if err := setRouterExamples(obj, "simple_examples", o.SimpleExamples); err != nil { + return nil, err + } + if err := setRouterExamples(obj, "complex_examples", o.ComplexExamples); err != nil { + return nil, err + } + return obj, nil +} + +func parseRouterCandidateMapping(raw string) (map[string][]string, error) { + obj, err := jsonutils.ParseString(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse candidate mapping JSON: %v", err) + } + mapping := map[string][]string{} + if err := obj.Unmarshal(&mapping); err != nil { + return nil, fmt.Errorf("failed to unmarshal candidate mapping: %v", err) + } + return mapping, nil +} + +func setRouterExamples(obj *jsonutils.JSONDict, key, raw string) error { + if raw == "" { + return nil + } + examples, err := parseRouterExamples(raw) + if err != nil { + return err + } + obj.Set(key, jsonutils.Marshal(examples)) + return nil +} + +func parseRouterExamples(raw string) ([]string, error) { + obj, err := jsonutils.ParseString(raw) + if err != nil { + return nil, fmt.Errorf("failed to parse router examples JSON: %v", err) + } + examples := []string{} + if err := obj.Unmarshal(&examples); err != nil { + return nil, fmt.Errorf("failed to unmarshal router examples: %v", err) + } + return examples, nil +} diff --git a/pkg/mcclient/options/llm/llm_sku.go b/pkg/mcclient/options/llm/llm_sku.go index 9e72bd1219..4baad709e2 100644 --- a/pkg/mcclient/options/llm/llm_sku.go +++ b/pkg/mcclient/options/llm/llm_sku.go @@ -13,7 +13,7 @@ import ( type LLMSkuListOptions struct { options.BaseListOptions - LLMType string `json:"llm_type" choices:"ollama|vllm|sglang|dify|comfyui|openclaw|hermes-agent|llm-router|desktop"` + LLMType string `json:"llm_type" choices:"ollama|vllm|sglang|dify|comfyui|openclaw|hermes-agent|desktop"` Source string `json:"source" help:"filter by source (huggingface, model_scope, local_path)"` Categories string `json:"categories" help:"filter by category (llm, embedding, image, ...)"` } @@ -36,7 +36,7 @@ type LLMSkuCreateOptions struct { MountedModels []string `help:"mounted models, e.g. qwen2:0.5b-dup" json:"mounted_models"` LLM_IMAGE_ID string `json:"llm_image_id"` - LLM_TYPE string `json:"llm_type" choices:"ollama|vllm|sglang|comfyui|hermes-agent|llm-router|desktop"` + LLM_TYPE string `json:"llm_type" choices:"ollama|vllm|sglang|comfyui|hermes-agent|desktop"` // Model source Source string `help:"model source: huggingface, model_scope, local_path" json:"source"` @@ -67,14 +67,6 @@ type LLMSkuCreateOptions struct { HermesModel string `token:"hermes-model" help:"model name for Hermes" json:"-"` HermesApiKey string `token:"hermes-api-key" help:"API key for Hermes custom provider; defaults to EMPTY when omitted" json:"-"` HermesContextLength int `token:"hermes-context-length" help:"Hermes model.context_length" json:"-"` - - RouterMethod string `token:"router-method" help:"LLM router method, e.g. routerdc" json:"-"` - RouterConfigPath string `token:"router-config-path" help:"LLM router config path inside container" json:"-"` - RouterModelDir string `token:"router-model-dir" help:"LLM router model directory inside container" json:"-"` - RouterRoutePath string `token:"router-route-path" help:"LLM router route API path, e.g. /v1/route" json:"-"` - RouterHealthPath string `token:"router-health-path" help:"LLM router health API path, e.g. /health" json:"-"` - RouterCandidateMappingPath string `token:"router-candidate-mapping-path" help:"LLM router candidate mapping path inside container" json:"-"` - RouterEnv []string `token:"router-env" help:"LLM router env in format key=value; repeatable" json:"-"` } func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) { @@ -134,49 +126,10 @@ func (o *LLMSkuCreateOptions) Params() (jsonutils.JSONObject, error) { spec := &api.LLMSpec{HermesAgent: hermesSpec} dict.Set("llm_spec", jsonutils.Marshal(spec)) } - case string(api.LLM_CONTAINER_LLM_ROUTER): - routerSpec, err := newLLMRouterSpecFromArgs(o) - if err != nil { - return nil, err - } - dict.Set("llm_spec", jsonutils.Marshal(&api.LLMSpec{LLMRouter: routerSpec})) } return dict, nil } -func newLLMRouterSpecFromArgs(o *LLMSkuCreateOptions) (*api.LLMSpecLLMRouter, error) { - method := strings.TrimSpace(o.RouterMethod) - if method == "" { - return nil, errors.Error("--router-method is required when llm_type=llm-router") - } - envs, err := newLLMRouterEnvsFromArgs(o.RouterEnv) - if err != nil { - return nil, err - } - return &api.LLMSpecLLMRouter{ - RouterMethod: method, - ConfigPath: strings.TrimSpace(o.RouterConfigPath), - ModelDir: strings.TrimSpace(o.RouterModelDir), - RoutePath: strings.TrimSpace(o.RouterRoutePath), - HealthPath: strings.TrimSpace(o.RouterHealthPath), - CandidateMappingPath: strings.TrimSpace(o.RouterCandidateMappingPath), - CustomizedEnvs: envs, - }, nil -} - -func newLLMRouterEnvsFromArgs(args []string) ([]*api.LLMRouterEnv, error) { - envs := make([]*api.LLMRouterEnv, 0, len(args)) - for _, arg := range args { - key, val, ok := strings.Cut(arg, "=") - key = strings.TrimSpace(key) - if !ok || key == "" { - return nil, errors.Errorf("invalid --router-env %q, expected key=value", arg) - } - envs = append(envs, &api.LLMRouterEnv{Key: key, Value: strings.TrimSpace(val)}) - } - return envs, nil -} - func (o *LLMSkuCreateOptions) buildModelSpec() (*api.InstantModelImportInput, error) { if o.ModelTag == "" { return nil, errors.Error("--model-tag is required for model spec")