diff --git a/pkg/apis/llm/llm_deployment.go b/pkg/apis/llm/llm_deployment.go index 2f9fabceba..4457665242 100644 --- a/pkg/apis/llm/llm_deployment.go +++ b/pkg/apis/llm/llm_deployment.go @@ -134,7 +134,8 @@ type LLMDeploymentCreateInput struct { GpuSelector *GpuSelector `json:"gpu_selector"` // Explicit GPU memory utilization fraction for inference backend. GpuMemoryUtilization *float64 `json:"gpu_memory_utilization,omitempty"` - // Calculate GPU memory utilization from mounted model VRAM and GPU memory. + // Calculate GPU memory utilization from mounted model VRAM and GPU memory + // (default true for supported backends; pass false to disable). AutoGpuMemoryUtilization *bool `json:"auto_gpu_memory_utilization,omitempty"` // Host label selector for scheduling (JSON) WorkerSelector map[string]string `json:"worker_selector"` diff --git a/pkg/llm/drivers/llm_container/sglang.go b/pkg/llm/drivers/llm_container/sglang.go index f0f2f62e13..4539e2d4be 100644 --- a/pkg/llm/drivers/llm_container/sglang.go +++ b/pkg/llm/drivers/llm_container/sglang.go @@ -108,6 +108,9 @@ func (s *sglang) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient if err != nil { return nil, err } + if err := applySGLangToolCallDefaults(ctx, input); err != nil { + return nil, err + } spec, err := s.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec) if err != nil { diff --git a/pkg/llm/drivers/llm_container/sglang_tool_call.go b/pkg/llm/drivers/llm_container/sglang_tool_call.go new file mode 100644 index 0000000000..6e756e8208 --- /dev/null +++ b/pkg/llm/drivers/llm_container/sglang_tool_call.go @@ -0,0 +1,142 @@ +package llm_container + +import ( + "context" + "strings" + + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +const ( + sglangArgToolCallParser = "tool-call-parser" + sglangArgReasoningParser = "reasoning-parser" +) + +type sglangToolCallProfile struct { + parser string + reasoningParser string + match func([]string) bool +} + +var sglangToolCallProfiles = []sglangToolCallProfile{ + {parser: "qwen3_coder", match: candidatesContainAll("qwen", "qwen3-coder")}, + {parser: "deepseekv32", match: candidatesContainAny("deepseek-v3.2", "deepseek_v3.2")}, + {parser: "deepseekv31", match: candidatesContainAny("deepseek-v3.1", "deepseek_v3.1")}, + {parser: "deepseekv3", reasoningParser: "deepseek-r1", match: candidatesContainAny("deepseek-r1")}, + {parser: "deepseekv3", match: candidatesContainAny("deepseek-v3", "deepseek_v3")}, + {parser: "gpt-oss", reasoningParser: "gpt-oss", match: candidatesContainAny("openai/gpt-oss", "gpt-oss-")}, + {parser: "kimi_k2", reasoningParser: "kimi_k2", match: candidatesContainAll("kimi-k2", "instruct")}, + {parser: "glm47", match: candidatesContainAny("glm-4.7", "glm4.7", "glm47")}, + {parser: "glm", reasoningParser: "glm45", match: candidatesContainAny("glm-4.5", "glm4.5", "glm45", "glm-4.6", "glm4.6", "glm46")}, + {parser: "cohere_command4", match: candidatesContainAny("command-a", "command-r", "cohere-command")}, + {parser: "hermes", match: candidatesContainAny("nousresearch/hermes", "hermes-2-", "hermes-3-")}, + {parser: "llama3", match: candidatesContainAny("llama-3.1", "llama3.1", "llama-3.2", "llama3.2", "llama-3.3", "llama3.3")}, + {parser: "mistral", match: candidatesContainAny("mistralai/mistral-7b-instruct-v0.3", "mistral-7b-instruct-v0.3")}, + {parser: "pythonic", match: candidatesContainAny("toolace", "ultravoxai/ultravox-v0_5")}, + {parser: "step3p5", match: candidatesContainAny("step-3.5", "step3.5", "step3p5")}, + {parser: "step3", match: candidatesContainAny("stepfun-ai/step-3", "step-3")}, + {parser: "apertus2509", match: candidatesContainAll("apertus", "2509")}, + {parser: "hunyuan", reasoningParser: "hunyuan", match: candidatesContainAll("hunyuan-a13b", "instruct")}, + {parser: "gigachat3", match: candidatesContainAny("gigachat3", "gigachat-3")}, + {parser: "gemma4", match: candidatesContainAny("gemma-4", "gemma4")}, + {parser: "interns1", match: candidatesContainAny("intern-s1", "interns1")}, + {parser: "lfm2", match: candidatesContainAny("lfm2", "lfm-2")}, + {parser: "mimo", match: candidatesContainAny("mimo")}, + {parser: "minicpm5", match: candidatesContainAny("minicpm5", "minicpm-5")}, + {parser: "minimax-m2", match: candidatesContainAny("minimax-m2")}, + {parser: "poolside_v1", match: candidatesContainAny("poolside-v1", "poolside_v1")}, + {parser: "trinity", match: candidatesContainAny("trinity")}, + {parser: "qwen", reasoningParser: "qwen3", match: candidatesContainAny("qwen/qwen3", "qwen3-")}, + {parser: "qwen", match: candidatesContainAny("qwen/qwen2.5", "qwen2.5-", "qwen/qwq", "qwq-")}, +} + +func applySGLangToolCallDefaults(ctx context.Context, input *api.LLMSkuCreateInput) error { + if input == nil || input.LLMType != string(api.LLM_CONTAINER_SGLANG) { + return nil + } + if input.LLMSpec == nil { + input.LLMSpec = &api.LLMSpec{} + } + if input.LLMSpec.SGLang == nil { + input.LLMSpec.SGLang = &api.LLMSpecSGLang{} + } + + candidates := collectToolCallCandidates(ctx, input) + profile, matched := resolveSGLangToolCallProfile(candidates) + if !matched { + return nil + } + + if !inputHasSGLangRuntimeArg(input, sglangArgToolCallParser) { + appendSGLangCustomizedArgIfMissing(input, sglangArgToolCallParser, profile.parser) + } + if profile.reasoningParser != "" && !inputHasSGLangRuntimeArg(input, sglangArgReasoningParser) { + appendSGLangCustomizedArgIfMissing(input, sglangArgReasoningParser, profile.reasoningParser) + } + return nil +} + +func resolveSGLangToolCallProfile(candidates []string) (sglangToolCallProfile, bool) { + for _, profile := range sglangToolCallProfiles { + if profile.match != nil && profile.match(candidates) { + return profile, true + } + } + return sglangToolCallProfile{}, false +} + +func inputHasSGLangRuntimeArg(input *api.LLMSkuCreateInput, key string) bool { + key = normalizeSGLangRuntimeArgKey(key) + if input == nil || key == "" { + return false + } + if sglangCustomizedArgsHaveKey(input.LLMSpec, key) { + return true + } + for _, item := range input.BackendParameters { + arg, ok, err := parseBackendParameterArg(item) + if err != nil || !ok { + continue + } + if normalizeSGLangRuntimeArgKey(arg.Key) == key { + return true + } + } + return false +} + +func sglangCustomizedArgsHaveKey(spec *api.LLMSpec, key string) bool { + if spec == nil || spec.SGLang == nil { + return false + } + key = normalizeSGLangRuntimeArgKey(key) + for _, arg := range spec.SGLang.CustomizedArgs { + if arg != nil && normalizeSGLangRuntimeArgKey(arg.Key) == key { + return true + } + } + return false +} + +func normalizeSGLangRuntimeArgKey(key string) string { + return strings.TrimPrefix(strings.TrimSpace(key), "--") +} + +func appendSGLangCustomizedArgIfMissing(input *api.LLMSkuCreateInput, key string, value string) { + if input == nil { + return + } + if input.LLMSpec == nil { + input.LLMSpec = &api.LLMSpec{} + } + if input.LLMSpec.SGLang == nil { + input.LLMSpec.SGLang = &api.LLMSpecSGLang{} + } + if sglangCustomizedArgsHaveKey(input.LLMSpec, key) { + return + } + input.LLMSpec.SGLang.CustomizedArgs = append(input.LLMSpec.SGLang.CustomizedArgs, &api.SGLangCustomizedArg{ + Key: key, + Value: value, + }) +} diff --git a/pkg/llm/drivers/llm_container/vllm.go b/pkg/llm/drivers/llm_container/vllm.go index 53349e1a9b..6d7a5f741a 100644 --- a/pkg/llm/drivers/llm_container/vllm.go +++ b/pkg/llm/drivers/llm_container/vllm.go @@ -310,6 +310,9 @@ func (v *vllm) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.T if err != nil { return nil, err } + if err := applyVLLMToolCallDefaults(ctx, input); err != nil { + return nil, err + } // Reuse ValidateLLMCreateSpec; ensure llm_spec.vllm always exists for vLLM SKU. spec, err := v.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec) diff --git a/pkg/llm/drivers/llm_container/vllm_tool_call.go b/pkg/llm/drivers/llm_container/vllm_tool_call.go new file mode 100644 index 0000000000..9a283ede53 --- /dev/null +++ b/pkg/llm/drivers/llm_container/vllm_tool_call.go @@ -0,0 +1,248 @@ +package llm_container + +import ( + "context" + "path/filepath" + "strings" + + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/llm" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/llm/models" +) + +const ( + vllmArgEnableAutoToolChoice = "enable-auto-tool-choice" + vllmArgToolCallParser = "tool-call-parser" + vllmArgReasoningParser = "reasoning-parser" +) + +type vllmToolCallProfile struct { + parser string + reasoningParser string + match func([]string) bool +} + +var vllmToolCallProfiles = []vllmToolCallProfile{ + {parser: "qwen3_xml", match: candidatesContainAll("qwen", "qwen3-coder")}, + {parser: "deepseek_v31", match: candidatesContainAny("deepseek-v3.1", "deepseek_v3.1")}, + {parser: "granite-20b-fc", match: candidatesContainAny("granite-20b-functioncalling")}, + {parser: "granite4", match: candidatesContainAny("granite-4.", "granite4")}, + {parser: "granite", match: candidatesContainAny("granite-3.", "granite3")}, + {parser: "xlam", match: candidatesContainAny("salesforce/xlam", "xlam")}, + {parser: "hermes", match: candidatesContainAny("nousresearch/hermes", "hermes-2-", "hermes-3-", "qwen/qwen2.5", "qwen2.5-", "qwen/qwq", "qwq-")}, + {parser: "mistral", match: candidatesContainAny("mistralai/mistral-7b-instruct-v0.3", "mistral-7b-instruct-v0.3")}, + {parser: "internlm", match: candidatesContainAny("internlm/internlm2_5", "internlm2_5")}, + {parser: "jamba", match: candidatesContainAny("ai21-jamba-1.5", "jamba-1.5")}, + {parser: "llama4_pythonic", match: candidatesContainAny("llama-4", "llama4")}, + {parser: "llama3_json", match: candidatesContainAny("llama-3.1", "llama3.1", "llama-3.2", "llama3.2", "llama-3.3", "llama3.3")}, + {parser: "deepseek_v3", match: candidatesContainAny("deepseek-v3", "deepseek_v3", "deepseek-r1-0528")}, + {parser: "kimi_k2", match: candidatesContainAll("kimi-k2", "instruct")}, + {parser: "openai", match: candidatesContainAny("openai/gpt-oss", "gpt-oss-")}, + {parser: "hunyuan_a13b", reasoningParser: "hunyuan_a13b", match: candidatesContainAll("hunyuan-a13b", "instruct")}, + {parser: "cohere_command3", reasoningParser: "cohere_command3", match: candidatesContainAny("command-a-reasoning")}, + {parser: "longcat", match: candidatesContainAny("meituan-longcat/longcat", "longcat-flash")}, + {parser: "glm47", match: candidatesContainAny("glm-4.7", "glm4.7", "glm47")}, + {parser: "glm45", match: candidatesContainAny("glm-4.5", "glm4.5", "glm45", "glm-4.6", "glm4.6", "glm46")}, + {parser: "functiongemma", match: candidatesContainAny("google/functiongemma", "functiongemma")}, + {parser: "olmo3", match: candidatesContainAny("olmo-3", "olmo3")}, + {parser: "gigachat3", match: candidatesContainAny("gigachat3", "gigachat-3")}, + {parser: "apertus", match: candidatesContainAny("apertus")}, + {parser: "pythonic", match: candidatesContainAny("toolace", "ultravoxai/ultravox-v0_5")}, +} + +func applyVLLMToolCallDefaults(ctx context.Context, input *api.LLMSkuCreateInput) error { + if input == nil || input.LLMType != string(api.LLM_CONTAINER_VLLM) { + return nil + } + if input.LLMSpec == nil { + input.LLMSpec = &api.LLMSpec{} + } + if input.LLMSpec.Vllm == nil { + input.LLMSpec.Vllm = &api.LLMSpecVllm{} + } + + candidates := collectToolCallCandidates(ctx, input) + profile, matched := resolveVLLMToolCallProfile(candidates) + explicitAutoChoice := inputHasVLLMRuntimeArg(input, vllmArgEnableAutoToolChoice) + explicitParser := inputHasVLLMRuntimeArg(input, vllmArgToolCallParser) + + if !matched { + if explicitAutoChoice && !explicitParser { + return errors.Wrap(httperrors.ErrInputParameter, "enable-auto-tool-choice requires tool-call-parser for unknown vLLM model") + } + return nil + } + + appendVLLMCustomizedArgIfMissing(input, vllmArgEnableAutoToolChoice, "") + if !explicitParser { + appendVLLMCustomizedArgIfMissing(input, vllmArgToolCallParser, profile.parser) + } + if profile.reasoningParser != "" && !inputHasVLLMRuntimeArg(input, vllmArgReasoningParser) { + appendVLLMCustomizedArgIfMissing(input, vllmArgReasoningParser, profile.reasoningParser) + } + return nil +} + +func collectToolCallCandidates(ctx context.Context, input *api.LLMSkuCreateInput) []string { + if input == nil { + return nil + } + out := make([]string, 0, 8+len(input.MountedModels)*3) + add := func(s string) { + s = normalizeToolCallCandidate(s) + if s != "" { + out = append(out, s) + } + } + add(input.HuggingfaceRepoId) + add(input.ModelScopeModelId) + add(input.LocalPath) + if input.LocalPath != "" { + add(filepath.Base(input.LocalPath)) + } + if input.ModelSpec != nil { + add(input.ModelSpec.RepoId) + add(input.ModelSpec.ModelName) + add(input.ModelSpec.ModelTag) + add(input.ModelSpec.Revision) + } + if input.LLMSpec != nil && input.LLMSpec.Vllm != nil { + add(input.LLMSpec.Vllm.PreferredModel) + if input.LLMSpec.Vllm.PreferredModel != "" { + add(filepath.Base(input.LLMSpec.Vllm.PreferredModel)) + } + } + for _, id := range input.MountedModels { + add(id) + if ctx == nil || strings.TrimSpace(id) == "" { + continue + } + obj, err := models.GetInstantModelManager().FetchById(id) + if err != nil { + continue + } + im, ok := obj.(*models.SInstantModel) + if !ok || im == nil { + continue + } + add(im.ModelName) + add(im.ModelId) + add(im.ModelTag) + } + return out +} + +func normalizeToolCallCandidate(s string) string { + return strings.ToLower(strings.TrimSpace(s)) +} + +func resolveVLLMToolCallProfile(candidates []string) (vllmToolCallProfile, bool) { + for _, profile := range vllmToolCallProfiles { + if profile.match != nil && profile.match(candidates) { + return profile, true + } + } + return vllmToolCallProfile{}, false +} + +func candidatesContainAny(needles ...string) func([]string) bool { + normalized := normalizeNeedles(needles) + return func(candidates []string) bool { + for _, candidate := range candidates { + for _, needle := range normalized { + if strings.Contains(candidate, needle) { + return true + } + } + } + return false + } +} + +func candidatesContainAll(needles ...string) func([]string) bool { + normalized := normalizeNeedles(needles) + return func(candidates []string) bool { + for _, candidate := range candidates { + matchedAll := true + for _, needle := range normalized { + if !strings.Contains(candidate, needle) { + matchedAll = false + break + } + } + if matchedAll { + return true + } + } + return false + } +} + +func normalizeNeedles(needles []string) []string { + out := make([]string, 0, len(needles)) + for _, needle := range needles { + needle = normalizeToolCallCandidate(needle) + if needle != "" { + out = append(out, needle) + } + } + return out +} + +func inputHasVLLMRuntimeArg(input *api.LLMSkuCreateInput, key string) bool { + key = normalizeVLLMRuntimeArgKey(key) + if input == nil || key == "" { + return false + } + if vllmCustomizedArgsHaveKey(input.LLMSpec, key) { + return true + } + for _, item := range input.BackendParameters { + arg, ok, err := parseBackendParameterArg(item) + if err != nil || !ok { + continue + } + if normalizeVLLMRuntimeArgKey(arg.Key) == key { + return true + } + } + return false +} + +func vllmCustomizedArgsHaveKey(spec *api.LLMSpec, key string) bool { + if spec == nil || spec.Vllm == nil { + return false + } + key = normalizeVLLMRuntimeArgKey(key) + for _, arg := range spec.Vllm.CustomizedArgs { + if arg != nil && normalizeVLLMRuntimeArgKey(arg.Key) == key { + return true + } + } + return false +} + +func normalizeVLLMRuntimeArgKey(key string) string { + return strings.TrimPrefix(strings.TrimSpace(key), "--") +} + +func appendVLLMCustomizedArgIfMissing(input *api.LLMSkuCreateInput, key string, value string) { + if input == nil { + return + } + if input.LLMSpec == nil { + input.LLMSpec = &api.LLMSpec{} + } + if input.LLMSpec.Vllm == nil { + input.LLMSpec.Vllm = &api.LLMSpecVllm{} + } + if vllmCustomizedArgsHaveKey(input.LLMSpec, key) { + return + } + input.LLMSpec.Vllm.CustomizedArgs = append(input.LLMSpec.Vllm.CustomizedArgs, &api.VllmCustomizedArg{ + Key: key, + Value: value, + }) +} diff --git a/pkg/llm/models/llm_deployment.go b/pkg/llm/models/llm_deployment.go index 7ea0e197a2..8fe8824aa0 100644 --- a/pkg/llm/models/llm_deployment.go +++ b/pkg/llm/models/llm_deployment.go @@ -145,6 +145,7 @@ func (man *SLLMDeploymentManager) ValidateCreateData( if err := validateDeploymentGpuMemoryUtilization(input.GpuMemoryUtilization, input.AutoGpuMemoryUtilization, lSku.LLMType); err != nil { return input, err } + defaultDeploymentAutoGpuMemoryUtilization(input, lSku.LLMType) if err := ValidateDeploymentDevices(lSku.LLMType, lSku); err != nil { return input, err } @@ -176,6 +177,7 @@ func (man *SLLMDeploymentManager) ValidateCreateData( if err := validateDeploymentGpuMemoryUtilization(input.GpuMemoryUtilization, input.AutoGpuMemoryUtilization, input.SkuSpec.LLMType); err != nil { return input, err } + defaultDeploymentAutoGpuMemoryUtilization(input, input.SkuSpec.LLMType) if err := ValidateDeploymentDevices(input.SkuSpec.LLMType, skuFromLLMSkuCreateInput(input.SkuSpec)); err != nil { return input, err } diff --git a/pkg/llm/models/llm_deployment_gpu.go b/pkg/llm/models/llm_deployment_gpu.go index f4b51f8b0d..065f3c22ef 100644 --- a/pkg/llm/models/llm_deployment_gpu.go +++ b/pkg/llm/models/llm_deployment_gpu.go @@ -2,6 +2,7 @@ package models import ( "context" + "encoding/json" "math" "strconv" "strings" @@ -12,6 +13,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/llm/utils/vram" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" @@ -21,14 +23,38 @@ const ( autoGpuMemoryUtilizationSafetyFactor = 1.10 autoGpuMemoryUtilizationMin = 0.05 autoGpuMemoryUtilizationMax = 0.95 - instantModelDynamicVramRatio = 0.15 - instantModelFixedVramMB = 500 + + // vLLM derives an unset max-model-len from model config. When auto GPU + // memory utilization is enabled, inject a conservative cap so the heuristic + // VRAM estimate is not paired with an unexpectedly large context. + autoGpuMemoryUtilizationDefaultContextTokens = int64(8192) + + sglangAutoGpuMemoryMetadataReserveMB = 512 ) func boolPtrValue(v *bool) bool { return v != nil && *v } +func deploymentAutoGpuMemoryUtilizationEnabled(auto *bool, llmType string) bool { + if auto != nil { + return *auto + } + _, ok := gpuMemoryUtilizationRuntimeArgKey(llmType) + return ok +} + +func defaultDeploymentAutoGpuMemoryUtilization(input *api.LLMDeploymentCreateInput, llmType string) { + if input == nil || input.GpuMemoryUtilization != nil || input.AutoGpuMemoryUtilization != nil { + return + } + if !deploymentAutoGpuMemoryUtilizationEnabled(nil, llmType) { + return + } + enabled := true + input.AutoGpuMemoryUtilization = &enabled +} + func validateDeploymentGpuMemoryUtilization(util *float64, auto *bool, llmType string) error { needsRuntimeArg := util != nil || boolPtrValue(auto) if util != nil { @@ -69,17 +95,183 @@ func calculateAutoGpuMemoryUtilization(requiredVramMB int64, gpuMemoryMB int64, ) } -func calculateAutoGpuMemoryUtilizationForModelSize(modelSizeMB int64, gpuMemoryMB int64, tensorParallelSize int) (float64, error) { - if modelSizeMB <= 0 { - return 0, errors.Wrap(httperrors.ErrInputParameter, "mounted model size is empty") +func instantModelEstimatedVramRequirementMB(model *SInstantModel) int64 { + if model == nil { + return 0 } - tp := normalizeTensorParallelSize(tensorParallelSize) - // Tensor parallel shards model weights, but runtime/KV/framework overhead is - // still charged per GPU here to avoid underestimating heterogeneous multi-GPU deployments. - perGPURequiredMB := float64(modelSizeMB)/float64(tp) + - float64(modelSizeMB)*instantModelDynamicVramRatio + - instantModelFixedVramMB - return calculateAutoGpuMemoryUtilizationFromPerGPURequired(perGPURequiredMB, gpuMemoryMB) + return int64(vram.EstimateClaimMb(model.WeightSizeBytes, model.LlmType)) +} + +func runtimeArgKeyIn(key string, keys []string) bool { + key = normalizeRuntimeArgKey(key) + for _, candidate := range keys { + if key == candidate { + return true + } + } + return false +} + +func runtimeHasExplicitTokenLimit(sku *SLLMSku) bool { + if sku == nil { + return false + } + switch api.LLMContainerType(sku.LLMType) { + case api.LLM_CONTAINER_VLLM: + return runtimeHasExplicitArg(sku, []string{"max-model-len"}) + default: + return false + } +} + +func runtimeHasExplicitArg(sku *SLLMSku, keys []string) bool { + if sku == nil { + return false + } + if backendParametersContainRuntimeArg(sku.BackendParameters, keys) { + return true + } + switch api.LLMContainerType(sku.LLMType) { + case api.LLM_CONTAINER_VLLM: + if sku.LLMSpec == nil || sku.LLMSpec.Vllm == nil { + return false + } + for _, arg := range sku.LLMSpec.Vllm.CustomizedArgs { + if arg != nil && runtimeArgKeyIn(arg.Key, keys) { + return true + } + } + case api.LLM_CONTAINER_SGLANG: + if sku.LLMSpec == nil || sku.LLMSpec.SGLang == nil { + return false + } + for _, arg := range sku.LLMSpec.SGLang.CustomizedArgs { + if arg != nil && runtimeArgKeyIn(arg.Key, keys) { + return true + } + } + } + return false +} + +func backendParametersContainRuntimeArg(raw string, keys []string) bool { + items := backendParameterItems(raw) + for i := range items { + argKey, _, ok := splitBackendParameterFlag(items[i]) + if ok && runtimeArgKeyIn(argKey, keys) { + return true + } + } + return false +} + +func tokenLimitFromBackendParameters(raw string, keys []string) (int64, bool) { + items := backendParameterItems(raw) + var tokenLimit int64 + found := false + for i := range items { + key, value, ok := splitBackendParameterFlag(items[i]) + if !ok || !runtimeArgKeyIn(key, keys) { + continue + } + if value == "" && i+1 < len(items) { + next := strings.TrimSpace(items[i+1]) + if next == "-1" || !strings.HasPrefix(next, "-") { + value = next + } + } + if val, ok := parseTokenLimitValue(value); ok { + tokenLimit = val + found = true + } + } + return tokenLimit, found +} + +func backendParameterItems(raw string) []string { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil + } + items := []string{} + if err := json.Unmarshal([]byte(raw), &items); err != nil { + return []string{raw} + } + return items +} + +func splitBackendParameterFlag(item string) (string, string, bool) { + item = strings.TrimSpace(item) + if item == "" || strings.HasPrefix(item, "-") && !strings.HasPrefix(item, "--") { + return "", "", false + } + item = strings.TrimSpace(strings.TrimPrefix(item, "--")) + if item == "" { + return "", "", false + } + key := item + value := "" + if idx := strings.Index(item, "="); idx >= 0 { + key = strings.TrimSpace(item[:idx]) + value = item[idx+1:] + } else if fields := strings.Fields(item); len(fields) > 1 { + key = fields[0] + value = strings.TrimSpace(item[len(key):]) + } + return key, trimArgumentValue(value), true +} + +func normalizeRuntimeArgKey(key string) string { + return strings.TrimPrefix(strings.TrimSpace(key), "--") +} + +func trimArgumentValue(value string) string { + value = strings.TrimSpace(value) + return strings.Trim(value, `"'`) +} + +func parseTokenLimitValue(value string) (int64, bool) { + value = trimArgumentValue(value) + if value == "" { + return 0, false + } + if strings.EqualFold(value, "auto") { + return 0, true + } + if val, err := strconv.ParseInt(value, 10, 64); err == nil { + if val <= 0 { + return 0, true + } + return val, true + } + + multiplier := float64(1) + number := value + switch value[len(value)-1] { + case 'k': + multiplier = 1000 + number = value[:len(value)-1] + case 'K': + multiplier = 1024 + number = value[:len(value)-1] + case 'm': + multiplier = 1000 * 1000 + number = value[:len(value)-1] + case 'M': + multiplier = 1024 * 1024 + number = value[:len(value)-1] + case 'g': + multiplier = 1000 * 1000 * 1000 + number = value[:len(value)-1] + case 'G': + multiplier = 1024 * 1024 * 1024 + number = value[:len(value)-1] + } + parsed, err := strconv.ParseFloat(strings.TrimSpace(number), 64) + if err != nil || parsed <= 0 { + return 0, false + } + return int64(math.Ceil(parsed * multiplier)), true } func calculateAutoGpuMemoryUtilizationFromPerGPURequired(perGPURequiredMB float64, gpuMemoryMB int64) (float64, error) { @@ -97,6 +289,125 @@ func calculateAutoGpuMemoryUtilizationFromPerGPURequired(perGPURequiredMB float6 return math.Ceil(raw*100) / 100, nil } +func calculateDeploymentAutoGpuMemoryUtilization(sku *SLLMSku, requiredVramMB int64, gpuMemoryMB int64, tensorParallelSize int) (float64, error) { + if sku != nil && api.LLMContainerType(sku.LLMType) == api.LLM_CONTAINER_SGLANG { + return calculateSGLangAutoGpuMemoryUtilization(sku, requiredVramMB, gpuMemoryMB, tensorParallelSize) + } + return calculateAutoGpuMemoryUtilization(requiredVramMB, gpuMemoryMB, tensorParallelSize) +} + +func calculateSGLangAutoGpuMemoryUtilization(sku *SLLMSku, requiredVramMB int64, gpuMemoryMB int64, tensorParallelSize int) (float64, error) { + modelUtilization, err := calculateAutoGpuMemoryUtilization(requiredVramMB, gpuMemoryMB, tensorParallelSize) + if err != nil { + return 0, err + } + runtimeUtilization, err := calculateSGLangMemFractionStatic(sku, gpuMemoryMB, tensorParallelSize) + if err != nil { + return 0, err + } + if runtimeUtilization > modelUtilization { + return runtimeUtilization, nil + } + return modelUtilization, nil +} + +func calculateSGLangMemFractionStatic(sku *SLLMSku, gpuMemoryMB int64, tensorParallelSize int) (float64, error) { + if gpuMemoryMB <= 0 { + return 0, errors.Wrap(httperrors.ErrInputParameter, "gpu memory_mb is empty") + } + tpSize := int64(normalizeTensorParallelSize(tensorParallelSize)) + if val, ok := sglangRuntimeIntArg(sku, []string{"tp-size", "tensor-parallel-size"}); ok && val > 0 { + tpSize = val + } + ppSize := int64(1) + if val, ok := sglangRuntimeIntArg(sku, []string{"pp-size", "pipeline-parallel-size"}); ok && val > 0 { + ppSize = val + } + chunkedPrefillSize, cudaGraphMaxBS := sglangRuntimeReserveDefaults(gpuMemoryMB, tpSize) + if val, ok := sglangRuntimeIntArg(sku, []string{"chunked-prefill-size"}); ok { + chunkedPrefillSize = val + } + if val, ok := sglangRuntimeIntArg(sku, []string{"cuda-graph-max-bs", "cuda-graph-max-bs-decode"}); ok && val > 0 { + cudaGraphMaxBS = val + } + + reservedMemMB := float64(sglangAutoGpuMemoryMetadataReserveMB) + if chunkedPrefillSize > 0 { + reservedMemMB += float64(maxInt64(chunkedPrefillSize, 2048)) * 1.5 + } else if maxPrefillTokens, ok := sglangRuntimeIntArg(sku, []string{"max-prefill-tokens"}); ok && maxPrefillTokens > 0 { + reservedMemMB += float64(maxInt64(maxPrefillTokens, 2048)) * 1.5 + } else { + reservedMemMB += 2048 * 1.5 + } + reservedMemMB += float64(cudaGraphMaxBS) * 2 + reservedMemMB += float64(tpSize*ppSize) / 8 * 1024 + if gpuMemoryMB > 60*1024 && reservedMemMB < 10*1024 { + reservedMemMB = 10 * 1024 + } + + utilization := (float64(gpuMemoryMB) - reservedMemMB) / float64(gpuMemoryMB) + if utilization > autoGpuMemoryUtilizationMax { + utilization = autoGpuMemoryUtilizationMax + } + if utilization < autoGpuMemoryUtilizationMin { + utilization = autoGpuMemoryUtilizationMin + } + return math.Round(utilization*1000) / 1000, nil +} + +func sglangRuntimeReserveDefaults(gpuMemoryMB int64, tpSize int64) (int64, int64) { + if gpuMemoryMB < 20*1024 { + return 2048, 8 + } + if gpuMemoryMB < 35*1024 { + if tpSize < 4 { + return 2048, 24 + } + return 2048, 80 + } + if gpuMemoryMB < 60*1024 { + if tpSize < 4 { + return 4096, 32 + } + return 4096, 160 + } + if gpuMemoryMB < 160*1024 { + if tpSize < 4 { + return 8192, 256 + } + return 8192, 512 + } + return 16384, 512 +} + +func sglangRuntimeIntArg(sku *SLLMSku, keys []string) (int64, bool) { + if sku == nil { + return 0, false + } + if val, ok := tokenLimitFromBackendParameters(sku.BackendParameters, keys); ok { + return val, true + } + if sku.LLMSpec == nil || sku.LLMSpec.SGLang == nil { + return 0, false + } + for _, arg := range sku.LLMSpec.SGLang.CustomizedArgs { + if arg == nil || !runtimeArgKeyIn(arg.Key, keys) { + continue + } + if val, ok := parseTokenLimitValue(arg.Value); ok { + return val, true + } + } + return 0, false +} + +func maxInt64(a int64, b int64) int64 { + if a > b { + return a + } + return b +} + func normalizeTensorParallelSize(tensorParallelSize int) int { if tensorParallelSize <= 0 { return 1 @@ -135,6 +446,26 @@ func buildGpuMemoryUtilizationLLMSpec(llmType string, utilization float64) (*api } } +func buildAutoGpuMemoryUtilizationLLMSpec(sku *SLLMSku, utilization float64) (*api.LLMSpec, error) { + if sku == nil { + return nil, nil + } + spec, err := buildGpuMemoryUtilizationLLMSpec(sku.LLMType, utilization) + if err != nil { + return nil, err + } + switch api.LLMContainerType(sku.LLMType) { + case api.LLM_CONTAINER_VLLM: + if spec != nil && spec.Vllm != nil && !runtimeHasExplicitTokenLimit(sku) { + spec.Vllm.CustomizedArgs = append(spec.Vllm.CustomizedArgs, &api.VllmCustomizedArg{ + Key: "max-model-len", + Value: strconv.FormatInt(autoGpuMemoryUtilizationDefaultContextTokens, 10), + }) + } + } + return spec, nil +} + func formatGpuMemoryUtilization(v float64) string { return strconv.FormatFloat(v, 'f', -1, 64) } @@ -146,14 +477,14 @@ func BuildDeploymentResolvedGpuMemoryLLMSpec(ctx context.Context, userCred mccli if deploy.GpuMemoryUtilization != nil { return buildDeploymentGpuMemoryLLMSpec(deploy, sku) } - if !boolPtrValue(deploy.AutoGpuMemoryUtilization) { + if !deploymentAutoGpuMemoryUtilizationEnabled(deploy.AutoGpuMemoryUtilization, sku.LLMType) { return nil, nil } tensorParallelSize := 1 if sku.Devices != nil && len(*sku.Devices) > 0 { tensorParallelSize = len(*sku.Devices) } - modelSizeMB, err := maxMountedModelSizeMB(sku) + requiredVramMB, err := maxMountedModelVramRequirementMB(sku) if err != nil { return nil, err } @@ -161,33 +492,33 @@ func BuildDeploymentResolvedGpuMemoryLLMSpec(ctx context.Context, userCred mccli if err != nil { return nil, err } - utilization, err := calculateAutoGpuMemoryUtilizationForModelSize(modelSizeMB, gpuMemoryMB, tensorParallelSize) + utilization, err := calculateDeploymentAutoGpuMemoryUtilization(sku, requiredVramMB, gpuMemoryMB, tensorParallelSize) if err != nil { return nil, err } - return buildGpuMemoryUtilizationLLMSpec(sku.LLMType, utilization) + return buildAutoGpuMemoryUtilizationLLMSpec(sku, utilization) } -func maxMountedModelSizeMB(sku *SLLMSku) (int64, error) { +func maxMountedModelVramRequirementMB(sku *SLLMSku) (int64, error) { modelIds := sku.GetMountedModels() if len(modelIds) == 0 { return 0, httperrors.NewInputParameterError("auto_gpu_memory_utilization requires mounted models: configure mounted_models on the LLM SKU") } - var maxSize int64 + var maxRequiredVramMB int64 for _, modelId := range modelIds { obj, err := GetInstantModelManager().FetchById(modelId) if err != nil { return 0, errors.Wrapf(err, "fetch InstantModel %s", modelId) } - sizeMB := int64(obj.(*SInstantModel).GetActualSizeMb()) - if sizeMB > maxSize { - maxSize = sizeMB + requiredVramMB := instantModelEstimatedVramRequirementMB(obj.(*SInstantModel)) + if requiredVramMB > maxRequiredVramMB { + maxRequiredVramMB = requiredVramMB } } - if maxSize <= 0 { - return 0, errors.Wrap(httperrors.ErrInputParameter, "mounted model size is empty") + if maxRequiredVramMB <= 0 { + return 0, errors.Wrap(httperrors.ErrInputParameter, "mounted model vram requirement is empty") } - return maxSize, nil + return maxRequiredVramMB, nil } func minGpuMemoryMB(ctx context.Context, userCred mcclient.TokenCredential, devices *api.Devices) (int64, error) { diff --git a/pkg/mcclient/options/llm/llm_deployment.go b/pkg/mcclient/options/llm/llm_deployment.go index e454905253..4cb6c7f940 100644 --- a/pkg/mcclient/options/llm/llm_deployment.go +++ b/pkg/mcclient/options/llm/llm_deployment.go @@ -83,7 +83,7 @@ type LLMDeploymentCreateOptions struct { DistributedInference *bool `help:"enable distributed inference" json:"distributed_inference"` GpuMemoryUtilization *float64 `token:"gpu-memory-utilization" help:"GPU memory utilization fraction for backend runtime (0-1)" json:"gpu_memory_utilization"` GpuUtilization *float64 `token:"gpu-utilization" help:"Alias of --gpu-memory-utilization" json:"-"` - AutoGpuMemoryUtilization *bool `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory" json:"auto_gpu_memory_utilization"` + AutoGpuMemoryUtilization *bool `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory (default true for supported backends; use false to disable)" json:"auto_gpu_memory_utilization"` RestartOnError *bool `help:"restart on error" json:"restart_on_error"` AccessPolicy string `help:"access policy" choices:"public|authed|allowed_users" json:"access_policy"` AutoRegisterAiproxy *bool `help:"auto register running replicas with aiproxy (default true; use --auto-register-aiproxy=false to disable)" json:"auto_register_aiproxy"` @@ -233,7 +233,7 @@ type LLMDeploymentUpdateOptions struct { PlacementStrategy string `help:"placement strategy" json:"placement_strategy"` GpuMemoryUtilization *float64 `token:"gpu-memory-utilization" help:"GPU memory utilization fraction for backend runtime (0-1)" json:"gpu_memory_utilization"` GpuUtilization *float64 `token:"gpu-utilization" help:"Alias of --gpu-memory-utilization" json:"-"` - AutoGpuMemoryUtilization *bool `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory" json:"auto_gpu_memory_utilization"` + AutoGpuMemoryUtilization *bool `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory (default true for supported backends; use false to disable)" json:"auto_gpu_memory_utilization"` AccessPolicy string `help:"access policy" json:"access_policy"` AutoRegisterAiproxy *bool `help:"auto register running replicas with aiproxy (default true; use --auto-register-aiproxy=false to disable)" json:"auto_register_aiproxy"` AiproxyModelPrefix *string `help:"deprecated; no longer affects aiproxy client model alias" json:"aiproxy_model_prefix"`