Automated cherry pick of #25074: feat(llm): tool call defaults & auto gpu ultilization (#25075)

* feat(llm): upgrade auto-gpu-ultilization

* feat(llm): add vllm tool call defaults

* feat(llm): add sglang tool call defaults

---------

Co-authored-by: cwz <cwz_eikoh@163.com>

pkg/apis/llm/llm_deployment.go

@@ -134,7 +134,8 @@ type LLMDeploymentCreateInput struct {
 	GpuSelector *GpuSelector `json:"gpu_selector"`
 	// Explicit GPU memory utilization fraction for inference backend.
 	GpuMemoryUtilization *float64 `json:"gpu_memory_utilization,omitempty"`
-	// Calculate GPU memory utilization from mounted model VRAM and GPU memory.
+	// Calculate GPU memory utilization from mounted model VRAM and GPU memory
+	// (default true for supported backends; pass false to disable).
 	AutoGpuMemoryUtilization *bool `json:"auto_gpu_memory_utilization,omitempty"`
 	// Host label selector for scheduling (JSON)
 	WorkerSelector map[string]string `json:"worker_selector"`

pkg/llm/drivers/llm_container/sglang.go

@@ -108,6 +108,9 @@ func (s *sglang) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient
 	if err != nil {
 		return nil, err
 	}
+	if err := applySGLangToolCallDefaults(ctx, input); err != nil {
+		return nil, err
+	}
 
 	spec, err := s.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec)
 	if err != nil {

pkg/llm/drivers/llm_container/sglang_tool_call.go

@@ -0,0 +1,142 @@
+package llm_container
+
+import (
+	"context"
+	"strings"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+)
+
+const (
+	sglangArgToolCallParser  = "tool-call-parser"
+	sglangArgReasoningParser = "reasoning-parser"
+)
+
+type sglangToolCallProfile struct {
+	parser          string
+	reasoningParser string
+	match           func([]string) bool
+}
+
+var sglangToolCallProfiles = []sglangToolCallProfile{
+	{parser: "qwen3_coder", match: candidatesContainAll("qwen", "qwen3-coder")},
+	{parser: "deepseekv32", match: candidatesContainAny("deepseek-v3.2", "deepseek_v3.2")},
+	{parser: "deepseekv31", match: candidatesContainAny("deepseek-v3.1", "deepseek_v3.1")},
+	{parser: "deepseekv3", reasoningParser: "deepseek-r1", match: candidatesContainAny("deepseek-r1")},
+	{parser: "deepseekv3", match: candidatesContainAny("deepseek-v3", "deepseek_v3")},
+	{parser: "gpt-oss", reasoningParser: "gpt-oss", match: candidatesContainAny("openai/gpt-oss", "gpt-oss-")},
+	{parser: "kimi_k2", reasoningParser: "kimi_k2", match: candidatesContainAll("kimi-k2", "instruct")},
+	{parser: "glm47", match: candidatesContainAny("glm-4.7", "glm4.7", "glm47")},
+	{parser: "glm", reasoningParser: "glm45", match: candidatesContainAny("glm-4.5", "glm4.5", "glm45", "glm-4.6", "glm4.6", "glm46")},
+	{parser: "cohere_command4", match: candidatesContainAny("command-a", "command-r", "cohere-command")},
+	{parser: "hermes", match: candidatesContainAny("nousresearch/hermes", "hermes-2-", "hermes-3-")},
+	{parser: "llama3", match: candidatesContainAny("llama-3.1", "llama3.1", "llama-3.2", "llama3.2", "llama-3.3", "llama3.3")},
+	{parser: "mistral", match: candidatesContainAny("mistralai/mistral-7b-instruct-v0.3", "mistral-7b-instruct-v0.3")},
+	{parser: "pythonic", match: candidatesContainAny("toolace", "ultravoxai/ultravox-v0_5")},
+	{parser: "step3p5", match: candidatesContainAny("step-3.5", "step3.5", "step3p5")},
+	{parser: "step3", match: candidatesContainAny("stepfun-ai/step-3", "step-3")},
+	{parser: "apertus2509", match: candidatesContainAll("apertus", "2509")},
+	{parser: "hunyuan", reasoningParser: "hunyuan", match: candidatesContainAll("hunyuan-a13b", "instruct")},
+	{parser: "gigachat3", match: candidatesContainAny("gigachat3", "gigachat-3")},
+	{parser: "gemma4", match: candidatesContainAny("gemma-4", "gemma4")},
+	{parser: "interns1", match: candidatesContainAny("intern-s1", "interns1")},
+	{parser: "lfm2", match: candidatesContainAny("lfm2", "lfm-2")},
+	{parser: "mimo", match: candidatesContainAny("mimo")},
+	{parser: "minicpm5", match: candidatesContainAny("minicpm5", "minicpm-5")},
+	{parser: "minimax-m2", match: candidatesContainAny("minimax-m2")},
+	{parser: "poolside_v1", match: candidatesContainAny("poolside-v1", "poolside_v1")},
+	{parser: "trinity", match: candidatesContainAny("trinity")},
+	{parser: "qwen", reasoningParser: "qwen3", match: candidatesContainAny("qwen/qwen3", "qwen3-")},
+	{parser: "qwen", match: candidatesContainAny("qwen/qwen2.5", "qwen2.5-", "qwen/qwq", "qwq-")},
+}
+
+func applySGLangToolCallDefaults(ctx context.Context, input *api.LLMSkuCreateInput) error {
+	if input == nil || input.LLMType != string(api.LLM_CONTAINER_SGLANG) {
+		return nil
+	}
+	if input.LLMSpec == nil {
+		input.LLMSpec = &api.LLMSpec{}
+	}
+	if input.LLMSpec.SGLang == nil {
+		input.LLMSpec.SGLang = &api.LLMSpecSGLang{}
+	}
+
+	candidates := collectToolCallCandidates(ctx, input)
+	profile, matched := resolveSGLangToolCallProfile(candidates)
+	if !matched {
+		return nil
+	}
+
+	if !inputHasSGLangRuntimeArg(input, sglangArgToolCallParser) {
+		appendSGLangCustomizedArgIfMissing(input, sglangArgToolCallParser, profile.parser)
+	}
+	if profile.reasoningParser != "" && !inputHasSGLangRuntimeArg(input, sglangArgReasoningParser) {
+		appendSGLangCustomizedArgIfMissing(input, sglangArgReasoningParser, profile.reasoningParser)
+	}
+	return nil
+}
+
+func resolveSGLangToolCallProfile(candidates []string) (sglangToolCallProfile, bool) {
+	for _, profile := range sglangToolCallProfiles {
+		if profile.match != nil && profile.match(candidates) {
+			return profile, true
+		}
+	}
+	return sglangToolCallProfile{}, false
+}
+
+func inputHasSGLangRuntimeArg(input *api.LLMSkuCreateInput, key string) bool {
+	key = normalizeSGLangRuntimeArgKey(key)
+	if input == nil || key == "" {
+		return false
+	}
+	if sglangCustomizedArgsHaveKey(input.LLMSpec, key) {
+		return true
+	}
+	for _, item := range input.BackendParameters {
+		arg, ok, err := parseBackendParameterArg(item)
+		if err != nil || !ok {
+			continue
+		}
+		if normalizeSGLangRuntimeArgKey(arg.Key) == key {
+			return true
+		}
+	}
+	return false
+}
+
+func sglangCustomizedArgsHaveKey(spec *api.LLMSpec, key string) bool {
+	if spec == nil || spec.SGLang == nil {
+		return false
+	}
+	key = normalizeSGLangRuntimeArgKey(key)
+	for _, arg := range spec.SGLang.CustomizedArgs {
+		if arg != nil && normalizeSGLangRuntimeArgKey(arg.Key) == key {
+			return true
+		}
+	}
+	return false
+}
+
+func normalizeSGLangRuntimeArgKey(key string) string {
+	return strings.TrimPrefix(strings.TrimSpace(key), "--")
+}
+
+func appendSGLangCustomizedArgIfMissing(input *api.LLMSkuCreateInput, key string, value string) {
+	if input == nil {
+		return
+	}
+	if input.LLMSpec == nil {
+		input.LLMSpec = &api.LLMSpec{}
+	}
+	if input.LLMSpec.SGLang == nil {
+		input.LLMSpec.SGLang = &api.LLMSpecSGLang{}
+	}
+	if sglangCustomizedArgsHaveKey(input.LLMSpec, key) {
+		return
+	}
+	input.LLMSpec.SGLang.CustomizedArgs = append(input.LLMSpec.SGLang.CustomizedArgs, &api.SGLangCustomizedArg{
+		Key:   key,
+		Value: value,
+	})
+}

pkg/llm/drivers/llm_container/vllm.go

@@ -310,6 +310,9 @@ func (v *vllm) ValidateLLMSkuCreateData(ctx context.Context, userCred mcclient.T
 	if err != nil {
 		return nil, err
 	}
+	if err := applyVLLMToolCallDefaults(ctx, input); err != nil {
+		return nil, err
+	}
 
 	// Reuse ValidateLLMCreateSpec; ensure llm_spec.vllm always exists for vLLM SKU.
 	spec, err := v.ValidateLLMCreateSpec(ctx, userCred, nil, input.LLMSpec)

pkg/llm/drivers/llm_container/vllm_tool_call.go

@@ -0,0 +1,248 @@
+package llm_container
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/llm"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/llm/models"
+)
+
+const (
+	vllmArgEnableAutoToolChoice = "enable-auto-tool-choice"
+	vllmArgToolCallParser       = "tool-call-parser"
+	vllmArgReasoningParser      = "reasoning-parser"
+)
+
+type vllmToolCallProfile struct {
+	parser          string
+	reasoningParser string
+	match           func([]string) bool
+}
+
+var vllmToolCallProfiles = []vllmToolCallProfile{
+	{parser: "qwen3_xml", match: candidatesContainAll("qwen", "qwen3-coder")},
+	{parser: "deepseek_v31", match: candidatesContainAny("deepseek-v3.1", "deepseek_v3.1")},
+	{parser: "granite-20b-fc", match: candidatesContainAny("granite-20b-functioncalling")},
+	{parser: "granite4", match: candidatesContainAny("granite-4.", "granite4")},
+	{parser: "granite", match: candidatesContainAny("granite-3.", "granite3")},
+	{parser: "xlam", match: candidatesContainAny("salesforce/xlam", "xlam")},
+	{parser: "hermes", match: candidatesContainAny("nousresearch/hermes", "hermes-2-", "hermes-3-", "qwen/qwen2.5", "qwen2.5-", "qwen/qwq", "qwq-")},
+	{parser: "mistral", match: candidatesContainAny("mistralai/mistral-7b-instruct-v0.3", "mistral-7b-instruct-v0.3")},
+	{parser: "internlm", match: candidatesContainAny("internlm/internlm2_5", "internlm2_5")},
+	{parser: "jamba", match: candidatesContainAny("ai21-jamba-1.5", "jamba-1.5")},
+	{parser: "llama4_pythonic", match: candidatesContainAny("llama-4", "llama4")},
+	{parser: "llama3_json", match: candidatesContainAny("llama-3.1", "llama3.1", "llama-3.2", "llama3.2", "llama-3.3", "llama3.3")},
+	{parser: "deepseek_v3", match: candidatesContainAny("deepseek-v3", "deepseek_v3", "deepseek-r1-0528")},
+	{parser: "kimi_k2", match: candidatesContainAll("kimi-k2", "instruct")},
+	{parser: "openai", match: candidatesContainAny("openai/gpt-oss", "gpt-oss-")},
+	{parser: "hunyuan_a13b", reasoningParser: "hunyuan_a13b", match: candidatesContainAll("hunyuan-a13b", "instruct")},
+	{parser: "cohere_command3", reasoningParser: "cohere_command3", match: candidatesContainAny("command-a-reasoning")},
+	{parser: "longcat", match: candidatesContainAny("meituan-longcat/longcat", "longcat-flash")},
+	{parser: "glm47", match: candidatesContainAny("glm-4.7", "glm4.7", "glm47")},
+	{parser: "glm45", match: candidatesContainAny("glm-4.5", "glm4.5", "glm45", "glm-4.6", "glm4.6", "glm46")},
+	{parser: "functiongemma", match: candidatesContainAny("google/functiongemma", "functiongemma")},
+	{parser: "olmo3", match: candidatesContainAny("olmo-3", "olmo3")},
+	{parser: "gigachat3", match: candidatesContainAny("gigachat3", "gigachat-3")},
+	{parser: "apertus", match: candidatesContainAny("apertus")},
+	{parser: "pythonic", match: candidatesContainAny("toolace", "ultravoxai/ultravox-v0_5")},
+}
+
+func applyVLLMToolCallDefaults(ctx context.Context, input *api.LLMSkuCreateInput) error {
+	if input == nil || input.LLMType != string(api.LLM_CONTAINER_VLLM) {
+		return nil
+	}
+	if input.LLMSpec == nil {
+		input.LLMSpec = &api.LLMSpec{}
+	}
+	if input.LLMSpec.Vllm == nil {
+		input.LLMSpec.Vllm = &api.LLMSpecVllm{}
+	}
+
+	candidates := collectToolCallCandidates(ctx, input)
+	profile, matched := resolveVLLMToolCallProfile(candidates)
+	explicitAutoChoice := inputHasVLLMRuntimeArg(input, vllmArgEnableAutoToolChoice)
+	explicitParser := inputHasVLLMRuntimeArg(input, vllmArgToolCallParser)
+
+	if !matched {
+		if explicitAutoChoice && !explicitParser {
+			return errors.Wrap(httperrors.ErrInputParameter, "enable-auto-tool-choice requires tool-call-parser for unknown vLLM model")
+		}
+		return nil
+	}
+
+	appendVLLMCustomizedArgIfMissing(input, vllmArgEnableAutoToolChoice, "")
+	if !explicitParser {
+		appendVLLMCustomizedArgIfMissing(input, vllmArgToolCallParser, profile.parser)
+	}
+	if profile.reasoningParser != "" && !inputHasVLLMRuntimeArg(input, vllmArgReasoningParser) {
+		appendVLLMCustomizedArgIfMissing(input, vllmArgReasoningParser, profile.reasoningParser)
+	}
+	return nil
+}
+
+func collectToolCallCandidates(ctx context.Context, input *api.LLMSkuCreateInput) []string {
+	if input == nil {
+		return nil
+	}
+	out := make([]string, 0, 8+len(input.MountedModels)*3)
+	add := func(s string) {
+		s = normalizeToolCallCandidate(s)
+		if s != "" {
+			out = append(out, s)
+		}
+	}
+	add(input.HuggingfaceRepoId)
+	add(input.ModelScopeModelId)
+	add(input.LocalPath)
+	if input.LocalPath != "" {
+		add(filepath.Base(input.LocalPath))
+	}
+	if input.ModelSpec != nil {
+		add(input.ModelSpec.RepoId)
+		add(input.ModelSpec.ModelName)
+		add(input.ModelSpec.ModelTag)
+		add(input.ModelSpec.Revision)
+	}
+	if input.LLMSpec != nil && input.LLMSpec.Vllm != nil {
+		add(input.LLMSpec.Vllm.PreferredModel)
+		if input.LLMSpec.Vllm.PreferredModel != "" {
+			add(filepath.Base(input.LLMSpec.Vllm.PreferredModel))
+		}
+	}
+	for _, id := range input.MountedModels {
+		add(id)
+		if ctx == nil || strings.TrimSpace(id) == "" {
+			continue
+		}
+		obj, err := models.GetInstantModelManager().FetchById(id)
+		if err != nil {
+			continue
+		}
+		im, ok := obj.(*models.SInstantModel)
+		if !ok || im == nil {
+			continue
+		}
+		add(im.ModelName)
+		add(im.ModelId)
+		add(im.ModelTag)
+	}
+	return out
+}
+
+func normalizeToolCallCandidate(s string) string {
+	return strings.ToLower(strings.TrimSpace(s))
+}
+
+func resolveVLLMToolCallProfile(candidates []string) (vllmToolCallProfile, bool) {
+	for _, profile := range vllmToolCallProfiles {
+		if profile.match != nil && profile.match(candidates) {
+			return profile, true
+		}
+	}
+	return vllmToolCallProfile{}, false
+}
+
+func candidatesContainAny(needles ...string) func([]string) bool {
+	normalized := normalizeNeedles(needles)
+	return func(candidates []string) bool {
+		for _, candidate := range candidates {
+			for _, needle := range normalized {
+				if strings.Contains(candidate, needle) {
+					return true
+				}
+			}
+		}
+		return false
+	}
+}
+
+func candidatesContainAll(needles ...string) func([]string) bool {
+	normalized := normalizeNeedles(needles)
+	return func(candidates []string) bool {
+		for _, candidate := range candidates {
+			matchedAll := true
+			for _, needle := range normalized {
+				if !strings.Contains(candidate, needle) {
+					matchedAll = false
+					break
+				}
+			}
+			if matchedAll {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func normalizeNeedles(needles []string) []string {
+	out := make([]string, 0, len(needles))
+	for _, needle := range needles {
+		needle = normalizeToolCallCandidate(needle)
+		if needle != "" {
+			out = append(out, needle)
+		}
+	}
+	return out
+}
+
+func inputHasVLLMRuntimeArg(input *api.LLMSkuCreateInput, key string) bool {
+	key = normalizeVLLMRuntimeArgKey(key)
+	if input == nil || key == "" {
+		return false
+	}
+	if vllmCustomizedArgsHaveKey(input.LLMSpec, key) {
+		return true
+	}
+	for _, item := range input.BackendParameters {
+		arg, ok, err := parseBackendParameterArg(item)
+		if err != nil || !ok {
+			continue
+		}
+		if normalizeVLLMRuntimeArgKey(arg.Key) == key {
+			return true
+		}
+	}
+	return false
+}
+
+func vllmCustomizedArgsHaveKey(spec *api.LLMSpec, key string) bool {
+	if spec == nil || spec.Vllm == nil {
+		return false
+	}
+	key = normalizeVLLMRuntimeArgKey(key)
+	for _, arg := range spec.Vllm.CustomizedArgs {
+		if arg != nil && normalizeVLLMRuntimeArgKey(arg.Key) == key {
+			return true
+		}
+	}
+	return false
+}
+
+func normalizeVLLMRuntimeArgKey(key string) string {
+	return strings.TrimPrefix(strings.TrimSpace(key), "--")
+}
+
+func appendVLLMCustomizedArgIfMissing(input *api.LLMSkuCreateInput, key string, value string) {
+	if input == nil {
+		return
+	}
+	if input.LLMSpec == nil {
+		input.LLMSpec = &api.LLMSpec{}
+	}
+	if input.LLMSpec.Vllm == nil {
+		input.LLMSpec.Vllm = &api.LLMSpecVllm{}
+	}
+	if vllmCustomizedArgsHaveKey(input.LLMSpec, key) {
+		return
+	}
+	input.LLMSpec.Vllm.CustomizedArgs = append(input.LLMSpec.Vllm.CustomizedArgs, &api.VllmCustomizedArg{
+		Key:   key,
+		Value: value,
+	})
+}

pkg/llm/models/llm_deployment.go

@@ -145,6 +145,7 @@ func (man *SLLMDeploymentManager) ValidateCreateData(
 		if err := validateDeploymentGpuMemoryUtilization(input.GpuMemoryUtilization, input.AutoGpuMemoryUtilization, lSku.LLMType); err != nil {
 			return input, err
 		}
+		defaultDeploymentAutoGpuMemoryUtilization(input, lSku.LLMType)
 		if err := ValidateDeploymentDevices(lSku.LLMType, lSku); err != nil {
 			return input, err
 		}
@@ -176,6 +177,7 @@ func (man *SLLMDeploymentManager) ValidateCreateData(
 		if err := validateDeploymentGpuMemoryUtilization(input.GpuMemoryUtilization, input.AutoGpuMemoryUtilization, input.SkuSpec.LLMType); err != nil {
 			return input, err
 		}
+		defaultDeploymentAutoGpuMemoryUtilization(input, input.SkuSpec.LLMType)
 		if err := ValidateDeploymentDevices(input.SkuSpec.LLMType, skuFromLLMSkuCreateInput(input.SkuSpec)); err != nil {
 			return input, err
 		}

pkg/llm/models/llm_deployment_gpu.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"context"
+	"encoding/json"
 	"math"
 	"strconv"
 	"strings"
@@ -12,6 +13,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/llm"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/llm/options"
+	"yunion.io/x/onecloud/pkg/llm/utils/vram"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
@@ -21,14 +23,38 @@ const (
 	autoGpuMemoryUtilizationSafetyFactor = 1.10
 	autoGpuMemoryUtilizationMin          = 0.05
 	autoGpuMemoryUtilizationMax          = 0.95
-	instantModelDynamicVramRatio         = 0.15
-	instantModelFixedVramMB              = 500
+
+	// vLLM derives an unset max-model-len from model config. When auto GPU
+	// memory utilization is enabled, inject a conservative cap so the heuristic
+	// VRAM estimate is not paired with an unexpectedly large context.
+	autoGpuMemoryUtilizationDefaultContextTokens = int64(8192)
+
+	sglangAutoGpuMemoryMetadataReserveMB = 512
 )
 
 func boolPtrValue(v *bool) bool {
 	return v != nil && *v
 }
 
+func deploymentAutoGpuMemoryUtilizationEnabled(auto *bool, llmType string) bool {
+	if auto != nil {
+		return *auto
+	}
+	_, ok := gpuMemoryUtilizationRuntimeArgKey(llmType)
+	return ok
+}
+
+func defaultDeploymentAutoGpuMemoryUtilization(input *api.LLMDeploymentCreateInput, llmType string) {
+	if input == nil || input.GpuMemoryUtilization != nil || input.AutoGpuMemoryUtilization != nil {
+		return
+	}
+	if !deploymentAutoGpuMemoryUtilizationEnabled(nil, llmType) {
+		return
+	}
+	enabled := true
+	input.AutoGpuMemoryUtilization = &enabled
+}
+
 func validateDeploymentGpuMemoryUtilization(util *float64, auto *bool, llmType string) error {
 	needsRuntimeArg := util != nil || boolPtrValue(auto)
 	if util != nil {
@@ -69,17 +95,183 @@ func calculateAutoGpuMemoryUtilization(requiredVramMB int64, gpuMemoryMB int64,
 	)
 }
 
-func calculateAutoGpuMemoryUtilizationForModelSize(modelSizeMB int64, gpuMemoryMB int64, tensorParallelSize int) (float64, error) {
-	if modelSizeMB <= 0 {
-		return 0, errors.Wrap(httperrors.ErrInputParameter, "mounted model size is empty")
+func instantModelEstimatedVramRequirementMB(model *SInstantModel) int64 {
+	if model == nil {
+		return 0
 	}
-	tp := normalizeTensorParallelSize(tensorParallelSize)
-	// Tensor parallel shards model weights, but runtime/KV/framework overhead is
-	// still charged per GPU here to avoid underestimating heterogeneous multi-GPU deployments.
-	perGPURequiredMB := float64(modelSizeMB)/float64(tp) +
-		float64(modelSizeMB)*instantModelDynamicVramRatio +
-		instantModelFixedVramMB
-	return calculateAutoGpuMemoryUtilizationFromPerGPURequired(perGPURequiredMB, gpuMemoryMB)
+	return int64(vram.EstimateClaimMb(model.WeightSizeBytes, model.LlmType))
+}
+
+func runtimeArgKeyIn(key string, keys []string) bool {
+	key = normalizeRuntimeArgKey(key)
+	for _, candidate := range keys {
+		if key == candidate {
+			return true
+		}
+	}
+	return false
+}
+
+func runtimeHasExplicitTokenLimit(sku *SLLMSku) bool {
+	if sku == nil {
+		return false
+	}
+	switch api.LLMContainerType(sku.LLMType) {
+	case api.LLM_CONTAINER_VLLM:
+		return runtimeHasExplicitArg(sku, []string{"max-model-len"})
+	default:
+		return false
+	}
+}
+
+func runtimeHasExplicitArg(sku *SLLMSku, keys []string) bool {
+	if sku == nil {
+		return false
+	}
+	if backendParametersContainRuntimeArg(sku.BackendParameters, keys) {
+		return true
+	}
+	switch api.LLMContainerType(sku.LLMType) {
+	case api.LLM_CONTAINER_VLLM:
+		if sku.LLMSpec == nil || sku.LLMSpec.Vllm == nil {
+			return false
+		}
+		for _, arg := range sku.LLMSpec.Vllm.CustomizedArgs {
+			if arg != nil && runtimeArgKeyIn(arg.Key, keys) {
+				return true
+			}
+		}
+	case api.LLM_CONTAINER_SGLANG:
+		if sku.LLMSpec == nil || sku.LLMSpec.SGLang == nil {
+			return false
+		}
+		for _, arg := range sku.LLMSpec.SGLang.CustomizedArgs {
+			if arg != nil && runtimeArgKeyIn(arg.Key, keys) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func backendParametersContainRuntimeArg(raw string, keys []string) bool {
+	items := backendParameterItems(raw)
+	for i := range items {
+		argKey, _, ok := splitBackendParameterFlag(items[i])
+		if ok && runtimeArgKeyIn(argKey, keys) {
+			return true
+		}
+	}
+	return false
+}
+
+func tokenLimitFromBackendParameters(raw string, keys []string) (int64, bool) {
+	items := backendParameterItems(raw)
+	var tokenLimit int64
+	found := false
+	for i := range items {
+		key, value, ok := splitBackendParameterFlag(items[i])
+		if !ok || !runtimeArgKeyIn(key, keys) {
+			continue
+		}
+		if value == "" && i+1 < len(items) {
+			next := strings.TrimSpace(items[i+1])
+			if next == "-1" || !strings.HasPrefix(next, "-") {
+				value = next
+			}
+		}
+		if val, ok := parseTokenLimitValue(value); ok {
+			tokenLimit = val
+			found = true
+		}
+	}
+	return tokenLimit, found
+}
+
+func backendParameterItems(raw string) []string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return nil
+	}
+	items := []string{}
+	if err := json.Unmarshal([]byte(raw), &items); err != nil {
+		return []string{raw}
+	}
+	return items
+}
+
+func splitBackendParameterFlag(item string) (string, string, bool) {
+	item = strings.TrimSpace(item)
+	if item == "" || strings.HasPrefix(item, "-") && !strings.HasPrefix(item, "--") {
+		return "", "", false
+	}
+	item = strings.TrimSpace(strings.TrimPrefix(item, "--"))
+	if item == "" {
+		return "", "", false
+	}
+	key := item
+	value := ""
+	if idx := strings.Index(item, "="); idx >= 0 {
+		key = strings.TrimSpace(item[:idx])
+		value = item[idx+1:]
+	} else if fields := strings.Fields(item); len(fields) > 1 {
+		key = fields[0]
+		value = strings.TrimSpace(item[len(key):])
+	}
+	return key, trimArgumentValue(value), true
+}
+
+func normalizeRuntimeArgKey(key string) string {
+	return strings.TrimPrefix(strings.TrimSpace(key), "--")
+}
+
+func trimArgumentValue(value string) string {
+	value = strings.TrimSpace(value)
+	return strings.Trim(value, `"'`)
+}
+
+func parseTokenLimitValue(value string) (int64, bool) {
+	value = trimArgumentValue(value)
+	if value == "" {
+		return 0, false
+	}
+	if strings.EqualFold(value, "auto") {
+		return 0, true
+	}
+	if val, err := strconv.ParseInt(value, 10, 64); err == nil {
+		if val <= 0 {
+			return 0, true
+		}
+		return val, true
+	}
+
+	multiplier := float64(1)
+	number := value
+	switch value[len(value)-1] {
+	case 'k':
+		multiplier = 1000
+		number = value[:len(value)-1]
+	case 'K':
+		multiplier = 1024
+		number = value[:len(value)-1]
+	case 'm':
+		multiplier = 1000 * 1000
+		number = value[:len(value)-1]
+	case 'M':
+		multiplier = 1024 * 1024
+		number = value[:len(value)-1]
+	case 'g':
+		multiplier = 1000 * 1000 * 1000
+		number = value[:len(value)-1]
+	case 'G':
+		multiplier = 1024 * 1024 * 1024
+		number = value[:len(value)-1]
+	}
+	parsed, err := strconv.ParseFloat(strings.TrimSpace(number), 64)
+	if err != nil || parsed <= 0 {
+		return 0, false
+	}
+	return int64(math.Ceil(parsed * multiplier)), true
 }
 
 func calculateAutoGpuMemoryUtilizationFromPerGPURequired(perGPURequiredMB float64, gpuMemoryMB int64) (float64, error) {
@@ -97,6 +289,125 @@ func calculateAutoGpuMemoryUtilizationFromPerGPURequired(perGPURequiredMB float6
 	return math.Ceil(raw*100) / 100, nil
 }
 
+func calculateDeploymentAutoGpuMemoryUtilization(sku *SLLMSku, requiredVramMB int64, gpuMemoryMB int64, tensorParallelSize int) (float64, error) {
+	if sku != nil && api.LLMContainerType(sku.LLMType) == api.LLM_CONTAINER_SGLANG {
+		return calculateSGLangAutoGpuMemoryUtilization(sku, requiredVramMB, gpuMemoryMB, tensorParallelSize)
+	}
+	return calculateAutoGpuMemoryUtilization(requiredVramMB, gpuMemoryMB, tensorParallelSize)
+}
+
+func calculateSGLangAutoGpuMemoryUtilization(sku *SLLMSku, requiredVramMB int64, gpuMemoryMB int64, tensorParallelSize int) (float64, error) {
+	modelUtilization, err := calculateAutoGpuMemoryUtilization(requiredVramMB, gpuMemoryMB, tensorParallelSize)
+	if err != nil {
+		return 0, err
+	}
+	runtimeUtilization, err := calculateSGLangMemFractionStatic(sku, gpuMemoryMB, tensorParallelSize)
+	if err != nil {
+		return 0, err
+	}
+	if runtimeUtilization > modelUtilization {
+		return runtimeUtilization, nil
+	}
+	return modelUtilization, nil
+}
+
+func calculateSGLangMemFractionStatic(sku *SLLMSku, gpuMemoryMB int64, tensorParallelSize int) (float64, error) {
+	if gpuMemoryMB <= 0 {
+		return 0, errors.Wrap(httperrors.ErrInputParameter, "gpu memory_mb is empty")
+	}
+	tpSize := int64(normalizeTensorParallelSize(tensorParallelSize))
+	if val, ok := sglangRuntimeIntArg(sku, []string{"tp-size", "tensor-parallel-size"}); ok && val > 0 {
+		tpSize = val
+	}
+	ppSize := int64(1)
+	if val, ok := sglangRuntimeIntArg(sku, []string{"pp-size", "pipeline-parallel-size"}); ok && val > 0 {
+		ppSize = val
+	}
+	chunkedPrefillSize, cudaGraphMaxBS := sglangRuntimeReserveDefaults(gpuMemoryMB, tpSize)
+	if val, ok := sglangRuntimeIntArg(sku, []string{"chunked-prefill-size"}); ok {
+		chunkedPrefillSize = val
+	}
+	if val, ok := sglangRuntimeIntArg(sku, []string{"cuda-graph-max-bs", "cuda-graph-max-bs-decode"}); ok && val > 0 {
+		cudaGraphMaxBS = val
+	}
+
+	reservedMemMB := float64(sglangAutoGpuMemoryMetadataReserveMB)
+	if chunkedPrefillSize > 0 {
+		reservedMemMB += float64(maxInt64(chunkedPrefillSize, 2048)) * 1.5
+	} else if maxPrefillTokens, ok := sglangRuntimeIntArg(sku, []string{"max-prefill-tokens"}); ok && maxPrefillTokens > 0 {
+		reservedMemMB += float64(maxInt64(maxPrefillTokens, 2048)) * 1.5
+	} else {
+		reservedMemMB += 2048 * 1.5
+	}
+	reservedMemMB += float64(cudaGraphMaxBS) * 2
+	reservedMemMB += float64(tpSize*ppSize) / 8 * 1024
+	if gpuMemoryMB > 60*1024 && reservedMemMB < 10*1024 {
+		reservedMemMB = 10 * 1024
+	}
+
+	utilization := (float64(gpuMemoryMB) - reservedMemMB) / float64(gpuMemoryMB)
+	if utilization > autoGpuMemoryUtilizationMax {
+		utilization = autoGpuMemoryUtilizationMax
+	}
+	if utilization < autoGpuMemoryUtilizationMin {
+		utilization = autoGpuMemoryUtilizationMin
+	}
+	return math.Round(utilization*1000) / 1000, nil
+}
+
+func sglangRuntimeReserveDefaults(gpuMemoryMB int64, tpSize int64) (int64, int64) {
+	if gpuMemoryMB < 20*1024 {
+		return 2048, 8
+	}
+	if gpuMemoryMB < 35*1024 {
+		if tpSize < 4 {
+			return 2048, 24
+		}
+		return 2048, 80
+	}
+	if gpuMemoryMB < 60*1024 {
+		if tpSize < 4 {
+			return 4096, 32
+		}
+		return 4096, 160
+	}
+	if gpuMemoryMB < 160*1024 {
+		if tpSize < 4 {
+			return 8192, 256
+		}
+		return 8192, 512
+	}
+	return 16384, 512
+}
+
+func sglangRuntimeIntArg(sku *SLLMSku, keys []string) (int64, bool) {
+	if sku == nil {
+		return 0, false
+	}
+	if val, ok := tokenLimitFromBackendParameters(sku.BackendParameters, keys); ok {
+		return val, true
+	}
+	if sku.LLMSpec == nil || sku.LLMSpec.SGLang == nil {
+		return 0, false
+	}
+	for _, arg := range sku.LLMSpec.SGLang.CustomizedArgs {
+		if arg == nil || !runtimeArgKeyIn(arg.Key, keys) {
+			continue
+		}
+		if val, ok := parseTokenLimitValue(arg.Value); ok {
+			return val, true
+		}
+	}
+	return 0, false
+}
+
+func maxInt64(a int64, b int64) int64 {
+	if a > b {
+		return a
+	}
+	return b
+}
+
 func normalizeTensorParallelSize(tensorParallelSize int) int {
 	if tensorParallelSize <= 0 {
 		return 1
@@ -135,6 +446,26 @@ func buildGpuMemoryUtilizationLLMSpec(llmType string, utilization float64) (*api
 	}
 }
 
+func buildAutoGpuMemoryUtilizationLLMSpec(sku *SLLMSku, utilization float64) (*api.LLMSpec, error) {
+	if sku == nil {
+		return nil, nil
+	}
+	spec, err := buildGpuMemoryUtilizationLLMSpec(sku.LLMType, utilization)
+	if err != nil {
+		return nil, err
+	}
+	switch api.LLMContainerType(sku.LLMType) {
+	case api.LLM_CONTAINER_VLLM:
+		if spec != nil && spec.Vllm != nil && !runtimeHasExplicitTokenLimit(sku) {
+			spec.Vllm.CustomizedArgs = append(spec.Vllm.CustomizedArgs, &api.VllmCustomizedArg{
+				Key:   "max-model-len",
+				Value: strconv.FormatInt(autoGpuMemoryUtilizationDefaultContextTokens, 10),
+			})
+		}
+	}
+	return spec, nil
+}
+
 func formatGpuMemoryUtilization(v float64) string {
 	return strconv.FormatFloat(v, 'f', -1, 64)
 }
@@ -146,14 +477,14 @@ func BuildDeploymentResolvedGpuMemoryLLMSpec(ctx context.Context, userCred mccli
 	if deploy.GpuMemoryUtilization != nil {
 		return buildDeploymentGpuMemoryLLMSpec(deploy, sku)
 	}
-	if !boolPtrValue(deploy.AutoGpuMemoryUtilization) {
+	if !deploymentAutoGpuMemoryUtilizationEnabled(deploy.AutoGpuMemoryUtilization, sku.LLMType) {
 		return nil, nil
 	}
 	tensorParallelSize := 1
 	if sku.Devices != nil && len(*sku.Devices) > 0 {
 		tensorParallelSize = len(*sku.Devices)
 	}
-	modelSizeMB, err := maxMountedModelSizeMB(sku)
+	requiredVramMB, err := maxMountedModelVramRequirementMB(sku)
 	if err != nil {
 		return nil, err
 	}
@@ -161,33 +492,33 @@ func BuildDeploymentResolvedGpuMemoryLLMSpec(ctx context.Context, userCred mccli
 	if err != nil {
 		return nil, err
 	}
-	utilization, err := calculateAutoGpuMemoryUtilizationForModelSize(modelSizeMB, gpuMemoryMB, tensorParallelSize)
+	utilization, err := calculateDeploymentAutoGpuMemoryUtilization(sku, requiredVramMB, gpuMemoryMB, tensorParallelSize)
 	if err != nil {
 		return nil, err
 	}
-	return buildGpuMemoryUtilizationLLMSpec(sku.LLMType, utilization)
+	return buildAutoGpuMemoryUtilizationLLMSpec(sku, utilization)
 }
 
-func maxMountedModelSizeMB(sku *SLLMSku) (int64, error) {
+func maxMountedModelVramRequirementMB(sku *SLLMSku) (int64, error) {
 	modelIds := sku.GetMountedModels()
 	if len(modelIds) == 0 {
 		return 0, httperrors.NewInputParameterError("auto_gpu_memory_utilization requires mounted models: configure mounted_models on the LLM SKU")
 	}
-	var maxSize int64
+	var maxRequiredVramMB int64
 	for _, modelId := range modelIds {
 		obj, err := GetInstantModelManager().FetchById(modelId)
 		if err != nil {
 			return 0, errors.Wrapf(err, "fetch InstantModel %s", modelId)
 		}
-		sizeMB := int64(obj.(*SInstantModel).GetActualSizeMb())
-		if sizeMB > maxSize {
-			maxSize = sizeMB
+		requiredVramMB := instantModelEstimatedVramRequirementMB(obj.(*SInstantModel))
+		if requiredVramMB > maxRequiredVramMB {
+			maxRequiredVramMB = requiredVramMB
 		}
 	}
-	if maxSize <= 0 {
-		return 0, errors.Wrap(httperrors.ErrInputParameter, "mounted model size is empty")
+	if maxRequiredVramMB <= 0 {
+		return 0, errors.Wrap(httperrors.ErrInputParameter, "mounted model vram requirement is empty")
 	}
-	return maxSize, nil
+	return maxRequiredVramMB, nil
 }
 
 func minGpuMemoryMB(ctx context.Context, userCred mcclient.TokenCredential, devices *api.Devices) (int64, error) {

pkg/mcclient/options/llm/llm_deployment.go

@@ -83,7 +83,7 @@ type LLMDeploymentCreateOptions struct {
 	DistributedInference     *bool    `help:"enable distributed inference" json:"distributed_inference"`
 	GpuMemoryUtilization     *float64 `token:"gpu-memory-utilization" help:"GPU memory utilization fraction for backend runtime (0-1)" json:"gpu_memory_utilization"`
 	GpuUtilization           *float64 `token:"gpu-utilization" help:"Alias of --gpu-memory-utilization" json:"-"`
-	AutoGpuMemoryUtilization *bool    `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory" json:"auto_gpu_memory_utilization"`
+	AutoGpuMemoryUtilization *bool    `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory (default true for supported backends; use false to disable)" json:"auto_gpu_memory_utilization"`
 	RestartOnError           *bool    `help:"restart on error" json:"restart_on_error"`
 	AccessPolicy             string   `help:"access policy" choices:"public|authed|allowed_users" json:"access_policy"`
 	AutoRegisterAiproxy      *bool    `help:"auto register running replicas with aiproxy (default true; use --auto-register-aiproxy=false to disable)" json:"auto_register_aiproxy"`
@@ -233,7 +233,7 @@ type LLMDeploymentUpdateOptions struct {
 	PlacementStrategy        string   `help:"placement strategy" json:"placement_strategy"`
 	GpuMemoryUtilization     *float64 `token:"gpu-memory-utilization" help:"GPU memory utilization fraction for backend runtime (0-1)" json:"gpu_memory_utilization"`
 	GpuUtilization           *float64 `token:"gpu-utilization" help:"Alias of --gpu-memory-utilization" json:"-"`
-	AutoGpuMemoryUtilization *bool    `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory" json:"auto_gpu_memory_utilization"`
+	AutoGpuMemoryUtilization *bool    `token:"auto-gpu-memory-utilization" help:"calculate GPU memory utilization from model VRAM and GPU memory (default true for supported backends; use false to disable)" json:"auto_gpu_memory_utilization"`
 	AccessPolicy             string   `help:"access policy" json:"access_policy"`
 	AutoRegisterAiproxy      *bool    `help:"auto register running replicas with aiproxy (default true; use --auto-register-aiproxy=false to disable)" json:"auto_register_aiproxy"`
 	AiproxyModelPrefix       *string  `help:"deprecated; no longer affects aiproxy client model alias" json:"aiproxy_model_prefix"`