From 4c0dd07a91d473b5ad97b1938653e792cbb3dc84 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Mon, 25 May 2026 16:27:20 +0800 Subject: [PATCH] feat(llm,scheduler): vram calculation (#24884) --- cmd/climc/shell/llm/instantmodel.go | 2 + pkg/apis/compute/api.go | 6 + pkg/apis/llm/instantmodel.go | 34 ++++++ pkg/apis/llm/sku.go | 4 + pkg/compute/models/isolated_devices.go | 44 ++++++- pkg/llm/models/instantmodel.go | 111 ++++++++++++++++++ pkg/llm/models/instantmodel_huggingface.go | 58 ++++++++- pkg/llm/models/llm_base_pod.go | 9 ++ pkg/llm/models/sku.go | 24 +++- .../tasks/llm/llm_deployment_create_task.go | 21 ++++ .../llm/llm_instant_model_import_task.go | 39 ++++++ pkg/llm/utils/vram/doc.go | 1 + pkg/llm/utils/vram/estimator.go | 66 +++++++++++ pkg/llm/utils/vram/estimator_test.go | 83 +++++++++++++ pkg/mcclient/options/llm/instantmodel.go | 8 ++ .../predicates/isolated_device_predicate.go | 65 ++++++++++ .../isolated_device_predicate_test.go | 97 +++++++++++++++ .../priorities/guest/gpu_vram_binpack.go | 92 +++++++++++++++ .../priorities/guest/gpu_vram_binpack_test.go | 104 ++++++++++++++++ pkg/scheduler/algorithmprovider/defaults.go | 1 + pkg/scheduler/cache/candidate/base.go | 1 + pkg/scheduler/core/types.go | 3 + 22 files changed, 863 insertions(+), 10 deletions(-) create mode 100644 pkg/llm/utils/vram/doc.go create mode 100644 pkg/llm/utils/vram/estimator.go create mode 100644 pkg/llm/utils/vram/estimator_test.go create mode 100644 pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go create mode 100644 pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go create mode 100644 pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack_test.go diff --git a/cmd/climc/shell/llm/instantmodel.go b/cmd/climc/shell/llm/instantmodel.go index 698f7ddbde..9d40ba7886 100644 --- a/cmd/climc/shell/llm/instantmodel.go +++ b/cmd/climc/shell/llm/instantmodel.go @@ -21,6 +21,8 @@ func init() { cmd.Perform("public", new(commonoptions.BasePublicOptions)) cmd.Perform("private", new(commonoptions.BaseIdOptions)) cmd.PerformClass("import", new(options.LLMInstantModelImportOptions)) + cmd.PerformClass("backfill-vram", new(options.LLMInstantModelBackfillVramOptions)) + cmd.Get("vram-requirement", new(commonoptions.BaseIdOptions)) cmd.GetProperty(new(options.LLMInstantModelHuggingFaceSearchOptions)) cmd.GetProperty(new(options.LLMInstantModelHuggingFaceRepoInfoOptions)) cmd.GetProperty(new(options.LLMInstantModelCommunityRegistryOptions)) diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 942a1dcc84..fe031004b6 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -320,6 +320,12 @@ type IsolatedDeviceConfig struct { WireId string `json:"wire_id"` DiskIndex *int8 `json:"disk_index"` DevicePath string `json:"device_path"` + // MemoryMb is the minimum on-device memory in MiB required from the + // candidate isolated_device (e.g. NVIDIA GPU VRAM). 0 means no constraint. + // The scheduler excludes devices whose memory_size > 0 and is below this + // threshold; devices with memory_size == 0 are treated as unknown and + // allowed through to avoid penalising hosts that haven't reported yet. + MemoryMb int `json:"memory_mb,omitempty"` } type BaremetalDiskConfig struct { diff --git a/pkg/apis/llm/instantmodel.go b/pkg/apis/llm/instantmodel.go index 7426f9fc7d..9bed5bb75d 100644 --- a/pkg/apis/llm/instantmodel.go +++ b/pkg/apis/llm/instantmodel.go @@ -87,6 +87,40 @@ type MountedByLLMInfo struct { type InstantModelSyncstatusInput struct { } +// InstantModelVramRequirement reports the heuristic VRAM needed to run this +// model, mirroring GPUStack's `estimate_model_vram`. Returns +// `vram_required_mb=0` when `weight_size_bytes` is unknown — callers should +// treat 0 as "no constraint", same as the scheduler does. +type InstantModelVramRequirement struct { + LlmType string `json:"llm_type"` + WeightSizeBytes int64 `json:"weight_size_bytes"` + VramRequiredMb int `json:"vram_required_mb"` +} + +type InstantModelBackfillVramInput struct { + // DryRun reports what would change but does not write to the database. + DryRun bool `json:"dry_run,omitempty"` +} + +type InstantModelBackfillVramItem struct { + Id string `json:"id"` + Name string `json:"name"` + ModelName string `json:"model_name"` + ModelTag string `json:"model_tag"` + WeightSizeBytes int64 `json:"weight_size_bytes"` + Status string `json:"status"` // updated, skipped, failed + Reason string `json:"reason,omitempty"` +} + +type InstantModelBackfillVramOutput struct { + DryRun bool `json:"dry_run"` + Scanned int `json:"scanned"` + Updated int `json:"updated"` + Skipped int `json:"skipped"` + Failed int `json:"failed"` + Items []InstantModelBackfillVramItem `json:"items,omitempty"` +} + type InstantAppCacheInput struct { } diff --git a/pkg/apis/llm/sku.go b/pkg/apis/llm/sku.go index f0d1c59768..426f22092b 100644 --- a/pkg/apis/llm/sku.go +++ b/pkg/apis/llm/sku.go @@ -150,6 +150,10 @@ type LLMSKuBaseCreateInput struct { Cpu int `json:"cpu"` Memory int `json:"memory"` Bandwidth int `json:"bandwidth"` + // VramClaimMb is the estimated VRAM (MiB) the inference instance will + // require. Optional — if 0, the deployment create task will auto-fill it + // from the mounted InstantModel's weight_size_bytes. + VramClaimMb int `json:"vram_claim_mb,omitempty"` Volumes *Volumes `json:"volumes"` HostPaths *HostPaths `json:"host_paths"` diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index b3e1afa8cc..461b4fa78c 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -478,13 +478,16 @@ func (manager *SIsolatedDeviceManager) fuzzyMatchModel(fuzzyStr string, devType q = q.Equals("dev_type", devType) } - qe := q.Equals("model", fuzzyStr) - cnt, err := qe.CountWithError() - if err != nil || cnt == 0 { - qe = q.Contains("model", fuzzyStr) + if fuzzyStr != "" { + qe := q.Equals("model", fuzzyStr) + cnt, err := qe.CountWithError() + if err != nil || cnt == 0 { + qe = q.Contains("model", fuzzyStr) + } + q = qe } - err = qe.First(&dev) + err := q.First(&dev) if err == nil { return &dev } @@ -644,6 +647,11 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s device_path %s on host %s", devConfig.Model, devConfig.DevicePath, host.Id) } + devs = filterDevicesByMemoryMb(devs, devConfig.MemoryMb) + if len(devs) == 0 { + return fmt.Errorf("device_path %s on host %s does not satisfy memory_mb=%d", + devConfig.DevicePath, host.Id, devConfig.MemoryMb) + } var selectedDev SIsolatedDevice for i := range devs { if _, ok := usedDevMap[devs[i].DevicePath]; !ok { @@ -657,6 +665,24 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByDevicePath(ctx c return guest.attachIsolatedDevice(ctx, userCred, &selectedDev, devConfig.NetworkIndex, devConfig.DiskIndex) } +// filterDevicesByMemoryMb drops devices whose MemorySize > 0 and is below the +// requested minMemMb. MemorySize == 0 means the host hasn't reported it yet +// and is treated as unknown (allowed through) to avoid mass-excluding rows +// pending backfill. minMemMb <= 0 short-circuits — no filtering. +func filterDevicesByMemoryMb(devs []SIsolatedDevice, minMemMb int) []SIsolatedDevice { + if minMemMb <= 0 { + return devs + } + out := devs[:0] + for _, d := range devs { + if d.MemorySize > 0 && d.MemorySize < minMemMb { + continue + } + out = append(out, d) + } + return out +} + type GroupDevs struct { DevPath string Devs []SIsolatedDevice @@ -814,6 +840,14 @@ func (manager *SIsolatedDeviceManager) attachHostDeviceToGuestByModel( if err != nil || len(devs) == 0 { return fmt.Errorf("Can't found model %s on host %s", devConfig.Model, host.Id) } + // Honour the request's VRAM floor. Predicate already verified enough + // fitting devices exist on the host; here we make sure attach picks one + // of them rather than a same-Model but smaller-VRAM card. + devs = filterDevicesByMemoryMb(devs, devConfig.MemoryMb) + if len(devs) == 0 { + return fmt.Errorf("model %s on host %s has no device with memory_mb>=%d", + devConfig.Model, host.Id, devConfig.MemoryMb) + } // 1. group devices by device_path and numa nodes //groupDevs := make(SorttedGroupDevs, 0) mapDevs := map[string][]SIsolatedDevice{} diff --git a/pkg/llm/models/instantmodel.go b/pkg/llm/models/instantmodel.go index 1d06ba90dd..2c652b8ce0 100644 --- a/pkg/llm/models/instantmodel.go +++ b/pkg/llm/models/instantmodel.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/llm/options" + "yunion.io/x/onecloud/pkg/llm/utils/vram" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" @@ -79,6 +80,13 @@ type SInstantModel struct { ActualSizeMb int32 `nullable:"true" list:"user" update:"user"` + // WeightSizeBytes is the sum of weight-file byte counts at the upstream + // source (HuggingFace .safetensors/.bin/etc. siblings). Distinct from + // `Size` (which mirrors image disk space). 0 means unknown — populated + // best-effort by LLMInstantModelImportTask.OnImportComplete and consumed + // by EstimateVramClaimMb to size SKU vram_claim_mb. + WeightSizeBytes int64 `nullable:"true" default:"0" list:"user"` + AutoCache bool `list:"user"` } @@ -840,6 +848,92 @@ func (man *SInstantModelManager) PerformImport( return man.DoImportWithParent(ctx, userCred, input, "") } +// PerformBackfillVram retroactively populates `weight_size_bytes` for +// InstantModel rows that were imported before the field was introduced. +// HuggingFace is the only supported source in this phase; rows whose ModelName +// doesn't look like a HF repo id (i.e. lacks a "/") are recorded as skipped. +// Already-populated rows are not revisited. Pass dry_run=true to preview. +func (man *SInstantModelManager) PerformBackfillVram( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input apis.InstantModelBackfillVramInput, +) (*apis.InstantModelBackfillVramOutput, error) { + var models []SInstantModel + q := man.Query().Equals("weight_size_bytes", 0).IsFalse("deleted") + if err := db.FetchModelObjects(man, q, &models); err != nil { + return nil, errors.Wrap(err, "FetchModelObjects") + } + + out := &apis.InstantModelBackfillVramOutput{ + DryRun: input.DryRun, + Scanned: len(models), + } + + for i := range models { + m := &models[i] + item := apis.InstantModelBackfillVramItem{ + Id: m.Id, + Name: m.Name, + ModelName: m.ModelName, + ModelTag: m.ModelTag, + } + + // Only HuggingFace is recoverable from the row alone — its ModelName + // is the repo id (e.g. "Qwen/Qwen3-0.6B"); ModelTag is the revision. + if !strings.Contains(m.ModelName, "/") { + item.Status = "skipped" + item.Reason = "unsupported source (only huggingface-style model_name with '/' is recoverable)" + out.Skipped++ + out.Items = append(out.Items, item) + continue + } + revision := m.ModelTag + if revision == "" { + revision = "main" + } + weight, err := FetchHuggingFaceWeightSize(ctx, m.ModelName, revision) + if err != nil { + item.Status = "failed" + item.Reason = err.Error() + out.Failed++ + out.Items = append(out.Items, item) + log.Warningf("BackfillVram: fetch HF weight size %s@%s: %s", m.ModelName, revision, err) + continue + } + if weight <= 0 { + item.Status = "skipped" + item.Reason = "no weight files found" + out.Skipped++ + out.Items = append(out.Items, item) + continue + } + + item.WeightSizeBytes = weight + if input.DryRun { + item.Status = "updated" + item.Reason = "dry-run (no write)" + out.Updated++ + out.Items = append(out.Items, item) + continue + } + if _, err := db.Update(m, func() error { + m.WeightSizeBytes = weight + return nil + }); err != nil { + item.Status = "failed" + item.Reason = errors.Wrap(err, "db.Update").Error() + out.Failed++ + out.Items = append(out.Items, item) + continue + } + item.Status = "updated" + out.Updated++ + out.Items = append(out.Items, item) + } + return out, nil +} + // DoImportWithParent creates a temporary InstantModel and starts an import task, // optionally chaining it to a parent task. When parentTaskId is non-empty, the // parent task will be notified when the import task completes (via subtask @@ -1118,6 +1212,23 @@ func (model *SInstantModel) GetEstimatedVramSizeMb() int64 { return model.GetEstimatedVramSizeBytes() / 1024 / 1024 } +// GetDetailsVramRequirement is the per-row endpoint +// `GET /instant-models/{id}/vram-requirement`. It returns the heuristic VRAM +// requirement computed by the GPUStack-equivalent formula +// (weight_size * 1.2 + framework_overhead). When `weight_size_bytes` is 0 +// (not yet backfilled / unknown source), `vram_required_mb` is also 0. +func (model *SInstantModel) GetDetailsVramRequirement( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, +) (*apis.InstantModelVramRequirement, error) { + return &apis.InstantModelVramRequirement{ + LlmType: model.LlmType, + WeightSizeBytes: model.WeightSizeBytes, + VramRequiredMb: vram.EstimateClaimMb(model.WeightSizeBytes, model.LlmType), + }, nil +} + func (model *SInstantModel) CleanupImportTmpDir(ctx context.Context, userCred mcclient.TokenCredential, tmpDir string) error { // sync image status err := model.syncImageStatus(ctx, userCred) diff --git a/pkg/llm/models/instantmodel_huggingface.go b/pkg/llm/models/instantmodel_huggingface.go index bd8b2b972e..a2336dd8df 100644 --- a/pkg/llm/models/instantmodel_huggingface.go +++ b/pkg/llm/models/instantmodel_huggingface.go @@ -304,9 +304,12 @@ func huggingFaceHTTPGet(ctx context.Context, reqURL string) ([]byte, http.Header } func getHuggingFaceRepoInfo(ctx context.Context, repoID string, revision string) (huggingFaceRepoInfoResponse, error) { - repoURL := fmt.Sprintf("%s/api/models/%s", huggingFaceMirrorEndpoint, escapeURLPathPreserveSlash(repoID)) + // blobs=true makes HF include each sibling's byte size in the response, + // equivalent to Python's `HfApi().model_info(files_metadata=True)`. Without + // it, sibling.Size is always 0 and weight-size summation returns 0. + repoURL := fmt.Sprintf("%s/api/models/%s?blobs=true", huggingFaceMirrorEndpoint, escapeURLPathPreserveSlash(repoID)) if revision != "" { - repoURL = fmt.Sprintf("%s?revision=%s", repoURL, url.QueryEscape(revision)) + repoURL = fmt.Sprintf("%s&revision=%s", repoURL, url.QueryEscape(revision)) } body, _, err := huggingFaceHTTPGet(ctx, repoURL) if err != nil { @@ -319,6 +322,57 @@ func getHuggingFaceRepoInfo(ctx context.Context, repoID string, revision string) return resp, nil } +// huggingFaceWeightExtensions mirrors GPUStack's WEIGHT_FILE_EXTENSIONS — the +// file types that count toward a model's GPU memory footprint. +var huggingFaceWeightExtensions = map[string]struct{}{ + ".safetensors": {}, + ".bin": {}, + ".pt": {}, + ".pth": {}, +} + +// huggingFaceWeightExcludeNames is the list of root-level filenames GPUStack +// explicitly drops to avoid double-counting (e.g. a sharded model that also +// ships a `consolidated.safetensors` covering all shards). +var huggingFaceWeightExcludeNames = map[string]struct{}{ + "consolidated.safetensors": {}, +} + +// FetchHuggingFaceWeightSize is the exported entry point used by external +// callers (the import task in particular). Delegates to the unexported +// implementation; see fetchHuggingFaceWeightSize for the contract. +func FetchHuggingFaceWeightSize(ctx context.Context, repoID, revision string) (int64, error) { + return fetchHuggingFaceWeightSize(ctx, repoID, revision) +} + +// fetchHuggingFaceWeightSize sums the byte sizes of root-level weight files +// in the given HF repo at the given revision. Mirrors GPUStack's +// `get_model_weight_size`. Returns 0 + error on transport / parse failure; +// caller logs and falls back without aborting the import. +func fetchHuggingFaceWeightSize(ctx context.Context, repoID, revision string) (int64, error) { + resp, err := getHuggingFaceRepoInfo(ctx, repoID, revision) + if err != nil { + return 0, errors.Wrap(err, "getHuggingFaceRepoInfo") + } + var total int64 + for _, s := range resp.Siblings { + // Root-level files only — GPUStack passes recursive=False and we want + // the same behaviour to avoid double-counting nested copies. + if strings.Contains(s.RFilename, "/") { + continue + } + if _, skip := huggingFaceWeightExcludeNames[s.RFilename]; skip { + continue + } + ext := strings.ToLower(filepath.Ext(s.RFilename)) + if _, ok := huggingFaceWeightExtensions[ext]; !ok { + continue + } + total += s.Size + } + return total, nil +} + func escapeURLPathPreserveSlash(p string) string { if p == "" { return "" diff --git a/pkg/llm/models/llm_base_pod.go b/pkg/llm/models/llm_base_pod.go index a121df2a51..4e10c6d2b3 100644 --- a/pkg/llm/models/llm_base_pod.go +++ b/pkg/llm/models/llm_base_pod.go @@ -66,11 +66,20 @@ func GetLLMBasePodCreateInput( if effectiveDevices != nil && !effectiveDevices.IsZero() { data.IsolatedDevices = make([]*computeapi.IsolatedDeviceConfig, 0) devices := *effectiveDevices + // Evenly split the SKU's vram_claim_mb across requested devices and + // stamp it onto each request entry. Ceiling division so the sum is + // never less than the claim (1 device → claim itself; 2 devices and + // 40 GiB claim → 20 GiB each). + perDevMinMemMb := 0 + if skuBase.VramClaimMb > 0 && len(devices) > 0 { + perDevMinMemMb = (skuBase.VramClaimMb + len(devices) - 1) / len(devices) + } for i := 0; i < len(devices); i++ { isolatedDevice := &computeapi.IsolatedDeviceConfig{ DevType: devices[i].DevType, Model: devices[i].Model, DevicePath: devices[i].DevicePath, + MemoryMb: perDevMinMemMb, } data.IsolatedDevices = append(data.IsolatedDevices, isolatedDevice) } diff --git a/pkg/llm/models/sku.go b/pkg/llm/models/sku.go index 3c041bac3d..48af415309 100644 --- a/pkg/llm/models/sku.go +++ b/pkg/llm/models/sku.go @@ -8,6 +8,7 @@ import ( "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/apis" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" api "yunion.io/x/onecloud/pkg/apis/llm" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" @@ -32,9 +33,14 @@ type SLLMSkuBaseManager struct { type SLLMSkuBase struct { db.SSharableVirtualResourceBase - Bandwidth int `nullable:"false" default:"0" create:"optional" list:"user" update:"user"` - Cpu int `nullable:"false" default:"1" create:"optional" list:"user" update:"user"` - Memory int `nullable:"false" default:"512" create:"optional" list:"user" update:"user"` + Bandwidth int `nullable:"false" default:"0" create:"optional" list:"user" update:"user"` + Cpu int `nullable:"false" default:"1" create:"optional" list:"user" update:"user"` + Memory int `nullable:"false" default:"512" create:"optional" list:"user" update:"user"` + // VramClaimMb is the heuristic VRAM (MiB) needed to start a single SLLM + // instance from this SKU. Auto-filled from the largest mounted InstantModel's + // WeightSizeBytes via EstimateVramClaimMb; user can override at create/update + // time (any explicit non-zero value bypasses the auto-fill). 0 means unknown. + VramClaimMb int `nullable:"false" default:"0" create:"optional" list:"user" update:"user"` Volumes *api.Volumes `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` HostPaths *api.HostPaths `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` PortMappings *api.PortMappings `charset:"utf8" length:"medium" nullable:"true" list:"user" update:"user" create:"optional"` @@ -74,6 +80,18 @@ func (man *SLLMSkuBaseManager) ValidateCreateData(ctx context.Context, userCred return input, errors.Wrap(httperrors.ErrInputParameter, "volumes cannot be empty") } + // Default DevType to NVIDIA_GPU when callers omit it (UI's "auto by VRAM" + // path posts {} for each device). Without this the scheduler's + // (DevType, MemoryMb) aggregation key is empty and the VRAM filter + // silently no-ops. + if input.Devices != nil { + for i := range *input.Devices { + if (*input.Devices)[i].DevType == "" { + (*input.Devices)[i].DevType = computeapi.CONTAINER_DEV_NVIDIA_GPU_SHARE + } + } + } + input.Status = api.STATUS_READY return input, nil } diff --git a/pkg/llm/tasks/llm/llm_deployment_create_task.go b/pkg/llm/tasks/llm/llm_deployment_create_task.go index ca182b4d55..9aaa901d78 100644 --- a/pkg/llm/tasks/llm/llm_deployment_create_task.go +++ b/pkg/llm/tasks/llm/llm_deployment_create_task.go @@ -15,6 +15,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/llm/models" + "yunion.io/x/onecloud/pkg/llm/utils/vram" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -202,6 +203,26 @@ func (task *LLMDeploymentCreateTask) createSkuAndReconcile(ctx context.Context, skuSpec.Name = fmt.Sprintf("%s-sku", model.Name) } + // Auto-fill VramClaimMb from the largest mounted InstantModel's + // weight_size_bytes. User-provided non-zero values are respected. + if skuSpec.VramClaimMb == 0 { + var maxWeight int64 + for _, id := range skuSpec.MountedModels { + obj, err := models.GetInstantModelManager().FetchById(id) + if err != nil { + continue + } + if w := obj.(*models.SInstantModel).WeightSizeBytes; w > maxWeight { + maxWeight = w + } + } + if maxWeight > 0 { + skuSpec.VramClaimMb = vram.EstimateClaimMb(maxWeight, skuSpec.LLMType) + log.Infof("LLMDeploymentCreateTask: auto vram_claim_mb=%d for sku=%s (weight=%d bytes, llm_type=%s)", + skuSpec.VramClaimMb, skuSpec.Name, maxWeight, skuSpec.LLMType) + } + } + skuParams := jsonutils.Marshal(skuSpec).(*jsonutils.JSONDict) // __meta__ from VirtualResourceCreateInput marshals as null when Metadata is nil; // keep params clean to avoid surprises in the framework dispatcher. diff --git a/pkg/llm/tasks/llm/llm_instant_model_import_task.go b/pkg/llm/tasks/llm/llm_instant_model_import_task.go index d508cd2019..c7af661660 100644 --- a/pkg/llm/tasks/llm/llm_instant_model_import_task.go +++ b/pkg/llm/tasks/llm/llm_instant_model_import_task.go @@ -4,6 +4,7 @@ import ( "context" "yunion.io/x/jsonutils" + "yunion.io/x/log" imageapi "yunion.io/x/onecloud/pkg/apis/image" apis "yunion.io/x/onecloud/pkg/apis/llm" @@ -73,12 +74,50 @@ func (task *LLMInstantModelImportTask) OnImportComplete(ctx context.Context, obj } } + // Best-effort: estimate the model's weight-file size for downstream + // VRAM-claim calculation. Failure here is a warning, not a fatal — most + // consumers tolerate `weight_size_bytes = 0` (treated as "unknown"). + if model.WeightSizeBytes == 0 { + input := apis.InstantModelImportInput{} + if err := task.Params.Unmarshal(&input, "import_input"); err == nil { + if w := fetchWeightSizeForImport(ctx, input); w > 0 { + if _, err := db.Update(model, func() error { + model.WeightSizeBytes = w + return nil + }); err != nil { + log.Warningf("LLMInstantModelImportTask: persist weight_size_bytes for %s: %s", model.Name, err) + } else { + log.Infof("LLMInstantModelImportTask: %s weight_size_bytes=%d", model.Name, w) + } + } + } + } + db.OpsLog.LogEvent(model, db.ACT_CREATE, model.GetShortDesc(ctx), task.UserCred) logclient.AddActionLogWithStartable(task, model, logclient.ACT_CREATE, model.GetShortDesc(ctx), task.UserCred, true) task.SetStageComplete(ctx, nil) } +// fetchWeightSizeForImport dispatches by import source. Only HuggingFace is +// supported in this phase; ModelScope / local_path / ollama silently return 0 +// (left as TODO; UI handles the unknown case gracefully). +func fetchWeightSizeForImport(ctx context.Context, input apis.InstantModelImportInput) int64 { + if input.Source == apis.InstantModelSourceHuggingFace && input.RepoId != "" { + rev := input.Revision + if rev == "" { + rev = "main" + } + w, err := models.FetchHuggingFaceWeightSize(ctx, input.RepoId, rev) + if err != nil { + log.Warningf("LLMInstantModelImportTask: fetch HF weight size for %s@%s: %s", input.RepoId, rev, err) + return 0 + } + return w + } + return 0 +} + func (task *LLMInstantModelImportTask) OnImportCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) { model := obj.(*models.SInstantModel) diff --git a/pkg/llm/utils/vram/doc.go b/pkg/llm/utils/vram/doc.go new file mode 100644 index 0000000000..37e0cb836d --- /dev/null +++ b/pkg/llm/utils/vram/doc.go @@ -0,0 +1 @@ +package vram // import "yunion.io/x/onecloud/pkg/llm/utils/vram" diff --git a/pkg/llm/utils/vram/estimator.go b/pkg/llm/utils/vram/estimator.go new file mode 100644 index 0000000000..d92ac0d69d --- /dev/null +++ b/pkg/llm/utils/vram/estimator.go @@ -0,0 +1,66 @@ +package vram + +import api "yunion.io/x/onecloud/pkg/apis/llm" + +// VRAM-claim estimation, mirroring GPUStack's `estimate_model_vram()` in +// `gpustack/policies/utils.py`. +// +// Formula: +// +// LLM: VRAM ≈ weight_size * 1.2 + 2 GiB +// Embedding: VRAM ≈ weight_size * 1.2 + 512 MiB +// Image: VRAM ≈ weight_size (no factor, no overhead) +// +// Reference for the 20% factor: +// https://blog.eleuther.ai/transformer-math/#total-inference-memory +// +// Reference numbers (bf16, llm overhead): +// +// 0.5B → ~3.1 GiB +// 3B → ~8.9 GiB +// 7B → ~19.0 GiB +// 72B → ~164.5 GiB +const ( + activationOverheadFactor = 1.2 + llmFrameworkOverheadMB = 2048 // 2 GiB + nonLlmFrameworkOverheadMB = 512 // 512 MiB +) + +// EstimateClaimMb returns the heuristic VRAM requirement in MiB. +// weightSizeBytes <= 0 (i.e. unknown) → returns 0; callers treat 0 as +// "no constraint" / "schedule without VRAM check". +func EstimateClaimMb(weightSizeBytes int64, llmType string) int { + if weightSizeBytes <= 0 { + return 0 + } + weightMb := weightSizeBytes / (1024 * 1024) + + // Image / diffusion: weight only, no factor, no overhead. + if isImageLLMType(llmType) { + return int(weightMb) + } + + overhead := llmFrameworkOverheadMB + if !isLLMType(llmType) { + overhead = nonLlmFrameworkOverheadMB + } + return int(float64(weightMb)*activationOverheadFactor) + overhead +} + +// isLLMType reports whether the backend serves text-generation LLMs that get +// the larger framework overhead (CUDA graphs, runtime buffers, KV scratch). +func isLLMType(t string) bool { + switch api.LLMContainerType(t) { + case api.LLM_CONTAINER_VLLM, api.LLM_CONTAINER_OLLAMA, + api.LLM_CONTAINER_SGLANG, api.LLM_CONTAINER_HERMES_AGENT: + return true + } + return false +} + +// isImageLLMType reports whether the backend is a diffusion / image generation +// runtime — these skip the activation factor + framework overhead because the +// inference shape is dominated by weight tensors alone. +func isImageLLMType(t string) bool { + return api.LLMContainerType(t) == api.LLM_CONTAINER_COMFYUI +} diff --git a/pkg/llm/utils/vram/estimator_test.go b/pkg/llm/utils/vram/estimator_test.go new file mode 100644 index 0000000000..bb79f9da84 --- /dev/null +++ b/pkg/llm/utils/vram/estimator_test.go @@ -0,0 +1,83 @@ +package vram + +import ( + "testing" + + api "yunion.io/x/onecloud/pkg/apis/llm" +) + +func TestEstimateClaimMb(t *testing.T) { + gib := int64(1024 * 1024 * 1024) + cases := []struct { + name string + weightBytes int64 + llmType string + // minMb / maxMb form an acceptance band (the formula is heuristic; + // we don't pin exact bytes, just confirm we're in GPUStack's ballpark). + minMb int + maxMb int + }{ + // Reference numbers from GPUStack's docstring (bf16): + // 0.5B → ~3.1 GiB + // 3B → ~8.9 GiB + // 7B → ~19.0 GiB + // 72B → ~164.5 GiB + { + name: "0.5B bf16 vllm", + weightBytes: 1 * gib, + llmType: string(api.LLM_CONTAINER_VLLM), + minMb: 3000, maxMb: 3500, // 1024*1.2 + 2048 = 3276 + }, + { + name: "7B bf16 vllm", + weightBytes: 14 * gib, + llmType: string(api.LLM_CONTAINER_VLLM), + minMb: 18500, maxMb: 19500, // 14336*1.2 + 2048 = 19251 + }, + { + name: "72B bf16 vllm", + weightBytes: 145 * gib, + llmType: string(api.LLM_CONTAINER_VLLM), + minMb: 180000, maxMb: 180500, // 148480*1.2 + 2048 = 180224 + }, + { + name: "ollama same formula as vllm", + weightBytes: 1 * gib, + llmType: string(api.LLM_CONTAINER_OLLAMA), + minMb: 3000, maxMb: 3500, + }, + { + name: "comfyui (image) — no factor, no overhead", + weightBytes: 2 * gib, + llmType: string(api.LLM_CONTAINER_COMFYUI), + minMb: 2000, maxMb: 2100, // 2048 + }, + { + name: "unknown backend → non-llm 512 MiB overhead", + weightBytes: 1 * gib, + llmType: "dify", + minMb: 1700, maxMb: 1800, // 1024*1.2 + 512 = 1740 + }, + { + name: "weight unknown → 0", + weightBytes: 0, + llmType: string(api.LLM_CONTAINER_VLLM), + minMb: 0, maxMb: 0, + }, + { + name: "negative weight → 0", + weightBytes: -1, + llmType: string(api.LLM_CONTAINER_VLLM), + minMb: 0, maxMb: 0, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := EstimateClaimMb(c.weightBytes, c.llmType) + if got < c.minMb || got > c.maxMb { + t.Errorf("got %d MiB, want in [%d, %d]", got, c.minMb, c.maxMb) + } + }) + } +} diff --git a/pkg/mcclient/options/llm/instantmodel.go b/pkg/mcclient/options/llm/instantmodel.go index 5599f9eba6..0f36f9acc4 100644 --- a/pkg/mcclient/options/llm/instantmodel.go +++ b/pkg/mcclient/options/llm/instantmodel.go @@ -88,6 +88,14 @@ func (o *LLMInstantModelImportOptions) Params() (jsonutils.JSONObject, error) { return jsonutils.Marshal(input), nil } +type LLMInstantModelBackfillVramOptions struct { + DryRun bool `help:"preview without writing" json:"dry_run"` +} + +func (o *LLMInstantModelBackfillVramOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(api.InstantModelBackfillVramInput{DryRun: o.DryRun}), nil +} + type LLMInstantModelCommunityRegistryOptions struct { } diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go index 427dc7fb05..fb341f8db1 100644 --- a/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go @@ -74,6 +74,42 @@ func (f *IsolatedDevicePredicate) getIsolatedDeviceCountByType(getter core.Candi } } +// countDevicesWithMinMemory counts free devices of the given dev_type whose +// MemorySize satisfies the minimum requirement. Devices with MemorySize == 0 +// are treated as "unknown" and pass through (so newly-introduced rows that +// haven't been backfilled yet don't accidentally exclude every host). +// For NVIDIA_MPS / NVIDIA_GPU_SHARE the count is deduplicated by DevicePath, +// matching getIsolatedDeviceCountByType. +func (f *IsolatedDevicePredicate) countDevicesWithMinMemory(getter core.CandidatePropertyGetter, devType string, minMemoryMb int) int { + devs := getter.UnusedIsolatedDevicesByType(devType) + isShared := devType == compute.CONTAINER_DEV_NVIDIA_MPS || devType == compute.CONTAINER_DEV_NVIDIA_GPU_SHARE + return countDevicesWithMinMemoryFromList(devs, isShared, minMemoryMb) +} + +// countDevicesWithMinMemoryFromList is the pure-function core of the memory +// fit count, factored out for unit testing. Callers pass an already-filtered +// list (typically by dev_type). +func countDevicesWithMinMemoryFromList(devs []*core.IsolatedDeviceDesc, isShared bool, minMemoryMb int) int { + if !isShared { + n := 0 + for _, d := range devs { + if d.MemorySize > 0 && d.MemorySize < minMemoryMb { + continue + } + n++ + } + return n + } + seen := map[string]struct{}{} + for _, d := range devs { + if d.MemorySize > 0 && d.MemorySize < minMemoryMb { + continue + } + seen[d.DevicePath] = struct{}{} + } + return len(seen) +} + func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { h := NewPredicateHelper(f, u, c) reqIsoDevs := u.SchedData().IsolatedDevices @@ -164,6 +200,35 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c } } + // check host device by (type, min_memory_mb) — VRAM-aware fit for GPUs. + // LLM scheduling stamps MemoryMb on each request entry so a SKU's + // vram_claim_mb is honoured. Devices with memory_size == 0 are passed + // through as unknown (see countDevicesWithMinMemory). + type vramReqKey struct { + devType string + minMemMb int + } + vramReq := make(map[vramReqKey]int) + for _, dev := range reqIsoDevs { + if dev.MemoryMb <= 0 { + continue + } + vramReq[vramReqKey{dev.DevType, dev.MemoryMb}]++ + } + for k, reqCnt := range vramReq { + fit := f.countDevicesWithMinMemory(getter, k.devType, k.minMemMb) + if fit < reqCnt { + h.Exclude(fmt.Sprintf( + "IsolatedDevice type %q with memory >= %d MiB not enough, request: %d, hostFree: %d", + k.devType, k.minMemMb, reqCnt, fit)) + return h.GetResult() + } + cap := fit / reqCnt + if int64(cap) < minCapacity { + minCapacity = int64(cap) + } + } + // check host device by device_path devicePathReq := make(map[string]int, 0) for _, dev := range reqIsoDevs { diff --git a/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go new file mode 100644 index 0000000000..2cc85907ef --- /dev/null +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go @@ -0,0 +1,97 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package predicates + +import ( + "testing" + + "yunion.io/x/onecloud/pkg/scheduler/core" +) + +func TestCountDevicesWithMinMemoryFromList(t *testing.T) { + mk := func(path string, memMb int) *core.IsolatedDeviceDesc { + return &core.IsolatedDeviceDesc{DevicePath: path, MemorySize: memMb} + } + + cases := []struct { + name string + devs []*core.IsolatedDeviceDesc + shared bool + minMemMb int + want int + }{ + { + name: "plain GPU: 3 cards 24/40/80 GiB, request 30 GiB → 2 fit", + devs: []*core.IsolatedDeviceDesc{ + mk("/dev/nvidia0", 24576), + mk("/dev/nvidia1", 40960), + mk("/dev/nvidia2", 81920), + }, + shared: false, minMemMb: 30000, want: 2, + }, + { + name: "plain GPU: request 0 (unconstrained) → all pass through", + devs: []*core.IsolatedDeviceDesc{ + mk("/dev/nvidia0", 24576), + mk("/dev/nvidia1", 40960), + }, + shared: false, minMemMb: 0, want: 2, + }, + { + name: "unknown MemorySize=0 → passes as unknown (avoid mass exclusion)", + devs: []*core.IsolatedDeviceDesc{ + mk("/dev/nvidia0", 0), + mk("/dev/nvidia1", 24576), + }, + shared: false, minMemMb: 40000, want: 1, // unknown stays in, 24GiB excluded + }, + { + name: "MPS share: 2 physical cards, 4 slices each, only 1 card meets req", + devs: []*core.IsolatedDeviceDesc{ + // card 0: 6 GiB per slice (4 slices × same path) + mk("/dev/nvidia0", 6144), mk("/dev/nvidia0", 6144), + mk("/dev/nvidia0", 6144), mk("/dev/nvidia0", 6144), + // card 1: 20 GiB per slice + mk("/dev/nvidia1", 20480), mk("/dev/nvidia1", 20480), + mk("/dev/nvidia1", 20480), mk("/dev/nvidia1", 20480), + }, + shared: true, minMemMb: 10000, want: 1, // only card 1 satisfies + }, + { + name: "MPS share: all slices pass through dedup → count by DevicePath", + devs: []*core.IsolatedDeviceDesc{ + mk("/dev/nvidia0", 24576), mk("/dev/nvidia0", 24576), + mk("/dev/nvidia1", 24576), + }, + shared: true, minMemMb: 10000, want: 2, // 2 distinct paths + }, + { + name: "empty pool → 0", + devs: []*core.IsolatedDeviceDesc{}, + shared: false, + minMemMb: 1000, + want: 0, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := countDevicesWithMinMemoryFromList(c.devs, c.shared, c.minMemMb) + if got != c.want { + t.Errorf("got %d, want %d", got, c.want) + } + }) + } +} diff --git a/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go b/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go new file mode 100644 index 0000000000..253ba04de9 --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack.go @@ -0,0 +1,92 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guest + +import ( + "yunion.io/x/onecloud/pkg/scheduler/algorithm/priorities" + "yunion.io/x/onecloud/pkg/scheduler/core" +) + +// GPUVramBinpackPriority prefers hosts whose free GPUs have memory close to +// the request's MemoryMb (bin-packing): avoids landing a small model on a +// large GPU when a tighter-fitting candidate exists. Hosts without a memory +// constraint in the request score 0 — they don't participate. +type GPUVramBinpackPriority struct { + priorities.BasePriority +} + +func (p *GPUVramBinpackPriority) Name() string { + return "guest-gpu-vram-binpack" +} + +func (p *GPUVramBinpackPriority) Clone() core.Priority { + return &GPUVramBinpackPriority{} +} + +func (p *GPUVramBinpackPriority) Map(u *core.Unit, c core.Candidater) (core.HostPriority, error) { + h := priorities.NewPriorityHelper(p, u, c) + + // Take the maximum MemoryMb across the request — for an LLM SKU this + // is the per-device share of vram_claim_mb (all entries equal); for mixed + // requests we treat the largest as the bin-pack target. + perDevMin := 0 + for _, d := range u.SchedData().IsolatedDevices { + if d.MemoryMb > perDevMin { + perDevMin = d.MemoryMb + } + } + if perDevMin == 0 { + // No VRAM constraint — don't influence ranking. + return h.GetResult() + } + + score := scoreVramBinpack(c.Getter().UnusedIsolatedDevices(), perDevMin) + h.SetScore(score) + return h.GetResult() +} + +// scoreVramBinpack is the pure scoring core, factored out so it can be unit +// tested without constructing a CandidatePropertyGetter mock. Average the +// VRAM of free GPUs that satisfy the request, then score 100 * perDevMin / +// avg — tighter fit ranks higher. Devices with MemorySize == 0 are neutral +// (don't influence the average) so unreported rows don't skew rankings. +func scoreVramBinpack(devs []*core.IsolatedDeviceDesc, perDevMin int) int { + if perDevMin <= 0 { + return 0 + } + var totalFitVram, fitCount int + for _, d := range devs { + if d.MemorySize <= 0 { + continue + } + if d.MemorySize < perDevMin { + continue + } + totalFitVram += d.MemorySize + fitCount++ + } + if fitCount == 0 { + return 0 + } + avgVram := totalFitVram / fitCount + score := 100 * perDevMin / avgVram + if score < 0 { + score = 0 + } + if score > 100 { + score = 100 + } + return score +} diff --git a/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack_test.go b/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack_test.go new file mode 100644 index 0000000000..3042f2f1fc --- /dev/null +++ b/pkg/scheduler/algorithm/priorities/guest/gpu_vram_binpack_test.go @@ -0,0 +1,104 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package guest + +import ( + "testing" + + "yunion.io/x/onecloud/pkg/scheduler/core" +) + +func TestScoreVramBinpack(t *testing.T) { + mk := func(memMb int) *core.IsolatedDeviceDesc { + return &core.IsolatedDeviceDesc{MemorySize: memMb} + } + + cases := []struct { + name string + devs []*core.IsolatedDeviceDesc + perDevMin int + // score is approximate; assert it falls in [minScore, maxScore]. + minScore int + maxScore int + }{ + { + name: "tight fit (24GiB request on 24GiB GPUs) → near 100", + devs: []*core.IsolatedDeviceDesc{mk(24576), mk(24576)}, + perDevMin: 24576, + minScore: 100, maxScore: 100, + }, + { + name: "loose fit (24GiB request on 80GiB GPUs) → low", + devs: []*core.IsolatedDeviceDesc{mk(81920), mk(81920)}, + perDevMin: 24576, + minScore: 25, maxScore: 35, // 24576/81920 ≈ 30 + }, + { + name: "mixed: ignore non-fitting; score from fitting only", + devs: []*core.IsolatedDeviceDesc{mk(10000), mk(24576)}, // 10G excluded (< 20G req) + perDevMin: 20000, + minScore: 80, maxScore: 85, // 20000/24576 ≈ 81 + }, + { + name: "unknown MemorySize=0 → ignored, no influence", + devs: []*core.IsolatedDeviceDesc{mk(0), mk(0), mk(24576)}, + perDevMin: 24576, + minScore: 100, maxScore: 100, // single 24G GPU → tight + }, + { + name: "no fitting devices → 0", + devs: []*core.IsolatedDeviceDesc{mk(8000)}, + perDevMin: 40000, + minScore: 0, maxScore: 0, + }, + { + name: "request 0 (unconstrained) → 0 (don't bias)", + devs: []*core.IsolatedDeviceDesc{mk(24576)}, + perDevMin: 0, + minScore: 0, maxScore: 0, + }, + { + name: "host A (24G average) vs host B (80G average) — A scores higher", + devs: []*core.IsolatedDeviceDesc{mk(24576)}, + perDevMin: 20480, + minScore: 80, maxScore: 100, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := scoreVramBinpack(c.devs, c.perDevMin) + if got < c.minScore || got > c.maxScore { + t.Errorf("got score %d, want in [%d, %d]", got, c.minScore, c.maxScore) + } + }) + } +} + +// Verify the A vs B ordering claim from the plan: same request lands +// higher-scoring on the tight-fit host than on the loose-fit host. +func TestScoreVramBinpack_HostOrdering(t *testing.T) { + mk := func(memMb int) *core.IsolatedDeviceDesc { + return &core.IsolatedDeviceDesc{MemorySize: memMb} + } + req := 20480 // 20 GiB + hostA := []*core.IsolatedDeviceDesc{mk(24576), mk(24576)} // tight + hostB := []*core.IsolatedDeviceDesc{mk(81920), mk(81920)} // loose + sA := scoreVramBinpack(hostA, req) + sB := scoreVramBinpack(hostB, req) + if sA <= sB { + t.Errorf("expected tight-fit host A (%d) > loose-fit host B (%d)", sA, sB) + } +} diff --git a/pkg/scheduler/algorithmprovider/defaults.go b/pkg/scheduler/algorithmprovider/defaults.go index f6d225a73d..2e0867e80f 100644 --- a/pkg/scheduler/algorithmprovider/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults.go @@ -59,6 +59,7 @@ func defaultPriorities() sets.String { factory.RegisterPriority("guest-lowload", &priorityguest.LowLoadPriority{}, 1), factory.RegisterPriority("guest-creating", &priorityguest.CreatingPriority{}, 1), factory.RegisterPriority("guest-capacity", &priorityguest.CapacityPriority{}, 1), + factory.RegisterPriority("guest-gpu-vram-binpack", &priorityguest.GPUVramBinpackPriority{}, 1), factory.RegisterPriority("guest-cpunumapin", &priorityguest.CpuNumaPinPriority{}, 1), ) } diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index 70b4aaece8..749f2759ab 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -588,6 +588,7 @@ func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.S VendorDeviceID: devModel.VendorDeviceId, WireId: devModel.WireId, DevicePath: devModel.DevicePath, + MemorySize: devModel.MemorySize, } devs[index] = dev } diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 202ed968e3..b552a93b99 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -278,6 +278,9 @@ type IsolatedDeviceDesc struct { VendorDeviceID string WireId string DevicePath string + // MemorySize is the on-device memory in MiB (NVIDIA GPU VRAM via + // `nvidia-smi memory.total`). 0 means unknown / not yet reported by host. + MemorySize int } func (i *IsolatedDeviceDesc) VendorID() string {