opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-10-30 18:23:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(scheduler): report HAMI hostFree before min-memory filter (#25372)

Browse Source 
Shortage messages now show remaining GPU memory from matched devices
before min-memory filtering, so users see actual free capacity instead of 0.
This commit is contained in:
Zexi Li
2026-08-19 16:40:51 +08:00
committed by GitHub
parent f2ffa5d034
commit 29f65063b7
2 changed files with 57 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/scheduler/algorithm/predicates/isolated_device_predicate.go
+22 -19
View File
@@ -64,6 +64,14 @@ func (f *IsolatedDevicePredicate) PreExecute(ctx context.Context, u *core.Unit,
}
func (f *IsolatedDevicePredicate) getIsolatedDeviceCountBySharingMode(sharingMode string, devs []*core.IsolatedDeviceDesc) int {
return isolatedDeviceHostFree(sharingMode, devs)
}
// isolatedDeviceHostFree is the value shown as hostFree in shortage messages.
// HAMI: sum of (memory_size - memory_size_allocated) on the matched pool,
// counted before min-memory filtering so remaining-but-too-small cards are
// not reported as 0.
func isolatedDeviceHostFree(sharingMode string, devs []*core.IsolatedDeviceDesc) int {
if sharingMode == compute.DEVICE_SHARING_MODE_HAMI {
ret := 0
for i := range devs {
@@ -295,17 +303,18 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c
}
for key, reqCount := range devTypeRequest {
devType, sharingMode := key.devType, key.sharingMode
devs := getter.AvailableIsolatedDevicesByTypeSharingMode(devType, sharingMode)
matched := getter.AvailableIsolatedDevicesByTypeSharingMode(devType, sharingMode)
match := func(d *compute.IsolatedDeviceConfig) bool {
return d.DevType == devType && d.SharingMode == sharingMode
}
minMem := maxMinMemoryForRequests(reqIsoDevs, match)
devs = filterDescsByMinMemory(devs, sharingMode, minMem)
hostFree := isolatedDeviceHostFree(sharingMode, matched)
devs := filterDescsByMinMemory(matched, sharingMode, minMem)
pendingCnt := pendingUsage.Get(path.Join(devType, sharingMode))
freeCount := f.getIsolatedDeviceCountBySharingMode(sharingMode, devs)
if freeCount < (reqCount + pendingCnt) {
spec := shortageSpecFromRequests(reqIsoDevs, match, devType, sharingMode, minMem)
h.Exclude(isolatedDeviceShortageMessage(spec, reqCount, freeCount, pendingCnt))
h.Exclude(isolatedDeviceShortageMessage(spec, reqCount, hostFree, pendingCnt))
return h.GetResult()
}
cap := freeCount / reqCount
@@ -337,22 +346,19 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c
}
}
for key, reqCount := range devVendorModelRequest {
devs := filterDevicesByTypeSharingMode(getter.AvailableIsolatedDevicesByVendorModel(key.vendorModel), key.devType, key.sharingMode)
matched := filterDevicesByTypeSharingMode(getter.AvailableIsolatedDevicesByVendorModel(key.vendorModel), key.devType, key.sharingMode)
match := func(d *compute.IsolatedDeviceConfig) bool {
return fmt.Sprintf("%s:%s", d.Vendor, d.Model) == key.vendorModel &&
d.DevType == key.devType && d.SharingMode == key.sharingMode
}
minMem := maxMinMemoryForRequests(reqIsoDevs, match)
devs = filterDescsByMinMemory(devs, key.sharingMode, minMem)
hostFree := isolatedDeviceHostFree(key.sharingMode, matched)
devs := filterDescsByMinMemory(matched, key.sharingMode, minMem)
spec := shortageSpecFromRequests(reqIsoDevs, match, key.devType, key.sharingMode, minMem)
if len(devs) == 0 {
h.Exclude(isolatedDeviceShortageMessage(spec, reqCount, 0, 0))
return h.GetResult()
}
pendingCnt := pendingUsage.Get(path.Join(key.devType, key.sharingMode))
freeCount := f.getIsolatedDeviceCountBySharingMode(key.sharingMode, devs)
if freeCount < (reqCount + pendingCnt) {
h.Exclude(isolatedDeviceShortageMessage(spec, reqCount, freeCount, pendingCnt))
if len(devs) == 0 || freeCount < (reqCount+pendingCnt) {
h.Exclude(isolatedDeviceShortageMessage(spec, reqCount, hostFree, pendingCnt))
return h.GetResult()
}
cap := freeCount / reqCount
@@ -419,22 +425,19 @@ func (f *IsolatedDevicePredicate) Execute(ctx context.Context, u *core.Unit, c c
}
}
for key, reqCnt := range devicePathReq {
devs := filterDevicesByTypeSharingMode(getter.AvailableIsolatedDevicesByDevicePath(key.devicePath), key.devType, key.sharingMode)
matched := filterDevicesByTypeSharingMode(getter.AvailableIsolatedDevicesByDevicePath(key.devicePath), key.devType, key.sharingMode)
match := func(d *compute.IsolatedDeviceConfig) bool {
return d.DevicePath == key.devicePath && d.DevType == key.devType && d.SharingMode == key.sharingMode
}
minMem := maxMinMemoryForRequests(reqIsoDevs, match)
devs = filterDescsByMinMemory(devs, key.sharingMode, minMem)
hostFree := isolatedDeviceHostFree(key.sharingMode, matched)
devs := filterDescsByMinMemory(matched, key.sharingMode, minMem)
spec := shortageSpecFromRequests(reqIsoDevs, match, key.devType, key.sharingMode, minMem)
spec.devicePath = key.devicePath
if len(devs) == 0 {
h.Exclude(isolatedDeviceShortageMessage(spec, reqCnt, 0, 0))
return h.GetResult()
}
pendingCnt := pendingUsage.Get(path.Join(key.devType, key.sharingMode))
freeCount := f.getIsolatedDeviceCountBySharingMode(key.sharingMode, devs)
if freeCount < (reqCnt + pendingCnt) {
h.Exclude(isolatedDeviceShortageMessage(spec, reqCnt, freeCount, pendingCnt))
if len(devs) == 0 || freeCount < (reqCnt+pendingCnt) {
h.Exclude(isolatedDeviceShortageMessage(spec, reqCnt, hostFree, pendingCnt))
return h.GetResult()
}
cap := freeCount / reqCnt
pkg/scheduler/algorithm/predicates/isolated_device_predicate_test.go
+35
View File
@@ -181,6 +181,41 @@ func TestMaxMinMemoryForRequests(t *testing.T) {
}
}
func TestIsolatedDeviceHostFreeHAMIRemainingBeforeMinMemFilter(t *testing.T) {
// Candidate: 16311 total, 11630 allocated → 4681 remaining, below request 5815.
dev := &core.IsolatedDeviceDesc{
DevType: computeapi.GPU_TYPE,
SharingMode: computeapi.DEVICE_SHARING_MODE_HAMI,
Model: "GeForce RTX 5060 Ti",
MemorySize: 16311,
MemorySizeAllocated: 11630,
VirtualNum: 1,
VirtualNumAllocated: 2,
}
matched := []*core.IsolatedDeviceDesc{dev}
hostFree := isolatedDeviceHostFree(computeapi.DEVICE_SHARING_MODE_HAMI, matched)
if hostFree != 4681 {
t.Fatalf("hostFree = %d, want 4681 (memory_size - memory_size_allocated)", hostFree)
}
fit := filterDescsByMinMemory(matched, computeapi.DEVICE_SHARING_MODE_HAMI, 5815)
if len(fit) != 0 {
t.Fatalf("card with 4681 remaining must not satisfy 5815, got %d", len(fit))
}
spec := shortageSpecFromRequests([]*computeapi.IsolatedDeviceConfig{
{
DevType: computeapi.GPU_TYPE, Vendor: "NVIDIA", Model: "GeForce RTX 5060 Ti",
SharingMode: computeapi.DEVICE_SHARING_MODE_HAMI, MemoryMb: 5815, MemoryRequest: 5815,
},
}, func(d *computeapi.IsolatedDeviceConfig) bool {
return d.DevType == computeapi.GPU_TYPE && d.SharingMode == computeapi.DEVICE_SHARING_MODE_HAMI
}, computeapi.GPU_TYPE, computeapi.DEVICE_SHARING_MODE_HAMI, 5815)
got := isolatedDeviceShortageMessage(spec, 5815, hostFree, 0)
want := `IsolatedDevice type "GPU" vendor "NVIDIA" model "GeForce RTX 5060 Ti" sharing_mode "HAMI" memory>=5815MiB not enough, request: 5815 MiB, hostFree: 4681 MiB`
if got != want {
t.Fatalf("got:\n%s\nwant:\n%s", got, want)
}
}
func TestIsolatedDeviceShortageMessage(t *testing.T) {
hamiMatchGPU := func(d *computeapi.IsolatedDeviceConfig) bool {
return d.DevType == computeapi.GPU_TYPE && d.SharingMode == computeapi.DEVICE_SHARING_MODE_HAMI