From b7f813512cea179ebc2d99b43bbc68f6674e96ba Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Sat, 19 Dec 2020 16:46:40 +0800 Subject: [PATCH] fix(scheduler): storage predicate add actualCapacity filter --- cmd/climc/shell/compute/storages.go | 2 +- pkg/mcclient/modules/mod_storages.go | 2 +- .../predicates/guest/storage_predicate.go | 69 ++++++++++++++----- pkg/scheduler/api/sched.go | 17 +++-- pkg/scheduler/cache/candidate/base.go | 6 +- pkg/scheduler/cache/candidate/hosts.go | 13 ++-- pkg/scheduler/core/types.go | 2 +- pkg/scheduler/handler/handler.go | 10 ++- pkg/scheduler/test/mock/core.go | 5 +- pkg/scheduler/test/prepare.go | 2 +- 10 files changed, 92 insertions(+), 36 deletions(-) diff --git a/cmd/climc/shell/compute/storages.go b/cmd/climc/shell/compute/storages.go index 088f4dab8d..c300e74147 100644 --- a/cmd/climc/shell/compute/storages.go +++ b/cmd/climc/shell/compute/storages.go @@ -61,7 +61,7 @@ func init() { ID string `help:"ID or Name of storage to update"` Name string `help:"New Name of storage"` Desc string `help:"Description"` - CommitBound float64 `help:"Upper bound of storage overcommit rate"` + CommitBound float64 `help:"Upper bound of storage overcommit rate" json:"cmtbound"` MediumType string `help:"Medium type" choices:"ssd|rotate"` RbdRadosMonOpTimeout int64 `help:"ceph rados_mon_op_timeout"` RbdRadosOsdOpTimeout int64 `help:"ceph rados_osd_op_timeout"` diff --git a/pkg/mcclient/modules/mod_storages.go b/pkg/mcclient/modules/mod_storages.go index 0d59d366ba..702fad625a 100644 --- a/pkg/mcclient/modules/mod_storages.go +++ b/pkg/mcclient/modules/mod_storages.go @@ -22,7 +22,7 @@ var ( func init() { Storages = NewComputeManager("storage", "storages", - []string{"ID", "Name", "Capacity", "Status", "Used_capacity", "Waste_capacity", "Free_capacity", "Storage_type", "Medium_type", "Virtual_capacity", "commit_bound", "commit_rate", "Enabled", "public_scope"}, + []string{"ID", "Name", "Capacity", "Actual_capacity_used", "Status", "Used_capacity", "Waste_capacity", "Free_capacity", "Storage_type", "Medium_type", "Virtual_capacity", "commit_bound", "commit_rate", "Enabled", "public_scope"}, []string{}) registerCompute(&Storages) diff --git a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go index 2800e54fbe..dda344a066 100644 --- a/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/storage_predicate.go @@ -72,11 +72,26 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return false } - getStorageCapacity := func(backend string, reqMaxSize int64, reqTotalSize int64, useRsvd bool) (int64, int64) { - totalFree := getter.GetFreeStorageSizeOfType(backend, useRsvd) - capacity := totalFree / utils.Max(reqTotalSize, 1) + type storageCapacity struct { + capacity int64 + free int64 + isActual bool + } - return capacity, totalFree + newStorageCapacity := func(capacity int64, free int64, isActual bool) *storageCapacity { + return &storageCapacity{ + capacity: capacity, + free: free, + isActual: isActual, + } + } + + getStorageCapacity := func(backend string, reqMaxSize int64, reqTotalSize int64, useRsvd bool) (*storageCapacity, *storageCapacity) { + totalFree, actualFree := getter.GetFreeStorageSizeOfType(backend, useRsvd) + reqTotalSize = utils.Max(reqTotalSize, 1) + capacity := totalFree / reqTotalSize + actualCapacity := actualFree / reqTotalSize + return newStorageCapacity(capacity, totalFree, false), newStorageCapacity(actualCapacity, actualFree, true) } getReqSizeStr := func(backend string) string { @@ -90,15 +105,21 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor return strings.Join(ss, "+") } - getStorageFreeStr := func(backend string, useRsvd bool) string { + getStorageFreeStr := func(backend string, useRsvd bool, isActual bool) string { ss := []string{} for _, s := range getter.Storages() { if s.StorageType == backend { - total := int64(float32(s.Capacity) * s.Cmtbound) - used := s.GetUsedCapacity(tristate.True) - waste := s.GetUsedCapacity(tristate.False) - free := total - int64(used) - int64(waste) - ss = append(ss, fmt.Sprintf("(%v-%v-%v=%v)", total, used, waste, free)) + if isActual { + total := s.Capacity + free := total - s.ActualCapacityUsed + ss = append(ss, fmt.Sprintf("actual_total:%d - actual_used:%d = free:%d", total, s.ActualCapacityUsed, free)) + } else { + total := int64(float32(s.Capacity) * s.Cmtbound) + used := s.GetUsedCapacity(tristate.True) + waste := s.GetUsedCapacity(tristate.False) + free := total - int64(used) - int64(waste) + ss = append(ss, fmt.Sprintf("total:%d - used:%d - waste:%d = free:%d", total, used, waste, free)) + } } } return strings.Join(ss, " + ") @@ -130,14 +151,28 @@ func (p *StoragePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []cor useRsvd := h.UseReserved() minCapacity := int64(0xFFFFFFFF) - for be, req := range sizeRequest { - capacity, totalFree := getStorageCapacity(be, req["max"], req["total"], useRsvd) - if capacity == 0 { - s := fmt.Sprintf("no enough %q storage, req=%v(%v), free=%v(%v)", - be, req["total"], getReqSizeStr(be), totalFree, getStorageFreeStr(be, useRsvd)) - h.AppendPredicateFailMsg(s) + + appendFailMsg := func(backend string, req map[string]int64, useRsvd bool, capacity *storageCapacity) { + reqStr := fmt.Sprintf("no enough %q storage, req=%v(%v)", backend, req["total"], getReqSizeStr(backend)) + freePrex := "free" + isActual := capacity.isActual + if isActual { + freePrex = "actual_free" } - minCapacity = utils.Min(minCapacity, capacity) + freeStr := fmt.Sprintf("%s=%v(%v)", freePrex, capacity.free, getStorageFreeStr(backend, useRsvd, isActual)) + msg := reqStr + ", " + freeStr + h.AppendPredicateFailMsg(msg) + } + + for be, req := range sizeRequest { + capacity, actualCapacity := getStorageCapacity(be, req["max"], req["total"], useRsvd) + tmpCap := utils.Min(capacity.capacity, actualCapacity.capacity) + if capacity.capacity <= 0 { + appendFailMsg(be, req, useRsvd, capacity) + } else if actualCapacity.capacity <= 0 { + appendFailMsg(be, req, useRsvd, actualCapacity) + } + minCapacity = utils.Min(minCapacity, tmpCap) } h.SetCapacity(minCapacity) diff --git a/pkg/scheduler/api/sched.go b/pkg/scheduler/api/sched.go index a1867e0d9d..a26e2ad932 100644 --- a/pkg/scheduler/api/sched.go +++ b/pkg/scheduler/api/sched.go @@ -53,7 +53,7 @@ type SchedInfo struct { UserCred mcclient.TokenCredential } -func fetchAuthToken(req *http.Request) (mcclient.TokenCredential, error) { +func FetchAuthToken(req *http.Request) (mcclient.TokenCredential, error) { tokenStr := req.Header.Get(identity.AUTH_TOKEN_HEADER) if tokenStr == "" { return nil, errors.Wrap(httperrors.ErrInvalidCredential, "missing token header") @@ -65,11 +65,20 @@ func fetchAuthToken(req *http.Request) (mcclient.TokenCredential, error) { return token, nil } -func FetchSchedInfo(req *http.Request) (*SchedInfo, error) { - token, err := fetchAuthToken(req) +func FetchUserCred(req *http.Request) (mcclient.TokenCredential, error) { + token, err := FetchAuthToken(req) if err != nil { return nil, errors.Wrap(err, "fetchAuthToken") } + userCred := policy.FilterPolicyCredential(token) + return userCred, nil +} + +func FetchSchedInfo(req *http.Request) (*SchedInfo, error) { + userCred, err := FetchUserCred(req) + if err != nil { + return nil, errors.Wrap(err, "fetch user cred") + } body, err := appsrv.FetchJSON(req) if err != nil { @@ -84,7 +93,7 @@ func FetchSchedInfo(req *http.Request) (*SchedInfo, error) { input = models.ApplySchedPolicies(input) data := NewSchedInfo(input) - data.UserCred = policy.FilterPolicyCredential(token) + data.UserCred = userCred domainId := data.Domain for _, net := range data.Networks { diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index df88ad9a60..a0e00cd105 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -207,14 +207,16 @@ func (b baseHostGetter) TotalMemorySize(_ bool) int64 { return int64(b.h.MemSize) } -func (b baseHostGetter) GetFreeStorageSizeOfType(storageType string, useRsvd bool) int64 { +func (b baseHostGetter) GetFreeStorageSizeOfType(storageType string, useRsvd bool) (int64, int64) { var size int64 + var actualSize int64 for _, s := range b.Storages() { if s.StorageType == storageType { size += int64(float32(s.Capacity) * s.Cmtbound) + actualSize += s.Capacity - s.ActualCapacityUsed } } - return size + return size, actualSize } func (b baseHostGetter) GetFreePort(netId string) int { diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index 6126ce7627..1be829cdde 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -85,7 +85,7 @@ func (h *hostGetter) StorageInfo() []*baremetal.BaremetalStorage { return nil } -func (h *hostGetter) GetFreeStorageSizeOfType(storageType string, useRsvd bool) int64 { +func (h *hostGetter) GetFreeStorageSizeOfType(storageType string, useRsvd bool) (int64, int64) { return h.h.GetFreeStorageSizeOfType(storageType, useRsvd) } @@ -317,15 +317,18 @@ func (h *HostDesc) freeStorageSize(onlyLocal, useRsvd bool) int64 { return total } -func (h *HostDesc) GetFreeStorageSizeOfType(sType string, useRsvd bool) int64 { +func (h *HostDesc) GetFreeStorageSizeOfType(sType string, useRsvd bool) (int64, int64) { return h.freeStorageSizeOfType(sType, useRsvd) } -func (h *HostDesc) freeStorageSizeOfType(storageType string, useRsvd bool) int64 { +func (h *HostDesc) freeStorageSizeOfType(storageType string, useRsvd bool) (int64, int64) { var total int64 + var actualTotal int64 + for _, storage := range h.Storages { if storage.StorageType == storageType { total += int64(storage.GetFreeCapacity()) + actualTotal += storage.Capacity - storage.ActualCapacityUsed } } if utils.IsLocalStorage(storageType) { @@ -336,10 +339,10 @@ func (h *HostDesc) freeStorageSizeOfType(storageType string, useRsvd bool) int64 } } if useRsvd { - return reservedResourceAddCal(total, h.GuestReservedStorageSizeFree(), useRsvd) + return reservedResourceAddCal(total, h.GuestReservedStorageSizeFree(), useRsvd), actualTotal } - return total - int64(h.GetPendingUsage().DiskUsage.Get(storageType)) + return total - int64(h.GetPendingUsage().DiskUsage.Get(storageType)), actualTotal } func (h *HostDesc) GetFreePort(netId string) int { diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index d4edd9004a..a936944fdb 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -97,7 +97,7 @@ type CandidatePropertyGetter interface { FreeMemorySize(useRsvd bool) int64 StorageInfo() []*baremetal.BaremetalStorage - GetFreeStorageSizeOfType(storageType string, useRsvd bool) int64 + GetFreeStorageSizeOfType(storageType string, useRsvd bool) (int64, int64) GetFreePort(netId string) int diff --git a/pkg/scheduler/handler/handler.go b/pkg/scheduler/handler/handler.go index 216d60ed66..6015d5ebd6 100644 --- a/pkg/scheduler/handler/handler.go +++ b/pkg/scheduler/handler/handler.go @@ -162,7 +162,13 @@ func doCandidateList(c *gin.Context) { } func doCandidateDetail(c *gin.Context, id string) { - hs, err := computemodels.HostManager.FetchById(id) + userCred, err := api.FetchUserCred(c.Request) + if err != nil { + c.AbortWithError(http.StatusBadRequest, err) + return + } + + hs, err := computemodels.HostManager.FetchByIdOrName(userCred, id) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return @@ -176,7 +182,7 @@ func doCandidateDetail(c *gin.Context, id string) { host := hs.(*computemodels.SHost) args := new(api.CandidateDetailArgs) - args.ID = id + args.ID = host.GetId() if host.HostType == computeapi.HOST_TYPE_BAREMETAL { args.Type = api.HostTypeBaremetal } else { diff --git a/pkg/scheduler/test/mock/core.go b/pkg/scheduler/test/mock/core.go index 61b7788645..fcbb1df47e 100644 --- a/pkg/scheduler/test/mock/core.go +++ b/pkg/scheduler/test/mock/core.go @@ -13,6 +13,7 @@ package mock import ( + "math" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -165,11 +166,11 @@ func (mr *MockCandidatePropertyGetterMockRecorder) GetFreePort(arg0 interface{}) } // GetFreeStorageSizeOfType mocks base method -func (m *MockCandidatePropertyGetter) GetFreeStorageSizeOfType(arg0 string, arg1 bool) int64 { +func (m *MockCandidatePropertyGetter) GetFreeStorageSizeOfType(arg0 string, arg1 bool) (int64, int64) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetFreeStorageSizeOfType", arg0, arg1) ret0, _ := ret[0].(int64) - return ret0 + return ret0, math.MaxInt64 } // GetFreeStorageSizeOfType indicates an expected call of GetFreeStorageSizeOfType diff --git a/pkg/scheduler/test/prepare.go b/pkg/scheduler/test/prepare.go index 5399897700..0a90334429 100644 --- a/pkg/scheduler/test/prepare.go +++ b/pkg/scheduler/test/prepare.go @@ -169,7 +169,7 @@ func buildGetter(ctrl *gomock.Controller, param sGetterParams) *mock.MockCandida cg.EXPECT().FreeCPUCount(gomock.Any()).AnyTimes().Return(param.FreeCPUCount) cg.EXPECT().TotalMemorySize(gomock.Any()).AnyTimes().Return(param.TotalMemorySize) cg.EXPECT().FreeMemorySize(gomock.Any()).AnyTimes().Return(param.FreeMemorySize) - cg.EXPECT().GetFreeStorageSizeOfType(gomock.Any(), gomock.Any()).AnyTimes().Return(param.FreeStorageSizeAnyType) + cg.EXPECT().GetFreeStorageSizeOfType(gomock.Any(), gomock.Any()).AnyTimes().Return(param.FreeStorageSizeAnyType, int64(0)) cg.EXPECT().GetFreePort(gomock.Any()).AnyTimes().Return(param.FreePort) if param.QuotaKeys != nil { cg.EXPECT().GetQuotaKeys(gomock.Any()).AnyTimes().Return(param.QuotaKeys)