From 3d0c09195e6dfc85491e5c336c43c43a37b9b156 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Wed, 12 Oct 2022 14:57:13 +0800 Subject: [PATCH] feat(region,host): hugepage optimize - host agent will not allocate hugepage on init - except hugepage, the remaining memory update to reserved memory - default hugepage size 1024M - host model add page size field Signed-off-by: wanyaoqi --- pkg/apis/compute/host.go | 2 + pkg/apis/scheduler/api.go | 5 +- pkg/compute/models/guest_actions.go | 1 + pkg/compute/models/hosts.go | 12 ++- pkg/hostman/guestman/qemu-kvm.go | 2 +- pkg/hostman/hostinfo/hostinfo.go | 86 +++++-------------- pkg/hostman/hostinfo/hostinfohelper.go | 1 + pkg/hostman/options/options.go | 1 + pkg/scheduler/algorithm/predicates/error.go | 1 + .../predicates/guest/migrate_predicate.go | 9 +- 10 files changed, 49 insertions(+), 71 deletions(-) diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index d22519bbc6..e1f218424c 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -366,6 +366,8 @@ type HostSizeAttributes struct { MemReserved string `json:"mem_reserved"` // 内存超分比 MemCmtbound *float32 `json:"mem_cmtbound"` + // 页大小 + PageSizeKB *int `json:"page_size_kb"` // 存储大小,单位Mb StorageSize *int `json:"storage_size"` diff --git a/pkg/apis/scheduler/api.go b/pkg/apis/scheduler/api.go index fc35475aa8..7f8c8164c1 100644 --- a/pkg/apis/scheduler/api.go +++ b/pkg/apis/scheduler/api.go @@ -86,8 +86,9 @@ type ScheduleInput struct { CpuMode string `json:"cpu_mode"` OsArch string `json:"os_arch"` - SkipKernelCheck *bool `json:"skip_kernel_check"` - TargetHostKernel string `json:"target_host_kernel"` + HostMemPageSizeKB int `json:"host_mem_page_size"` + SkipKernelCheck *bool `json:"skip_kernel_check"` + TargetHostKernel string `json:"target_host_kernel"` // In the migrate and create backup cases // we don't need reallocate network diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index a9776273cd..bd1bfe08e0 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -507,6 +507,7 @@ func (self *SGuest) GetSchedMigrateParams( if host != nil { schedDesc.TargetHostKernel, _ = host.SysInfo.GetString("kernel_version") schedDesc.SkipKernelCheck = &input.SkipKernelCheck + schedDesc.HostMemPageSizeKB = host.PageSizeKB } } schedDesc.ReuseNetwork = true diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 7919cd2da0..d450bc6adf 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -144,6 +144,8 @@ type SHost struct { MemReserved int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"` // 内存超分比 MemCmtbound float32 `nullable:"true" default:"1" list:"domain" update:"domain" create:"domain_optional"` + // 页大小 + PageSizeKB int `nullable:"false" default:"4" list:"domain" update:"domain" create:"domain_optional"` // 存储大小,单位Mb StorageSize int `nullable:"true" list:"domain" update:"domain" create:"domain_optional"` @@ -703,8 +705,12 @@ func (self *SHost) GetMemSize() int { } } +func (self *SHost) IsHugePage() bool { + return self.PageSizeKB > 4 +} + func (self *SHost) GetMemoryOvercommitBound() float32 { - if self.MemCmtbound > 0 { + if self.MemCmtbound > 0 && self.IsHugePage() { return self.MemCmtbound } return options.Options.DefaultMemoryOvercommitBound @@ -3648,6 +3654,10 @@ func (self *SHost) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok return input, errors.Wrap(err, "inputUniquenessCheck") } + if self.IsHugePage() && (input.MemReserved != "" || input.MemCmtbound != nil) { + return input, errors.Errorf("host mem is hugepage, cannot update mem_reserved or mem_cmtbound") + } + input.HostSizeAttributes, err = HostManager.ValidateSizeParams(input.HostSizeAttributes) if err != nil { return input, errors.Wrap(err, "ValidateSizeParams") diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 20920d8399..f768acc2fb 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -2158,7 +2158,7 @@ func (s *SKVMGuestInstance) OnResumeSyncMetadataInfo() { if len(s.VncPassword) > 0 { meta.Set("__vnc_password", jsonutils.NewString(s.VncPassword)) } - if options.HostOptions.HugepagesOption == "native" { + if s.manager.host.IsHugepagesEnabled() { meta.Set("__hugepage", jsonutils.NewString("native")) } // not exactly diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index b5666ffe8c..1b44a56bc8 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -428,22 +428,22 @@ func (h *SHostInfo) prepareEnv() error { case "disable": h.DisableHugepages() case "native": - err := h.EnableNativeHugepages(0) - if err != nil { - return errors.Wrap(err, "EnableNativeHugepages") - } + h.EnableNativeHugepages() hp, err := h.Mem.GetHugepages() if err != nil { return errors.Wrap(err, "Mem.GetHugepages") } - szlist := hp.PageSizes() - if len(szlist) == 0 { - return errors.Error("invalid hugepages total size") + for i := 0; i < len(hp); i++ { + if hp[i].SizeKb == options.HostOptions.HugepageSizeMb*1024 { + nr := hp[i].Total + h.sysinfo.HugepageNr = &nr + h.sysinfo.HugepageSizeKb = hp[i].SizeKb + break + } } - if len(szlist) > 1 { - return errors.Error("cannot support more than 1 type of hugepage size") + if h.sysinfo.HugepageNr == nil || *h.sysinfo.HugepageNr == 0 { + return errors.Errorf("hugepage %d nr 0", options.HostOptions.HugepageSizeMb) } - h.sysinfo.HugepageSizeKb = szlist[0] case "transparent": h.EnableTransparentHugepages() default: @@ -587,7 +587,7 @@ func (h *SHostInfo) GetMemory() int { return int64(nr), nil } */ -func (h *SHostInfo) EnableNativeHugepages(reservedMb int) error { +func (h *SHostInfo) EnableNativeHugepages() { kv := map[string]string{ "/sys/kernel/mm/transparent_hugepage/enabled": "never", "/sys/kernel/mm/transparent_hugepage/defrag": "never", @@ -595,43 +595,6 @@ func (h *SHostInfo) EnableNativeHugepages(reservedMb int) error { for k, v := range kv { sysutils.SetSysConfig(k, v) } - // check reserved memory - hp, err := h.Mem.GetHugepages() - if err != nil { - return err - } - pgList := hp.PageSizes() - if len(pgList) == 0 { - // not initialized yet, manually setup, usage page_size 2MB - } else if len(pgList) == 1 { - // already setup, depends on the PageSize - if pgList[0] == 2048 { - } else { - // readonly, cannot adjust any more - return nil - } - } else { - return errors.Error("cannot support more than 1 type of hugepage sizes") - } - mem := h.GetMemory() - if reservedMb > 0 { - mem -= reservedMb - } else { - mem -= h.getReservedMemMb() - } - desiredSz := 2 // ONLY 2MB Hugepage Supported - desiredNr := mem / desiredSz - if desiredNr*desiredSz < mem { - desiredNr += 1 - } - log.Infof("Hugepage %dGB(%dMB) available Mem %dMB, to reserve %d hugepages with size %d", hp.BytesMb()/1024, hp.BytesMb(), mem, desiredNr, desiredSz) - // not setup hugepage yet, or reserved too many hugepages - err = timeutils2.CommandWithTimeout(1, "sh", "-c", - fmt.Sprintf("echo %d > /sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", desiredNr, desiredSz*1024)).Run() - if err != nil { - return err - } - return nil } func (h *SHostInfo) EnableKsm(sleepSec int) { @@ -1203,6 +1166,10 @@ func (h *SHostInfo) updateHostRecord(hostId string) { // first time create input.MemReserved = fmt.Sprintf("%d", h.getReservedMemMb()) } + if h.IsHugepagesEnabled() { + pageSizeKb := options.HostOptions.HugepageSizeMb * 1024 + input.PageSizeKB = &pageSizeKb + } input.StorageDriver = api.DISK_DRIVER_LINUX input.StorageType = h.sysinfo.StorageType storageSize := storageman.GetManager().GetTotalCapacity() @@ -1300,19 +1267,10 @@ func (h *SHostInfo) onUpdateHostInfoSucc(hostbody jsonutils.JSONObject) { h.initCgroup() memReservedMb, _ := hostbody.Int("mem_reserved") - if options.HostOptions.HugepagesOption == "native" && memReservedMb > int64(h.getReservedMemMb()) { - err := h.EnableNativeHugepages(int(memReservedMb)) - if err != nil { - h.onFail(err) - return - } + if h.IsHugepagesEnabled() && int64(h.getReservedMemMb()) != memReservedMb { + h.updateHostReservedMem(h.getReservedMemMb()) } - reserved := h.getReportedReservedMemMb() - if reserved != int(memReservedMb) { - h.updateHostReservedMem(reserved) - return - } h.getNetworkInfo() } @@ -1353,16 +1311,12 @@ func (h *SHostInfo) getOSReservedMemMb() int { } func (h *SHostInfo) getReservedMemMb() int { - return h.getOSReservedMemMb() + h.getKubeReservedMemMb() -} - -func (h *SHostInfo) getReportedReservedMemMb() int { - if options.HostOptions.HugepagesOption == "native" { - // return total minus mem in huagepage pool + if h.IsHugepagesEnabled() { hp, _ := h.Mem.GetHugepages() return h.GetMemory() - int(hp.BytesMb()) + } else { + return h.getOSReservedMemMb() + h.getKubeReservedMemMb() } - return h.getReservedMemMb() } func (h *SHostInfo) PutHostOnline() error { diff --git a/pkg/hostman/hostinfo/hostinfohelper.go b/pkg/hostman/hostinfo/hostinfohelper.go index cc849303ea..040887d868 100644 --- a/pkg/hostman/hostinfo/hostinfohelper.go +++ b/pkg/hostman/hostinfo/hostinfohelper.go @@ -327,6 +327,7 @@ type SSysInfo struct { HugepagesOption string `json:"hugepages_option"` HugepageSizeKb int `json:"hugepage_size_kb"` + HugepageNr *int `json:"hugepage_nr"` Topology *hostapi.HostTopology `json:"topology"` CPUInfo *hostapi.HostCPUInfo `json:"cpu_info"` diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index 6d2e1cd211..66b8b1002f 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -86,6 +86,7 @@ type SHostOptions struct { BlockIoScheduler string `help:"Block IO scheduler, deadline or cfq" default:"deadline"` EnableKsm bool `help:"Enable Kernel Same Page Merging"` HugepagesOption string `help:"Hugepages option: disable|native|transparent" default:"transparent"` + HugepageSizeMb int `help:"hugepage size mb default 1G" default:"1024"` PrivatePrefixes []string `help:"IPv4 private prefixes"` LocalImagePath []string `help:"Local image storage paths"` diff --git a/pkg/scheduler/algorithm/predicates/error.go b/pkg/scheduler/algorithm/predicates/error.go index 8b132a792a..23abde6e91 100644 --- a/pkg/scheduler/algorithm/predicates/error.go +++ b/pkg/scheduler/algorithm/predicates/error.go @@ -36,6 +36,7 @@ const ( ErrHostIsSpecifiedForMigration = `host_id specified for migration` ErrHostCpuModelIsNotMatchForLiveMigrate = `host cpu mode not match for live migrate` ErrHostCpuMicrocodeNotMatchForLiveMigrate = `host cpu microcode not match for live migrate` + ErrHostMemPageSizeNotMatchForLiveMigrate = `host mem page size not match for live migrate` ErrHostKernelNotMatchForLiveMigrate = `host kernel not match for live migrate` ErrMoreThanOneSizeUnspecificSplit = `more than 1 size unspecific split` ErrNoMoreSpaceForUnspecificSplit = `no more space for an unspecific split` diff --git a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go index 995ae66f7a..ffb03545e9 100644 --- a/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go @@ -51,6 +51,13 @@ func (p *MigratePredicate) Execute(ctx context.Context, u *core.Unit, c core.Can // live migrate check if schedData.LiveMigrate { host := c.Getter().Host() + + // target host mem page size check + if schedData.HostMemPageSizeKB != host.PageSizeKB { + h.Exclude(predicates.ErrHostMemPageSizeNotMatchForLiveMigrate) + return h.GetResult() + } + // target host cpu check if schedData.CpuMode != compute.CPU_MODE_QEMU && (schedData.SkipCpuCheck == nil || *schedData.SkipCpuCheck == false) { if schedData.CpuDesc != host.CpuDesc { @@ -67,7 +74,7 @@ func (p *MigratePredicate) Execute(ctx context.Context, u *core.Unit, c core.Can if schedData.SkipKernelCheck != nil && !*schedData.SkipKernelCheck { kv, _ := host.SysInfo.GetString("kernel_version") if schedData.TargetHostKernel != "" && schedData.TargetHostKernel != kv { - h.Exclude2(predicates.ErrHostCpuMicrocodeNotMatchForLiveMigrate, kv, schedData.TargetHostKernel) + h.Exclude2(predicates.ErrHostKernelNotMatchForLiveMigrate, kv, schedData.TargetHostKernel) return h.GetResult() } }