opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-21 06:09:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15168 from wanyaoqi/fix/hugepages

Browse Source 
feat(region,host): hugepage optimize
This commit is contained in:
Zexi Li
2022-10-13 09:47:18 +08:00
committed by GitHub
parent fadec7480b 3d0c09195e
commit 9952c9ad8d
10 changed files with 49 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/compute/host.go
+2
View File
@@ -366,6 +366,8 @@ type HostSizeAttributes struct {
MemReserved string `json:"mem_reserved"`
// 内存超分比
MemCmtbound *float32 `json:"mem_cmtbound"`
// 页大小
PageSizeKB *int `json:"page_size_kb"`
// 存储大小,单位Mb
StorageSize *int `json:"storage_size"`
pkg/apis/scheduler/api.go
+3 -2
View File
@@ -86,8 +86,9 @@ type ScheduleInput struct {
CpuMode string `json:"cpu_mode"`
OsArch string `json:"os_arch"`
SkipKernelCheck *bool `json:"skip_kernel_check"`
TargetHostKernel string `json:"target_host_kernel"`
HostMemPageSizeKB int `json:"host_mem_page_size"`
SkipKernelCheck *bool `json:"skip_kernel_check"`
TargetHostKernel string `json:"target_host_kernel"`
// In the migrate and create backup cases
// we don't need reallocate network
pkg/compute/models/guest_actions.go
+1
View File
@@ -508,6 +508,7 @@ func (self *SGuest) GetSchedMigrateParams(
if host != nil {
schedDesc.TargetHostKernel, _ = host.SysInfo.GetString("kernel_version")
schedDesc.SkipKernelCheck = &input.SkipKernelCheck
schedDesc.HostMemPageSizeKB = host.PageSizeKB
}
}
schedDesc.ReuseNetwork = true
pkg/compute/models/hosts.go
+11 -1
View File
@@ -144,6 +144,8 @@ type SHost struct {
MemReserved int `nullable:"true" default:"0" list:"domain" update:"domain" create:"domain_optional"`
// 内存超分比
MemCmtbound float32 `nullable:"true" default:"1" list:"domain" update:"domain" create:"domain_optional"`
// 页大小
PageSizeKB int `nullable:"false" default:"4" list:"domain" update:"domain" create:"domain_optional"`
// 存储大小,单位Mb
StorageSize int `nullable:"true" list:"domain" update:"domain" create:"domain_optional"`
@@ -703,8 +705,12 @@ func (self *SHost) GetMemSize() int {
}
}
func (self *SHost) IsHugePage() bool {
return self.PageSizeKB > 4
}
func (self *SHost) GetMemoryOvercommitBound() float32 {
if self.MemCmtbound > 0 {
if self.MemCmtbound > 0 && self.IsHugePage() {
return self.MemCmtbound
}
return options.Options.DefaultMemoryOvercommitBound
@@ -3648,6 +3654,10 @@ func (self *SHost) ValidateUpdateData(ctx context.Context, userCred mcclient.Tok
return input, errors.Wrap(err, "inputUniquenessCheck")
}
if self.IsHugePage() && (input.MemReserved != "" || input.MemCmtbound != nil) {
return input, errors.Errorf("host mem is hugepage, cannot update mem_reserved or mem_cmtbound")
}
input.HostSizeAttributes, err = HostManager.ValidateSizeParams(input.HostSizeAttributes)
if err != nil {
return input, errors.Wrap(err, "ValidateSizeParams")
pkg/hostman/guestman/qemu-kvm.go
+1 -1
View File
@@ -2158,7 +2158,7 @@ func (s *SKVMGuestInstance) OnResumeSyncMetadataInfo() {
if len(s.VncPassword) > 0 {
meta.Set("__vnc_password", jsonutils.NewString(s.VncPassword))
}
if options.HostOptions.HugepagesOption == "native" {
if s.manager.host.IsHugepagesEnabled() {
meta.Set("__hugepage", jsonutils.NewString("native"))
}
// not exactly
pkg/hostman/hostinfo/hostinfo.go
+20 -66
View File
@@ -433,22 +433,22 @@ func (h *SHostInfo) prepareEnv() error {
case "disable":
h.DisableHugepages()
case "native":
err := h.EnableNativeHugepages(0)
if err != nil {
return errors.Wrap(err, "EnableNativeHugepages")
}
h.EnableNativeHugepages()
hp, err := h.Mem.GetHugepages()
if err != nil {
return errors.Wrap(err, "Mem.GetHugepages")
}
szlist := hp.PageSizes()
if len(szlist) == 0 {
return errors.Error("invalid hugepages total size")
for i := 0; i < len(hp); i++ {
if hp[i].SizeKb == options.HostOptions.HugepageSizeMb*1024 {
nr := hp[i].Total
h.sysinfo.HugepageNr = &nr
h.sysinfo.HugepageSizeKb = hp[i].SizeKb
break
}
}
if len(szlist) > 1 {
return errors.Error("cannot support more than 1 type of hugepage size")
if h.sysinfo.HugepageNr == nil || *h.sysinfo.HugepageNr == 0 {
return errors.Errorf("hugepage %d nr 0", options.HostOptions.HugepageSizeMb)
}
h.sysinfo.HugepageSizeKb = szlist[0]
case "transparent":
h.EnableTransparentHugepages()
default:
@@ -600,7 +600,7 @@ func (h *SHostInfo) GetMemoryTotal() int {
return int64(nr), nil
} */
func (h *SHostInfo) EnableNativeHugepages(reservedMb int) error {
func (h *SHostInfo) EnableNativeHugepages() {
kv := map[string]string{
"/sys/kernel/mm/transparent_hugepage/enabled": "never",
"/sys/kernel/mm/transparent_hugepage/defrag": "never",
@@ -608,43 +608,6 @@ func (h *SHostInfo) EnableNativeHugepages(reservedMb int) error {
for k, v := range kv {
sysutils.SetSysConfig(k, v)
}
// check reserved memory
hp, err := h.Mem.GetHugepages()
if err != nil {
return err
}
pgList := hp.PageSizes()
if len(pgList) == 0 {
// not initialized yet, manually setup, usage page_size 2MB
} else if len(pgList) == 1 {
// already setup, depends on the PageSize
if pgList[0] == 2048 {
} else {
// readonly, cannot adjust any more
return nil
}
} else {
return errors.Error("cannot support more than 1 type of hugepage sizes")
}
mem := h.GetMemory()
if reservedMb > 0 {
mem -= reservedMb
} else {
mem -= h.getReservedMemMb()
}
desiredSz := 2 // ONLY 2MB Hugepage Supported
desiredNr := mem / desiredSz
if desiredNr*desiredSz < mem {
desiredNr += 1
}
log.Infof("Hugepage %dGB(%dMB) available Mem %dMB, to reserve %d hugepages with size %d", hp.BytesMb()/1024, hp.BytesMb(), mem, desiredNr, desiredSz)
// not setup hugepage yet, or reserved too many hugepages
err = timeutils2.CommandWithTimeout(1, "sh", "-c",
fmt.Sprintf("echo %d > /sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", desiredNr, desiredSz*1024)).Run()
if err != nil {
return err
}
return nil
}
func (h *SHostInfo) EnableKsm(sleepSec int) {
@@ -1343,6 +1306,10 @@ func (h *SHostInfo) updateHostRecord(hostId string) {
// first time create
input.MemReserved = fmt.Sprintf("%d", h.getReservedMemMb())
}
if h.IsHugepagesEnabled() {
pageSizeKb := options.HostOptions.HugepageSizeMb * 1024
input.PageSizeKB = &pageSizeKb
}
input.StorageDriver = api.DISK_DRIVER_LINUX
input.StorageType = h.sysinfo.StorageType
storageSize := storageman.GetManager().GetTotalCapacity()
@@ -1440,19 +1407,10 @@ func (h *SHostInfo) onUpdateHostInfoSucc(hostbody jsonutils.JSONObject) {
h.initCgroup()
memReservedMb, _ := hostbody.Int("mem_reserved")
if options.HostOptions.HugepagesOption == "native" && memReservedMb > int64(h.getReservedMemMb()) {
err := h.EnableNativeHugepages(int(memReservedMb))
if err != nil {
h.onFail(err)
return
}
if h.IsHugepagesEnabled() && int64(h.getReservedMemMb()) != memReservedMb {
h.updateHostReservedMem(h.getReservedMemMb())
}
reserved := h.getReportedReservedMemMb()
if reserved != int(memReservedMb) {
h.updateHostReservedMem(reserved)
return
}
h.getNetworkInfo()
}
@@ -1493,16 +1451,12 @@ func (h *SHostInfo) getOSReservedMemMb() int {
}
func (h *SHostInfo) getReservedMemMb() int {
return h.getOSReservedMemMb() + h.getKubeReservedMemMb()
}
func (h *SHostInfo) getReportedReservedMemMb() int {
if options.HostOptions.HugepagesOption == "native" {
// return total minus mem in huagepage pool
if h.IsHugepagesEnabled() {
hp, _ := h.Mem.GetHugepages()
return h.GetMemory() - int(hp.BytesMb())
} else {
return h.getOSReservedMemMb() + h.getKubeReservedMemMb()
}
return h.getReservedMemMb()
}
func (h *SHostInfo) PutHostOnline() error {
pkg/hostman/hostinfo/hostinfohelper.go
+1
View File
@@ -321,6 +321,7 @@ type SSysInfo struct {
HugepagesOption string `json:"hugepages_option"`
HugepageSizeKb int `json:"hugepage_size_kb"`
HugepageNr *int `json:"hugepage_nr"`
Topology *hostapi.HostTopology `json:"topology"`
CPUInfo *hostapi.HostCPUInfo `json:"cpu_info"`
pkg/hostman/options/options.go
+1
View File
@@ -86,6 +86,7 @@ type SHostOptions struct {
BlockIoScheduler string `help:"Block IO scheduler, deadline or cfq" default:"deadline"`
EnableKsm bool `help:"Enable Kernel Same Page Merging"`
HugepagesOption string `help:"Hugepages option: disable|native|transparent" default:"transparent"`
HugepageSizeMb int `help:"hugepage size mb default 1G" default:"1024"`
PrivatePrefixes []string `help:"IPv4 private prefixes"`
LocalImagePath []string `help:"Local image storage paths"`
pkg/scheduler/algorithm/predicates/error.go
+1
View File
@@ -36,6 +36,7 @@ const (
ErrHostIsSpecifiedForMigration = `host_id specified for migration`
ErrHostCpuModelIsNotMatchForLiveMigrate = `host cpu mode not match for live migrate`
ErrHostCpuMicrocodeNotMatchForLiveMigrate = `host cpu microcode not match for live migrate`
ErrHostMemPageSizeNotMatchForLiveMigrate = `host mem page size not match for live migrate`
ErrHostKernelNotMatchForLiveMigrate = `host kernel not match for live migrate`
ErrMoreThanOneSizeUnspecificSplit = `more than 1 size unspecific split`
ErrNoMoreSpaceForUnspecificSplit = `no more space for an unspecific split`
pkg/scheduler/algorithm/predicates/guest/migrate_predicate.go
+8 -1
View File
@@ -51,6 +51,13 @@ func (p *MigratePredicate) Execute(ctx context.Context, u *core.Unit, c core.Can
// live migrate check
if schedData.LiveMigrate {
host := c.Getter().Host()
// target host mem page size check
if schedData.HostMemPageSizeKB != host.PageSizeKB {
h.Exclude(predicates.ErrHostMemPageSizeNotMatchForLiveMigrate)
return h.GetResult()
}
// target host cpu check
if schedData.CpuMode != compute.CPU_MODE_QEMU && (schedData.SkipCpuCheck == nil || *schedData.SkipCpuCheck == false) {
if schedData.CpuDesc != host.CpuDesc {
@@ -67,7 +74,7 @@ func (p *MigratePredicate) Execute(ctx context.Context, u *core.Unit, c core.Can
if schedData.SkipKernelCheck != nil && !*schedData.SkipKernelCheck {
kv, _ := host.SysInfo.GetString("kernel_version")
if schedData.TargetHostKernel != "" && schedData.TargetHostKernel != kv {
h.Exclude2(predicates.ErrHostCpuMicrocodeNotMatchForLiveMigrate, kv, schedData.TargetHostKernel)
h.Exclude2(predicates.ErrHostKernelNotMatchForLiveMigrate, kv, schedData.TargetHostKernel)
return h.GetResult()
}
}