From 130c82dfd3352e9911393337688610fabe698738 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Mon, 18 Nov 2019 11:53:41 +0800 Subject: [PATCH] hugepage fix --- pkg/compute/guestdrivers/azure.go | 2 +- pkg/compute/guestdrivers/base.go | 2 +- pkg/compute/guestdrivers/kvm.go | 5 ++-- pkg/compute/models/guest_actions.go | 2 +- pkg/compute/models/guestdrivers.go | 2 +- pkg/hostman/guestman/qemu-kvm.go | 3 +++ pkg/hostman/guestman/qemu-kvmhelper.go | 2 +- pkg/hostman/hostinfo/hostinfo.go | 34 +++++++++++++++++------ pkg/hostman/hostinfo/hostinfohelper.go | 37 ++++++++++++++++++++++++++ 9 files changed, 74 insertions(+), 15 deletions(-) diff --git a/pkg/compute/guestdrivers/azure.go b/pkg/compute/guestdrivers/azure.go index f3e8981cc0..ad6e776841 100644 --- a/pkg/compute/guestdrivers/azure.go +++ b/pkg/compute/guestdrivers/azure.go @@ -144,6 +144,6 @@ func (self *SAzureGuestDriver) IsSupportedBillingCycle(bc billing.SBillingCycle) return false } -func (self *SAzureGuestDriver) NeedStopForChangeSpec(guest *models.SGuest) bool { +func (self *SAzureGuestDriver) NeedStopForChangeSpec(guest *models.SGuest, cpuChanged, memChanged bool) bool { return false } diff --git a/pkg/compute/guestdrivers/base.go b/pkg/compute/guestdrivers/base.go index 48b49f4a4e..1171b06afb 100644 --- a/pkg/compute/guestdrivers/base.go +++ b/pkg/compute/guestdrivers/base.go @@ -259,7 +259,7 @@ func (self *SBaseGuestDriver) RequestAssociateEip(ctx context.Context, userCred return fmt.Errorf("SBaseGuestDriver: Not Implement RequestAssociateEip") } -func (self *SBaseGuestDriver) NeedStopForChangeSpec(guest *models.SGuest) bool { +func (self *SBaseGuestDriver) NeedStopForChangeSpec(guest *models.SGuest, cpuChanged, memChanged bool) bool { return true } diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index f74de59fac..183e05bcb6 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -298,8 +298,9 @@ func (self *SKVMGuestDriver) OnDeleteGuestFinalCleanup(ctx context.Context, gues return nil } -func (self *SKVMGuestDriver) NeedStopForChangeSpec(guest *models.SGuest) bool { - return guest.GetMetadata("hotplug_cpu_mem", nil) != "enable" +func (self *SKVMGuestDriver) NeedStopForChangeSpec(guest *models.SGuest, cpuChanged, memChanged bool) bool { + return guest.GetMetadata("hotplug_cpu_mem", nil) != "enable" || + (memChanged && guest.GetMetadata("__hugepage", nil) == "native") } func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, instanceType string, vcpuCount, vmemSize int64) error { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index 3d879cfa9b..32acf84f8e 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -2230,7 +2230,7 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T } } - if self.Status == api.VM_RUNNING && (cpuChanged || memChanged) && self.GetDriver().NeedStopForChangeSpec(self) { + if self.Status == api.VM_RUNNING && (cpuChanged || memChanged) && self.GetDriver().NeedStopForChangeSpec(self, cpuChanged, memChanged) { return nil, httperrors.NewInvalidStatusError("cannot change CPU/Memory spec in status %s", self.Status) } diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 4654a40322..c0d99f9574 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -165,7 +165,7 @@ type IGuestDriver interface { ValidateCreateEip(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) error RequestAssociateEip(ctx context.Context, userCred mcclient.TokenCredential, guest *SGuest, eip *SElasticip, task taskman.ITask) error - NeedStopForChangeSpec(guest *SGuest) bool + NeedStopForChangeSpec(guest *SGuest, cpuChanged, memChanged bool) bool OnGuestChangeCpuMemFailed(ctx context.Context, guest *SGuest, data *jsonutils.JSONDict, task taskman.ITask) error IsSupportGuestClone() bool diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index a7f5371e7b..54673f9bf4 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -1254,6 +1254,9 @@ func (s *SKVMGuestInstance) OnResumeSyncMetadataInfo() { if len(s.VncPassword) > 0 { meta.Set("__vnc_password", jsonutils.NewString(s.VncPassword)) } + if options.HostOptions.HugepagesOption == "native" { + meta.Set("__hugepage", jsonutils.NewString("native")) + } if s.syncMeta != nil { meta.Update(s.syncMeta) } diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 39a910ce31..eb9dca1108 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -662,7 +662,7 @@ func (s *SKVMGuestInstance) generateStopScript(data *jsonutils.JSONDict) string cmd += "fi\n" if options.HostOptions.HugepagesOption == "native" { - cmd += fmt.Sprintf("if [ -f /dev/hugepages/%s ]; then\n", uuid) + cmd += fmt.Sprintf("if [ -d /dev/hugepages/%s ]; then\n", uuid) cmd += fmt.Sprintf(" umount /dev/hugepages/%s\n", uuid) cmd += fmt.Sprintf(" rm -rf /dev/hugepages/%s\n", uuid) cmd += "fi\n" diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 39402bad3f..66a178a51b 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -16,6 +16,7 @@ package hostinfo import ( "context" + "errors" "fmt" "io/ioutil" "os" @@ -136,7 +137,9 @@ func (h *SHostInfo) Init() error { } func (h *SHostInfo) parseConfig() error { - if h.GetMemory() < 64 { // MB + if mem, err := h.GetMemory(); err != nil { + return err + } else if mem < 64 { // MB return fmt.Errorf("Not enough memory!") } for _, n := range options.HostOptions.Networks { @@ -234,9 +237,13 @@ func (h *SHostInfo) prepareEnv() error { case "disable": h.DisableHugepages() case "native": - if err := h.EnableNativeHugepages(); err != nil { + size, err := h.Mem.GetHugepageTotal() + if err != nil { return err } + if size <= 0 { + return errors.New("invalid hugepages total size") + } case "transparent": h.EnableTransparentHugepages() default: @@ -342,8 +349,11 @@ func (h *SHostInfo) EnableTransparentHugepages() { } } -func (h *SHostInfo) GetMemory() int { - return h.Mem.Total // - options.reserved_memory +func (h *SHostInfo) GetMemory() (int, error) { + if options.HostOptions.HugepagesOption == "native" { + return h.Mem.GetHugepageTotal() + } + return h.Mem.Total, nil // - options.reserved_memory } func (h *SHostInfo) EnableNativeHugepages() error { @@ -359,8 +369,12 @@ func (h *SHostInfo) EnableNativeHugepages() error { for k, v := range kv { sysutils.SetSysConfig(k, v) } - preAllocPagesNum := h.GetMemory()/h.Mem.GetHugepagesizeMb() + 1 - err := timeutils2.CommandWithTimeout(1, "sh", "-c", fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", preAllocPagesNum)).Run() + mem, err := h.GetMemory() + if err != nil { + return err + } + preAllocPagesNum := mem/h.Mem.GetHugepagesizeMb() + 1 + err = timeutils2.CommandWithTimeout(1, "sh", "-c", fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", preAllocPagesNum)).Run() if err != nil { log.Errorln(err) _, err = procutils.NewCommand("sh", "-c", "echo 0 > /proc/sys/vm/nr_hugepages").Run() @@ -767,7 +781,12 @@ func (h *SHostInfo) updateHostRecord(hostId string) { content.Set("cpu_microcode", jsonutils.NewString(h.Cpu.cpuInfoProc.Microcode)) content.Set("cpu_mhz", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Freq))) content.Set("cpu_cache", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Cache))) - content.Set("mem_size", jsonutils.NewInt(int64(h.Mem.MemInfo.Total))) + memTotal, err := h.GetMemory() + if err != nil { + log.Errorln(err) + h.onFail() + } + content.Set("mem_size", jsonutils.NewInt(int64(memTotal))) content.Set("storage_driver", jsonutils.NewString(api.DISK_DRIVER_LINUX)) content.Set("storage_type", jsonutils.NewString(h.sysinfo.StorageType)) content.Set("storage_size", jsonutils.NewInt(int64(storageman.GetManager().GetTotalCapacity()))) @@ -787,7 +806,6 @@ func (h *SHostInfo) updateHostRecord(hostId string) { var ( res jsonutils.JSONObject - err error ) if !isInit { res, err = modules.Hosts.Update(h.GetSession(), hostId, content) diff --git a/pkg/hostman/hostinfo/hostinfohelper.go b/pkg/hostman/hostinfo/hostinfohelper.go index 8bcd95e69e..442200b443 100644 --- a/pkg/hostman/hostinfo/hostinfohelper.go +++ b/pkg/hostman/hostinfo/hostinfohelper.go @@ -24,6 +24,7 @@ import ( "strings" "time" + "github.com/pkg/errors" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/mem" @@ -159,6 +160,42 @@ func DetectMemoryInfo() (*SMemory, error) { return smem, nil } +func (m *SMemory) GetHugepageTotal() (int, error) { + file, err := os.Open("/proc/meminfo") + if err != nil { + return 0, errors.Wrap(err, "open meminfo") + } + defer file.Close() + var ( + nrHugePage int + sizeHugePage int + ) + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "Hugepagesize:") { + re := regexp.MustCompile(`\s+`) + segs := re.Split(line, -1) + v, err := strconv.Atoi(segs[1]) + if err != nil { + return 0, errors.Wrap(err, "get hugepage size") + } + sizeHugePage = v / 1024 // MB + log.Debugf("Huge page size %v", sizeHugePage) + } else if strings.HasPrefix(line, "HugePages_Total:") { + re := regexp.MustCompile(`\s+`) + segs := re.Split(line, -1) + v, err := strconv.Atoi(segs[1]) + if err != nil { + return 0, errors.Wrap(err, "get hugepages total") + } + nrHugePage = v + log.Debugf("Huge page number %v", nrHugePage) + } + } + return nrHugePage * sizeHugePage, nil +} + func (m *SMemory) GetHugepagesizeMb() int { file, err := os.Open("/proc/meminfo") if err != nil {