diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index b3da7f2f1c..1a0c48fc20 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -391,9 +391,9 @@ func (self *SKVMGuestDriver) RequestAssociateEip(ctx context.Context, userCred m } func (self *SKVMGuestDriver) NeedStopForChangeSpec(ctx context.Context, guest *models.SGuest, cpuChanged, memChanged bool) bool { - return guest.GetMetadata(ctx, "hotplug_cpu_mem", nil) != "enable" || - (memChanged && guest.GetMetadata(ctx, "__hugepage", nil) == "native") || - apis.IsARM(guest.OsArch) + return guest.GetMetadata(ctx, "hotplug_cpu_mem", nil) != "enable" || apis.IsARM(guest.OsArch) + // (memChanged && guest.GetMetadata(ctx, "__hugepage", nil) == "native") || + // apis.IsARM(guest.OsArch) } func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, instanceType string, vcpuCount, vmemSize int64) error { diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 9d462c1c48..12c35e373d 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -1529,11 +1529,43 @@ func (task *SGuestHotplugCpuMemTask) startAddMem() { func (task *SGuestHotplugCpuMemTask) onGetSlotIndex(index int) { var newIndex = index task.memSlotNewIndex = &newIndex - params := map[string]string{ - "id": fmt.Sprintf("mem%d", *task.memSlotNewIndex), - "size": fmt.Sprintf("%dM", task.addMemSize), + if task.manager.host.IsHugepagesEnabled() { + memPath := fmt.Sprintf("/dev/hugepages/%s-%d", task.GetId(), index) + + err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", memPath).Run() + if err != nil { + reason := fmt.Sprintf("mkdir %s fail: %s", memPath, err) + log.Errorf("%s", reason) + task.onFail(reason) + return + } + err = procutils.NewRemoteCommandAsFarAsPossible("mount", "-t", "hugetlbfs", "-o", + fmt.Sprintf("pagesize=%dK,size=%dM", task.manager.host.HugepageSizeKb(), task.addMemSize), + fmt.Sprintf("hugetlbfs-%s-%d", task.GetId(), index), + memPath, + ).Run() + if err != nil { + reason := fmt.Sprintf("mount %s fail: %s", memPath, err) + log.Errorf("%s", reason) + task.onFail(reason) + return + } + + params := map[string]string{ + "id": fmt.Sprintf("mem%d", *task.memSlotNewIndex), + "size": fmt.Sprintf("%dM", task.addMemSize), + "mem-path": memPath, + "share": "on", + "prealloc": "on", + } + task.Monitor.ObjectAdd("memory-backend-file", params, task.onAddMemObject) + } else { + params := map[string]string{ + "id": fmt.Sprintf("mem%d", *task.memSlotNewIndex), + "size": fmt.Sprintf("%dM", task.addMemSize), + } + task.Monitor.ObjectAdd("memory-backend-ram", params, task.onAddMemObject) } - task.Monitor.ObjectAdd("memory-backend-ram", params, task.onAddMemObject) } func (task *SGuestHotplugCpuMemTask) onAddMemFailed(reason string) { diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index 3c36cc1a51..f29c81fb68 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -292,8 +292,8 @@ func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (strin if input.HugepagesEnabled { cmd += fmt.Sprintf("mkdir -p /dev/hugepages/%s\n", input.UUID) - cmd += fmt.Sprintf("mount -t hugetlbfs -o size=%dM hugetlbfs-%s /dev/hugepages/%s\n", - input.Mem, input.UUID, input.UUID) + cmd += fmt.Sprintf("mount -t hugetlbfs -o pagesize=%dK,size=%dM hugetlbfs-%s /dev/hugepages/%s\n", + s.manager.host.HugepageSizeKb(), input.Mem, input.UUID, input.UUID) } cmd += "sleep 1\n" @@ -627,12 +627,14 @@ func (s *SKVMGuestInstance) generateStopScript(data *jsonutils.JSONDict) string cmd += " rm -f $PID_FILE\n" cmd += "fi\n" - if s.manager.host.IsHugepagesEnabled() { - cmd += fmt.Sprintf("if [ -d /dev/hugepages/%s ]; then\n", uuid) - cmd += fmt.Sprintf(" umount /dev/hugepages/%s\n", uuid) - cmd += fmt.Sprintf(" rm -rf /dev/hugepages/%s\n", uuid) - cmd += "fi\n" - } + cmd += fmt.Sprintf("for d in $(ls -d /dev/hugepages/%s*)\n", uuid) + cmd += fmt.Sprintf("do\n") + cmd += fmt.Sprintf(" if [ -d $d ]; then\n") + cmd += fmt.Sprintf(" umount $d\n") + cmd += fmt.Sprintf(" rm -rf $d\n") + cmd += fmt.Sprintf(" fi\n") + cmd += fmt.Sprintf("done\n") + for _, nic := range nics { ifname, _ := nic.GetString("ifname") downscript := s.getNicDownScriptPath(nic) diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index 6b1d4525cc..decd0b8138 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -127,12 +127,13 @@ func GenerateStartOptions( drvOpt.Memory(input.Mem), ) + var memDev string if input.HugepagesEnabled { - opts = append(opts, - drvOpt.MemPrealloc(), - drvOpt.MemPath(fmt.Sprintf("/dev/hugepages/%s", input.UUID)), - ) + memDev = drvOpt.MemPath(input.Mem, fmt.Sprintf("/dev/hugepages/%s", input.UUID)) + } else { + memDev = drvOpt.MemDev(input.Mem) } + opts = append(opts, memDev) // bootOrder enableMenu := false diff --git a/pkg/hostman/guestman/qemu/qemu.go b/pkg/hostman/guestman/qemu/qemu.go index f7238923cc..e34215cf5e 100644 --- a/pkg/hostman/guestman/qemu/qemu.go +++ b/pkg/hostman/guestman/qemu/qemu.go @@ -93,8 +93,8 @@ type QemuOptions interface { Name(name string) string UUID(enable bool, uuid string) string Memory(sizeMB uint64) string - MemPrealloc() string - MemPath(p string) string + MemPath(sizeMB uint64, p string) string + MemDev(sizeMB uint64) string Boot(order string, enableMenu bool) string BIOS(file string) string Device(devStr string) string @@ -240,8 +240,12 @@ func (o baseOptions) MemPrealloc() string { return "-mem-prealloc" } -func (o baseOptions) MemPath(p string) string { - return "-mem-path " + p +func (o baseOptions) MemPath(sizeMB uint64, p string) string { + return fmt.Sprintf("-object memory-backend-file,id=mem,size=%dM,mem-path=%s,share=on,prealloc=on -numa node,memdev=mem", sizeMB, p) +} + +func (o baseOptions) MemDev(sizeMB uint64) string { + return fmt.Sprintf("-object memory-backend-ram,id=mem,size=%dM -numa node,memdev=mem", sizeMB) } func (o baseOptions) Boot(order string, enableMenu bool) string { diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 61152c8726..e389e0aa2b 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -17,6 +17,7 @@ package hostinfo import ( "context" "fmt" + "math" "net" "os" "path" @@ -75,9 +76,8 @@ type SHostInfo struct { kubeletConfig kubelet.KubeletConfig - isInit bool - enableHugePages bool - onHostDown string + isInit bool + onHostDown string IsolatedDeviceMan isolated_device.IsolatedDeviceManager @@ -151,7 +151,11 @@ func (h *SHostInfo) IsNestedVirtualization() bool { } func (h *SHostInfo) IsHugepagesEnabled() bool { - return h.enableHugePages || options.HostOptions.HugepagesOption == "native" + return options.HostOptions.HugepagesOption == "native" +} + +func (h *SHostInfo) HugepageSizeKb() int { + return h.sysinfo.HugepageSizeKb } /* In this order init host service: @@ -269,9 +273,8 @@ func (h *SHostInfo) generateLocalNetworkConfig() (string, error) { } func (h *SHostInfo) parseConfig() error { - if mem, err := h.GetMemory(); err != nil { - return err - } else if mem < 64 { // MB + mem := h.GetMemory() + if mem < 64 { // MB return fmt.Errorf("Not enough memory!") } if len(options.HostOptions.Networks) == 0 { @@ -393,7 +396,7 @@ func (h *SHostInfo) prepareEnv() error { // return err // } - if options.HostOptions.EnableKsm { + if options.HostOptions.EnableKsm && options.HostOptions.HugepagesOption == "disable" { h.EnableKsm(900) } else { h.DisableKsm() @@ -403,19 +406,30 @@ func (h *SHostInfo) prepareEnv() error { case "disable": h.DisableHugepages() case "native": - size, err := h.Mem.GetHugepageTotal() + err := h.EnableNativeHugepages(0) if err != nil { - return err + return errors.Wrap(err, "EnableNativeHugepages") } - if size <= 0 { + hp, err := h.Mem.GetHugepages() + if err != nil { + return errors.Wrap(err, "Mem.GetHugepages") + } + szlist := hp.PageSizes() + if len(szlist) == 0 { return errors.New("invalid hugepages total size") } + if len(szlist) > 1 { + return errors.New("cannot support more than 1 type of hugepage size") + } + h.sysinfo.HugepageSizeKb = szlist[0] case "transparent": h.EnableTransparentHugepages() default: return fmt.Errorf("Invalid hugepages option") } + h.sysinfo.HugepagesOption = options.HostOptions.HugepagesOption + h.PreventArpFlux() h.tuneSystem() return nil @@ -521,23 +535,11 @@ func (h *SHostInfo) EnableTransparentHugepages() { } } -func (h *SHostInfo) GetMemory() (int, error) { - if options.HostOptions.HugepagesOption == "native" { - return h.Mem.GetHugepageTotal() - } - total := h.Mem.Total - if h.kubeletConfig != nil { - memThreshold := h.kubeletConfig.GetEvictionConfig().GetHard().GetMemoryAvailable() - memBytes, _ := memThreshold.Value.Quantity.AsInt64() - memMb := int(memBytes / 1024 / 1024) - subMem := total - memMb - log.Infof("Get total memory %d, kubelet memory threshold subtracted: (%d - %d)", subMem, total, memMb) - total = subMem - } - return total, nil // - options.reserved_memory +func (h *SHostInfo) GetMemory() int { + return h.Mem.Total } -func (h *SHostInfo) getCurrentHugepageNr() (int64, error) { +/* func (h *SHostInfo) getCurrentHugepageNr() (int64, error) { nrStr, err := fileutils2.FileGetContents("/proc/sys/vm/nr_hugepages") if err != nil { return 0, errors.Wrap(err, "file get content nr hugepages") @@ -547,9 +549,9 @@ func (h *SHostInfo) getCurrentHugepageNr() (int64, error) { return 0, errors.Wrap(err, "nr str atoi") } return int64(nr), nil -} +} */ -func (h *SHostInfo) EnableNativeHugepages() error { +func (h *SHostInfo) EnableNativeHugepages(reservedMb int) error { kv := map[string]string{ "/sys/kernel/mm/transparent_hugepage/enabled": "never", "/sys/kernel/mm/transparent_hugepage/defrag": "never", @@ -557,32 +559,42 @@ func (h *SHostInfo) EnableNativeHugepages() error { for k, v := range kv { sysutils.SetSysConfig(k, v) } - nr, err := h.getCurrentHugepageNr() + // check reserved memory + hp, err := h.Mem.GetHugepages() if err != nil { return err } - mem, err := h.GetMemory() - if err != nil { - return err - } - mem -= h.getReservedMem() - desiredNr := int64(mem/h.Mem.GetHugepagesizeMb() + 1) - if nr < desiredNr { - err = timeutils2.CommandWithTimeout(1, "sh", "-c", - fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", desiredNr)).Run() - if err != nil { - return err + pgList := hp.PageSizes() + if len(pgList) == 0 { + // not initialized yet, manually setup, usage page_size 2MB + } else if len(pgList) == 1 { + // already setup, depends on the PageSize + if pgList[0] == 2048 { + } else { + // readonly, cannot adjust any more + return nil } + } else { + return errors.New("cannot support more than 1 type of hugepage sizes") } - currentNr, err := h.getCurrentHugepageNr() + mem := h.GetMemory() + if reservedMb > 0 { + mem -= reservedMb + } else { + mem -= h.getReservedMemMb() + } + desiredSz := 2 // ONLY 2MB Hugepage Supported + desiredNr := mem / desiredSz + if desiredNr*desiredSz < mem { + desiredNr += 1 + } + log.Infof("Hugepage %dGB(%dMB) available Mem %dMB, to reserve %d hugepages with size %d", hp.BytesMb()/1024, hp.BytesMb(), mem, desiredNr, desiredSz) + // not setup hugepage yet, or reserved too many hugepages + err = timeutils2.CommandWithTimeout(1, "sh", "-c", + fmt.Sprintf("echo %d > /sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", desiredNr, desiredSz*1024)).Run() if err != nil { return err } - if currentNr < desiredNr { - err = timeutils2.CommandWithTimeout(1, "sh", "-c", - fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", nr)).Run() - return fmt.Errorf("no enough memory to resize hugepage, current nr %d, desired nr %d", currentNr, desiredNr) - } return nil } @@ -1083,14 +1095,11 @@ func (h *SHostInfo) updateHostRecord(hostId string) { content.Set("cpu_mhz", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Freq))) } content.Set("cpu_cache", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Cache))) - memTotal, err := h.GetMemory() - if err != nil { - h.onFail(err) - return - } + memTotal := h.GetMemory() content.Set("mem_size", jsonutils.NewInt(int64(memTotal))) if len(hostId) == 0 { - content.Set("mem_reserved", jsonutils.NewInt(int64(h.getReservedMem()))) + // first time create + content.Set("mem_reserved", jsonutils.NewInt(int64(h.getReservedMemMb()))) } content.Set("storage_driver", jsonutils.NewString(api.DISK_DRIVER_LINUX)) content.Set("storage_type", jsonutils.NewString(h.sysinfo.StorageType)) @@ -1112,6 +1121,7 @@ func (h *SHostInfo) updateHostRecord(hostId string) { var ( res jsonutils.JSONObject + err error ) if !h.isInit { res, err = modules.Hosts.Update(h.GetSession(), hostId, content) @@ -1159,42 +1169,29 @@ func (h *SHostInfo) onUpdateHostInfoSucc(hostbody jsonutils.JSONObject) { h.onFail(err) return } - - if options.HostOptions.HugepagesOption == "native" { - if h.isInit && len(h.IsolatedDeviceMan.GetDevices()) > 0 { - meta := jsonutils.NewDict() - meta.Set("__enable_hugepages", jsonutils.NewString("true")) - _, err := modules.Hosts.SetMetadata(h.GetSession(), h.HostId, meta) - if err != nil { - h.onFail(fmt.Sprintf("failed ")) - return - } - h.enableHugePages = true - } else if hugepage, _ := hostbody.GetString("metadata", "__enable_hugepages"); hugepage == "true" { - h.enableHugePages = true - } - if h.enableHugePages { - err := h.EnableNativeHugepages() - if err != nil { - h.onFail(err) - return - } - } - } h.onHostDown, _ = hostbody.GetString("metadata", "__on_host_down") - if memReserved, _ := hostbody.Int("mem_reserved"); memReserved == 0 { - h.updateHostReservedMem() + memReservedMb, _ := hostbody.Int("mem_reserved") + if options.HostOptions.HugepagesOption == "native" && memReservedMb > int64(h.getReservedMemMb()) { + err := h.EnableNativeHugepages(int(memReservedMb)) + if err != nil { + h.onFail(err) + return + } + } + + reserved := h.getReportedReservedMemMb() + if reserved != int(memReservedMb) { + h.updateHostReservedMem(reserved) } else { h.PutHostOffline("") } } -func (h *SHostInfo) updateHostReservedMem() { +func (h *SHostInfo) updateHostReservedMem(reserved int) { content := jsonutils.NewDict() - content.Set("mem_reserved", jsonutils.NewInt(int64(h.getReservedMem()))) - res, err := modules.Hosts.Update(h.GetSession(), - h.HostId, content) + content.Set("mem_reserved", jsonutils.NewInt(int64(reserved))) + res, err := modules.Hosts.Update(h.GetSession(), h.HostId, content) if err != nil { h.onFail(err) return @@ -1203,7 +1200,20 @@ func (h *SHostInfo) updateHostReservedMem() { } } -func (h *SHostInfo) getReservedMem() int { +func (h *SHostInfo) getKubeReservedMemMb() int { + // reserved for Kubelet + if h.kubeletConfig != nil { + memThreshold := h.kubeletConfig.GetEvictionConfig().GetHard().GetMemoryAvailable() + memBytes, _ := memThreshold.Value.Quantity.AsInt64() + memMb := int(math.Ceil(float64(memBytes) / 1024 / 1024)) + log.Infof("Kubelet memory threshold subtracted: %dMB", memMb) + return memMb + } + return 0 +} + +func (h *SHostInfo) getOSReservedMemMb() int { + // reserved memory for OS reserved := h.Mem.MemInfo.Total / 10 if reserved > options.HostOptions.MaxReservedMemory { return options.HostOptions.MaxReservedMemory @@ -1214,6 +1224,19 @@ func (h *SHostInfo) getReservedMem() int { return reserved } +func (h *SHostInfo) getReservedMemMb() int { + return h.getOSReservedMemMb() + h.getKubeReservedMemMb() +} + +func (h *SHostInfo) getReportedReservedMemMb() int { + if options.HostOptions.HugepagesOption == "native" { + // return total minus mem in huagepage pool + hp, _ := h.Mem.GetHugepages() + return h.GetMemory() - int(hp.BytesMb()) + } + return h.getReservedMemMb() +} + func (h *SHostInfo) PutHostOffline(reason string) { data := jsonutils.NewDict() if options.HostOptions.EnableHealthChecker { diff --git a/pkg/hostman/hostinfo/hostinfohelper.go b/pkg/hostman/hostinfo/hostinfohelper.go index e060420b7f..77072c10c6 100644 --- a/pkg/hostman/hostinfo/hostinfohelper.go +++ b/pkg/hostman/hostinfo/hostinfohelper.go @@ -19,7 +19,6 @@ import ( "context" "fmt" "os" - "regexp" "strconv" "strings" "time" @@ -166,67 +165,8 @@ func DetectMemoryInfo() (*SMemory, error) { return smem, nil } -func (m *SMemory) GetHugepageTotal() (int, error) { - file, err := os.Open("/proc/meminfo") - if err != nil { - return 0, errors.Wrap(err, "open meminfo") - } - defer file.Close() - var ( - nrHugePage int - sizeHugePage int - ) - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "Hugepagesize:") { - re := regexp.MustCompile(`\s+`) - segs := re.Split(line, -1) - v, err := strconv.Atoi(segs[1]) - if err != nil { - return 0, errors.Wrap(err, "get hugepage size") - } - sizeHugePage = v / 1024 // MB - log.Debugf("Huge page size %v", sizeHugePage) - } else if strings.HasPrefix(line, "HugePages_Total:") { - re := regexp.MustCompile(`\s+`) - segs := re.Split(line, -1) - v, err := strconv.Atoi(segs[1]) - if err != nil { - return 0, errors.Wrap(err, "get hugepages total") - } - nrHugePage = v - log.Debugf("Huge page number %v", nrHugePage) - } - } - return nrHugePage * sizeHugePage, nil -} - -func (m *SMemory) GetHugepagesizeMb() int { - file, err := os.Open("/proc/meminfo") - if err != nil { - log.Errorln(err) - return 0 - } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "Hugepagesize:") { - re := regexp.MustCompile(`\s+`) - segs := re.Split(line, -1) - v, err := strconv.Atoi(segs[1]) - if err != nil { - log.Errorln(err) - return 0 - } - return int(v) / 1024 - } - } - if err := scanner.Err(); err != nil { - log.Errorln(err) - } - return 0 +func (m *SMemory) GetHugepages() (sysutils.THugepages, error) { + return sysutils.GetHugepages() } type SNIC struct { @@ -381,6 +321,9 @@ type SSysInfo struct { CpuMicrocode string `json:"cpu_microcode"` StorageType string `json:"storage_type"` + + HugepagesOption string `json:"hugepages_option"` + HugepageSizeKb int `json:"hugepage_size_kb"` } func StartDetachStorages(hs []jsonutils.JSONObject) { diff --git a/pkg/hostman/hostutils/hostutils.go b/pkg/hostman/hostutils/hostutils.go index 42634af2a8..311be7d703 100644 --- a/pkg/hostman/hostutils/hostutils.go +++ b/pkg/hostman/hostutils/hostutils.go @@ -43,7 +43,9 @@ type IHost interface { GetMasterIp() string GetCpuArchitecture() string IsAarch64() bool + IsHugepagesEnabled() bool + HugepageSizeKb() int IsKvmSupport() bool IsNestedVirtualization() bool diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index ff22fb4d1d..75d1ffffcc 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -42,6 +42,8 @@ type SHostOptions struct { RecycleDiskfile bool `help:"Recycle instead of remove deleted disk file" default:"true"` RecycleDiskfileKeepDays int `help:"How long recycled files kept, default 28 days" default:"28"` + ZeroCleanDiskData bool `help:"Clean disk data by writing zeros" default:"false"` + EnableTemplateBacking bool `help:"Use template as backing file"` AutoMergeBackingTemplate bool `help:"Automatically stream merging backing file"` AutoMergeDelaySeconds int `help:"Seconds to delay mergeing backing file after VM start, default 15 minutes" default:"900"` diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index b4a36554a9..1b405c12ef 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -37,6 +37,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" + "yunion.io/x/onecloud/pkg/util/zeroclean" ) const MINIMAL_FREE_SPACE = 128 @@ -354,6 +355,10 @@ func cleanDailyFiles(storagePath, subDir string, keepDay int) { if date.Before(markTime) { log.Infof("Cron Job Clean Recycle Bin: start delete %s", file.Name()) subDirPath := path.Join(recycleDir, file.Name()) + if options.HostOptions.ZeroCleanDiskData { + // try to zero clean files in subdir + zeroclean.ZeroDir(subDirPath) + } if output, err := procutils.NewCommand("rm", "-rf", subDirPath).Output(); err != nil { log.Errorf("clean recycle dir %s error: %s, %s", subDirPath, err, output) } diff --git a/pkg/util/sysutils/hugepages.go b/pkg/util/sysutils/hugepages.go new file mode 100644 index 0000000000..a663aabea3 --- /dev/null +++ b/pkg/util/sysutils/hugepages.go @@ -0,0 +1,109 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysutils + +import ( + "io/ioutil" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +type SHugepageInfo struct { + SizeKb int + Total int + Free int +} + +func (h SHugepageInfo) BytesMb() int64 { + return int64(h.Total) * int64(h.SizeKb) / 1024 +} + +type THugepages []SHugepageInfo + +func (a THugepages) Len() int { return len(a) } +func (a THugepages) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a THugepages) Less(i, j int) bool { return a[i].SizeKb < a[j].SizeKb } + +func (a THugepages) BytesMb() int64 { + ret := int64(0) + for _, h := range a { + ret += h.BytesMb() + } + return ret +} + +func (a THugepages) PageSizes() []int { + ret := make([]int, 0) + for _, h := range a { + if h.Total > 0 { + ret = append(ret, h.SizeKb) + } + } + return ret +} + +func fetchHugepageInfo(sizeKb int, dir string) (SHugepageInfo, error) { + info := SHugepageInfo{ + SizeKb: sizeKb, + } + cont, err := ioutil.ReadFile(filepath.Join(dir, "nr_hugepages")) + if err != nil { + return info, errors.Wrap(err, "FileGetContents nr_hugepages") + } + total, _ := strconv.Atoi(strings.TrimSpace(string(cont))) + cont, err = ioutil.ReadFile(filepath.Join(dir, "free_hugepages")) + if err != nil { + return info, errors.Wrap(err, "FileGetContents free_hugepages") + } + free, _ := strconv.Atoi(strings.TrimSpace(string(cont))) + info.Total = int(total) + info.Free = int(free) + return info, nil +} + +func GetHugepages() (THugepages, error) { + const hugepageDir = "/sys/kernel/mm/hugepages" + files, err := ioutil.ReadDir(hugepageDir) + if err != nil { + return nil, errors.Wrapf(err, "ReadDir %s", hugepageDir) + } + re := regexp.MustCompile(`hugepages-(\d+)kB`) + infos := make(THugepages, 0) + for _, dir := range files { + if !dir.IsDir() { + continue + } + ms := re.FindAllStringSubmatch(dir.Name(), -1) + if len(ms) > 0 && len(ms[0]) > 1 { + sizeKb, _ := strconv.Atoi(ms[0][1]) + if sizeKb > 0 { + info, err := fetchHugepageInfo(sizeKb, filepath.Join(hugepageDir, dir.Name())) + if err != nil { + log.Errorf("fetchHugepageInfo %s fail %s", dir.Name(), err) + } else { + infos = append(infos, info) + } + } + } + } + sort.Sort(infos) + return infos, nil +} diff --git a/pkg/util/sysutils/hugepages_test.go b/pkg/util/sysutils/hugepages_test.go new file mode 100644 index 0000000000..a757462684 --- /dev/null +++ b/pkg/util/sysutils/hugepages_test.go @@ -0,0 +1,30 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sysutils + +import ( + "testing" + + "yunion.io/x/jsonutils" +) + +func TestGetHugepages(t *testing.T) { + hp, err := GetHugepages() + if err != nil { + t.Errorf("GetHugepages fail %s", err) + } else { + t.Logf("%s: size: %dMb", jsonutils.Marshal(hp), hp.BytesMb()) + } +} diff --git a/pkg/util/zeroclean/doc.go b/pkg/util/zeroclean/doc.go new file mode 100644 index 0000000000..8866702b30 --- /dev/null +++ b/pkg/util/zeroclean/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zeroclean // import "yunion.io/x/onecloud/pkg/util/zeroclean" diff --git a/pkg/util/zeroclean/zeroclean.go b/pkg/util/zeroclean/zeroclean.go new file mode 100644 index 0000000000..a51efcd367 --- /dev/null +++ b/pkg/util/zeroclean/zeroclean.go @@ -0,0 +1,67 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package zeroclean + +import ( + "os" + "path/filepath" + + "yunion.io/x/pkg/errors" +) + +func ZeroFile(filename string) error { + f, err := os.OpenFile(filename, os.O_RDWR, 0644) + if err != nil { + return errors.Wrap(err, "os.OpenFile") + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return errors.Wrap(err, "f.Stat") + } + + zeroBuf := make([]byte, 4096) + offset := int64(0) + + for offset < info.Size() { + if offset+int64(len(zeroBuf)) > info.Size() { + zeroBuf = zeroBuf[:info.Size()-offset] + } + n, err := f.WriteAt(zeroBuf, offset) + if err != nil { + return errors.Wrapf(err, "zero at %d", offset) + } + offset += int64(n) + } + + return nil +} + +func ZeroDir(dirname string) error { + err := filepath.Walk(dirname, func(path string, d os.FileInfo, err error) error { + if err != nil { + return errors.Wrapf(err, "WalkDIr %s", path) + } + if !d.IsDir() { + err := ZeroFile(path) + if err != nil { + return errors.Wrapf(err, "Zerofiles %s", path) + } + } + return nil + }) + return errors.Wrap(err, "filepath.WalkDir") +}