Merge pull request #13376 from swordqiu/feature/qj-native-hugepage-cleanup

feature: cleanup native hugepage codes

pkg/compute/guestdrivers/kvm.go

@@ -391,9 +391,9 @@ func (self *SKVMGuestDriver) RequestAssociateEip(ctx context.Context, userCred m
 }
 
 func (self *SKVMGuestDriver) NeedStopForChangeSpec(ctx context.Context, guest *models.SGuest, cpuChanged, memChanged bool) bool {
-	return guest.GetMetadata(ctx, "hotplug_cpu_mem", nil) != "enable" ||
-		(memChanged && guest.GetMetadata(ctx, "__hugepage", nil) == "native") ||
-		apis.IsARM(guest.OsArch)
+	return guest.GetMetadata(ctx, "hotplug_cpu_mem", nil) != "enable" || apis.IsARM(guest.OsArch)
+	// (memChanged && guest.GetMetadata(ctx, "__hugepage", nil) == "native") ||
+	// apis.IsARM(guest.OsArch)
 }
 
 func (self *SKVMGuestDriver) RequestChangeVmConfig(ctx context.Context, guest *models.SGuest, task taskman.ITask, instanceType string, vcpuCount, vmemSize int64) error {

pkg/hostman/guestman/guesttasks.go

@@ -1529,11 +1529,43 @@ func (task *SGuestHotplugCpuMemTask) startAddMem() {
 func (task *SGuestHotplugCpuMemTask) onGetSlotIndex(index int) {
 	var newIndex = index
 	task.memSlotNewIndex = &newIndex
-	params := map[string]string{
-		"id":   fmt.Sprintf("mem%d", *task.memSlotNewIndex),
-		"size": fmt.Sprintf("%dM", task.addMemSize),
+	if task.manager.host.IsHugepagesEnabled() {
+		memPath := fmt.Sprintf("/dev/hugepages/%s-%d", task.GetId(), index)
+
+		err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", memPath).Run()
+		if err != nil {
+			reason := fmt.Sprintf("mkdir %s fail: %s", memPath, err)
+			log.Errorf("%s", reason)
+			task.onFail(reason)
+			return
+		}
+		err = procutils.NewRemoteCommandAsFarAsPossible("mount", "-t", "hugetlbfs", "-o",
+			fmt.Sprintf("pagesize=%dK,size=%dM", task.manager.host.HugepageSizeKb(), task.addMemSize),
+			fmt.Sprintf("hugetlbfs-%s-%d", task.GetId(), index),
+			memPath,
+		).Run()
+		if err != nil {
+			reason := fmt.Sprintf("mount %s fail: %s", memPath, err)
+			log.Errorf("%s", reason)
+			task.onFail(reason)
+			return
+		}
+
+		params := map[string]string{
+			"id":       fmt.Sprintf("mem%d", *task.memSlotNewIndex),
+			"size":     fmt.Sprintf("%dM", task.addMemSize),
+			"mem-path": memPath,
+			"share":    "on",
+			"prealloc": "on",
+		}
+		task.Monitor.ObjectAdd("memory-backend-file", params, task.onAddMemObject)
+	} else {
+		params := map[string]string{
+			"id":   fmt.Sprintf("mem%d", *task.memSlotNewIndex),
+			"size": fmt.Sprintf("%dM", task.addMemSize),
+		}
+		task.Monitor.ObjectAdd("memory-backend-ram", params, task.onAddMemObject)
 	}
-	task.Monitor.ObjectAdd("memory-backend-ram", params, task.onAddMemObject)
 }
 
 func (task *SGuestHotplugCpuMemTask) onAddMemFailed(reason string) {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -292,8 +292,8 @@ func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (strin
 
 	if input.HugepagesEnabled {
 		cmd += fmt.Sprintf("mkdir -p /dev/hugepages/%s\n", input.UUID)
-		cmd += fmt.Sprintf("mount -t hugetlbfs -o size=%dM hugetlbfs-%s /dev/hugepages/%s\n",
-			input.Mem, input.UUID, input.UUID)
+		cmd += fmt.Sprintf("mount -t hugetlbfs -o pagesize=%dK,size=%dM hugetlbfs-%s /dev/hugepages/%s\n",
+			s.manager.host.HugepageSizeKb(), input.Mem, input.UUID, input.UUID)
 	}
 
 	cmd += "sleep 1\n"
@@ -627,12 +627,14 @@ func (s *SKVMGuestInstance) generateStopScript(data *jsonutils.JSONDict) string
 	cmd += "  rm -f $PID_FILE\n"
 	cmd += "fi\n"
 
-	if s.manager.host.IsHugepagesEnabled() {
-		cmd += fmt.Sprintf("if [ -d /dev/hugepages/%s ]; then\n", uuid)
-		cmd += fmt.Sprintf("  umount /dev/hugepages/%s\n", uuid)
-		cmd += fmt.Sprintf("  rm -rf /dev/hugepages/%s\n", uuid)
-		cmd += "fi\n"
-	}
+	cmd += fmt.Sprintf("for d in $(ls -d /dev/hugepages/%s*)\n", uuid)
+	cmd += fmt.Sprintf("do\n")
+	cmd += fmt.Sprintf("  if [ -d $d ]; then\n")
+	cmd += fmt.Sprintf("    umount $d\n")
+	cmd += fmt.Sprintf("    rm -rf $d\n")
+	cmd += fmt.Sprintf("  fi\n")
+	cmd += fmt.Sprintf("done\n")
+
 	for _, nic := range nics {
 		ifname, _ := nic.GetString("ifname")
 		downscript := s.getNicDownScriptPath(nic)

pkg/hostman/guestman/qemu/generate.go

@@ -127,12 +127,13 @@ func GenerateStartOptions(
 		drvOpt.Memory(input.Mem),
 	)
 
+	var memDev string
 	if input.HugepagesEnabled {
-		opts = append(opts,
-			drvOpt.MemPrealloc(),
-			drvOpt.MemPath(fmt.Sprintf("/dev/hugepages/%s", input.UUID)),
-		)
+		memDev = drvOpt.MemPath(input.Mem, fmt.Sprintf("/dev/hugepages/%s", input.UUID))
+	} else {
+		memDev = drvOpt.MemDev(input.Mem)
 	}
+	opts = append(opts, memDev)
 
 	// bootOrder
 	enableMenu := false

pkg/hostman/guestman/qemu/qemu.go

@@ -93,8 +93,8 @@ type QemuOptions interface {
 	Name(name string) string
 	UUID(enable bool, uuid string) string
 	Memory(sizeMB uint64) string
-	MemPrealloc() string
-	MemPath(p string) string
+	MemPath(sizeMB uint64, p string) string
+	MemDev(sizeMB uint64) string
 	Boot(order string, enableMenu bool) string
 	BIOS(file string) string
 	Device(devStr string) string
@@ -240,8 +240,12 @@ func (o baseOptions) MemPrealloc() string {
 	return "-mem-prealloc"
 }
 
-func (o baseOptions) MemPath(p string) string {
-	return "-mem-path " + p
+func (o baseOptions) MemPath(sizeMB uint64, p string) string {
+	return fmt.Sprintf("-object memory-backend-file,id=mem,size=%dM,mem-path=%s,share=on,prealloc=on -numa node,memdev=mem", sizeMB, p)
+}
+
+func (o baseOptions) MemDev(sizeMB uint64) string {
+	return fmt.Sprintf("-object memory-backend-ram,id=mem,size=%dM -numa node,memdev=mem", sizeMB)
 }
 
 func (o baseOptions) Boot(order string, enableMenu bool) string {

pkg/hostman/hostinfo/hostinfo.go

@@ -17,6 +17,7 @@ package hostinfo
 import (
 	"context"
 	"fmt"
+	"math"
 	"net"
 	"os"
 	"path"
@@ -75,9 +76,8 @@ type SHostInfo struct {
 
 	kubeletConfig kubelet.KubeletConfig
 
-	isInit          bool
-	enableHugePages bool
-	onHostDown      string
+	isInit     bool
+	onHostDown string
 
 	IsolatedDeviceMan isolated_device.IsolatedDeviceManager
 
@@ -151,7 +151,11 @@ func (h *SHostInfo) IsNestedVirtualization() bool {
 }
 
 func (h *SHostInfo) IsHugepagesEnabled() bool {
-	return h.enableHugePages || options.HostOptions.HugepagesOption == "native"
+	return options.HostOptions.HugepagesOption == "native"
+}
+
+func (h *SHostInfo) HugepageSizeKb() int {
+	return h.sysinfo.HugepageSizeKb
 }
 
 /* In this order init host service:
@@ -269,9 +273,8 @@ func (h *SHostInfo) generateLocalNetworkConfig() (string, error) {
 }
 
 func (h *SHostInfo) parseConfig() error {
-	if mem, err := h.GetMemory(); err != nil {
-		return err
-	} else if mem < 64 { // MB
+	mem := h.GetMemory()
+	if mem < 64 { // MB
 		return fmt.Errorf("Not enough memory!")
 	}
 	if len(options.HostOptions.Networks) == 0 {
@@ -393,7 +396,7 @@ func (h *SHostInfo) prepareEnv() error {
 	//  return err
 	// }
 
-	if options.HostOptions.EnableKsm {
+	if options.HostOptions.EnableKsm && options.HostOptions.HugepagesOption == "disable" {
 		h.EnableKsm(900)
 	} else {
 		h.DisableKsm()
@@ -403,19 +406,30 @@ func (h *SHostInfo) prepareEnv() error {
 	case "disable":
 		h.DisableHugepages()
 	case "native":
-		size, err := h.Mem.GetHugepageTotal()
+		err := h.EnableNativeHugepages(0)
 		if err != nil {
-			return err
+			return errors.Wrap(err, "EnableNativeHugepages")
 		}
-		if size <= 0 {
+		hp, err := h.Mem.GetHugepages()
+		if err != nil {
+			return errors.Wrap(err, "Mem.GetHugepages")
+		}
+		szlist := hp.PageSizes()
+		if len(szlist) == 0 {
 			return errors.New("invalid hugepages total size")
 		}
+		if len(szlist) > 1 {
+			return errors.New("cannot support more than 1 type of hugepage size")
+		}
+		h.sysinfo.HugepageSizeKb = szlist[0]
 	case "transparent":
 		h.EnableTransparentHugepages()
 	default:
 		return fmt.Errorf("Invalid hugepages option")
 	}
 
+	h.sysinfo.HugepagesOption = options.HostOptions.HugepagesOption
+
 	h.PreventArpFlux()
 	h.tuneSystem()
 	return nil
@@ -521,23 +535,11 @@ func (h *SHostInfo) EnableTransparentHugepages() {
 	}
 }
 
-func (h *SHostInfo) GetMemory() (int, error) {
-	if options.HostOptions.HugepagesOption == "native" {
-		return h.Mem.GetHugepageTotal()
-	}
-	total := h.Mem.Total
-	if h.kubeletConfig != nil {
-		memThreshold := h.kubeletConfig.GetEvictionConfig().GetHard().GetMemoryAvailable()
-		memBytes, _ := memThreshold.Value.Quantity.AsInt64()
-		memMb := int(memBytes / 1024 / 1024)
-		subMem := total - memMb
-		log.Infof("Get total memory %d, kubelet memory threshold subtracted: (%d - %d)", subMem, total, memMb)
-		total = subMem
-	}
-	return total, nil // - options.reserved_memory
+func (h *SHostInfo) GetMemory() int {
+	return h.Mem.Total
 }
 
-func (h *SHostInfo) getCurrentHugepageNr() (int64, error) {
+/* func (h *SHostInfo) getCurrentHugepageNr() (int64, error) {
 	nrStr, err := fileutils2.FileGetContents("/proc/sys/vm/nr_hugepages")
 	if err != nil {
 		return 0, errors.Wrap(err, "file get content nr hugepages")
@@ -547,9 +549,9 @@ func (h *SHostInfo) getCurrentHugepageNr() (int64, error) {
 		return 0, errors.Wrap(err, "nr str atoi")
 	}
 	return int64(nr), nil
-}
+} */
 
-func (h *SHostInfo) EnableNativeHugepages() error {
+func (h *SHostInfo) EnableNativeHugepages(reservedMb int) error {
 	kv := map[string]string{
 		"/sys/kernel/mm/transparent_hugepage/enabled": "never",
 		"/sys/kernel/mm/transparent_hugepage/defrag":  "never",
@@ -557,32 +559,42 @@ func (h *SHostInfo) EnableNativeHugepages() error {
 	for k, v := range kv {
 		sysutils.SetSysConfig(k, v)
 	}
-	nr, err := h.getCurrentHugepageNr()
+	// check reserved memory
+	hp, err := h.Mem.GetHugepages()
 	if err != nil {
 		return err
 	}
-	mem, err := h.GetMemory()
-	if err != nil {
-		return err
-	}
-	mem -= h.getReservedMem()
-	desiredNr := int64(mem/h.Mem.GetHugepagesizeMb() + 1)
-	if nr < desiredNr {
-		err = timeutils2.CommandWithTimeout(1, "sh", "-c",
-			fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", desiredNr)).Run()
-		if err != nil {
-			return err
+	pgList := hp.PageSizes()
+	if len(pgList) == 0 {
+		// not initialized yet, manually setup, usage page_size 2MB
+	} else if len(pgList) == 1 {
+		// already setup, depends on the PageSize
+		if pgList[0] == 2048 {
+		} else {
+			// readonly, cannot adjust any more
+			return nil
 		}
+	} else {
+		return errors.New("cannot support more than 1 type of hugepage sizes")
 	}
-	currentNr, err := h.getCurrentHugepageNr()
+	mem := h.GetMemory()
+	if reservedMb > 0 {
+		mem -= reservedMb
+	} else {
+		mem -= h.getReservedMemMb()
+	}
+	desiredSz := 2 // ONLY 2MB Hugepage Supported
+	desiredNr := mem / desiredSz
+	if desiredNr*desiredSz < mem {
+		desiredNr += 1
+	}
+	log.Infof("Hugepage %dGB(%dMB) available Mem %dMB, to reserve %d hugepages with size %d", hp.BytesMb()/1024, hp.BytesMb(), mem, desiredNr, desiredSz)
+	// not setup hugepage yet, or reserved too many hugepages
+	err = timeutils2.CommandWithTimeout(1, "sh", "-c",
+		fmt.Sprintf("echo %d > /sys/kernel/mm/hugepages/hugepages-%dkB/nr_hugepages", desiredNr, desiredSz*1024)).Run()
 	if err != nil {
 		return err
 	}
-	if currentNr < desiredNr {
-		err = timeutils2.CommandWithTimeout(1, "sh", "-c",
-			fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", nr)).Run()
-		return fmt.Errorf("no enough memory to resize hugepage, current nr %d, desired nr %d", currentNr, desiredNr)
-	}
 	return nil
 }
 
@@ -1083,14 +1095,11 @@ func (h *SHostInfo) updateHostRecord(hostId string) {
 		content.Set("cpu_mhz", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Freq)))
 	}
 	content.Set("cpu_cache", jsonutils.NewInt(int64(h.Cpu.cpuInfoProc.Cache)))
-	memTotal, err := h.GetMemory()
-	if err != nil {
-		h.onFail(err)
-		return
-	}
+	memTotal := h.GetMemory()
 	content.Set("mem_size", jsonutils.NewInt(int64(memTotal)))
 	if len(hostId) == 0 {
-		content.Set("mem_reserved", jsonutils.NewInt(int64(h.getReservedMem())))
+		// first time create
+		content.Set("mem_reserved", jsonutils.NewInt(int64(h.getReservedMemMb())))
 	}
 	content.Set("storage_driver", jsonutils.NewString(api.DISK_DRIVER_LINUX))
 	content.Set("storage_type", jsonutils.NewString(h.sysinfo.StorageType))
@@ -1112,6 +1121,7 @@ func (h *SHostInfo) updateHostRecord(hostId string) {
 
 	var (
 		res jsonutils.JSONObject
+		err error
 	)
 	if !h.isInit {
 		res, err = modules.Hosts.Update(h.GetSession(), hostId, content)
@@ -1159,42 +1169,29 @@ func (h *SHostInfo) onUpdateHostInfoSucc(hostbody jsonutils.JSONObject) {
 		h.onFail(err)
 		return
 	}
-
-	if options.HostOptions.HugepagesOption == "native" {
-		if h.isInit && len(h.IsolatedDeviceMan.GetDevices()) > 0 {
-			meta := jsonutils.NewDict()
-			meta.Set("__enable_hugepages", jsonutils.NewString("true"))
-			_, err := modules.Hosts.SetMetadata(h.GetSession(), h.HostId, meta)
-			if err != nil {
-				h.onFail(fmt.Sprintf("failed "))
-				return
-			}
-			h.enableHugePages = true
-		} else if hugepage, _ := hostbody.GetString("metadata", "__enable_hugepages"); hugepage == "true" {
-			h.enableHugePages = true
-		}
-		if h.enableHugePages {
-			err := h.EnableNativeHugepages()
-			if err != nil {
-				h.onFail(err)
-				return
-			}
-		}
-	}
 	h.onHostDown, _ = hostbody.GetString("metadata", "__on_host_down")
 
-	if memReserved, _ := hostbody.Int("mem_reserved"); memReserved == 0 {
-		h.updateHostReservedMem()
+	memReservedMb, _ := hostbody.Int("mem_reserved")
+	if options.HostOptions.HugepagesOption == "native" && memReservedMb > int64(h.getReservedMemMb()) {
+		err := h.EnableNativeHugepages(int(memReservedMb))
+		if err != nil {
+			h.onFail(err)
+			return
+		}
+	}
+
+	reserved := h.getReportedReservedMemMb()
+	if reserved != int(memReservedMb) {
+		h.updateHostReservedMem(reserved)
 	} else {
 		h.PutHostOffline("")
 	}
 }
 
-func (h *SHostInfo) updateHostReservedMem() {
+func (h *SHostInfo) updateHostReservedMem(reserved int) {
 	content := jsonutils.NewDict()
-	content.Set("mem_reserved", jsonutils.NewInt(int64(h.getReservedMem())))
-	res, err := modules.Hosts.Update(h.GetSession(),
-		h.HostId, content)
+	content.Set("mem_reserved", jsonutils.NewInt(int64(reserved)))
+	res, err := modules.Hosts.Update(h.GetSession(), h.HostId, content)
 	if err != nil {
 		h.onFail(err)
 		return
@@ -1203,7 +1200,20 @@ func (h *SHostInfo) updateHostReservedMem() {
 	}
 }
 
-func (h *SHostInfo) getReservedMem() int {
+func (h *SHostInfo) getKubeReservedMemMb() int {
+	// reserved for Kubelet
+	if h.kubeletConfig != nil {
+		memThreshold := h.kubeletConfig.GetEvictionConfig().GetHard().GetMemoryAvailable()
+		memBytes, _ := memThreshold.Value.Quantity.AsInt64()
+		memMb := int(math.Ceil(float64(memBytes) / 1024 / 1024))
+		log.Infof("Kubelet memory threshold subtracted: %dMB", memMb)
+		return memMb
+	}
+	return 0
+}
+
+func (h *SHostInfo) getOSReservedMemMb() int {
+	// reserved memory for OS
 	reserved := h.Mem.MemInfo.Total / 10
 	if reserved > options.HostOptions.MaxReservedMemory {
 		return options.HostOptions.MaxReservedMemory
@@ -1214,6 +1224,19 @@ func (h *SHostInfo) getReservedMem() int {
 	return reserved
 }
 
+func (h *SHostInfo) getReservedMemMb() int {
+	return h.getOSReservedMemMb() + h.getKubeReservedMemMb()
+}
+
+func (h *SHostInfo) getReportedReservedMemMb() int {
+	if options.HostOptions.HugepagesOption == "native" {
+		// return total minus mem in huagepage pool
+		hp, _ := h.Mem.GetHugepages()
+		return h.GetMemory() - int(hp.BytesMb())
+	}
+	return h.getReservedMemMb()
+}
+
 func (h *SHostInfo) PutHostOffline(reason string) {
 	data := jsonutils.NewDict()
 	if options.HostOptions.EnableHealthChecker {

pkg/hostman/hostinfo/hostinfohelper.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -166,67 +165,8 @@ func DetectMemoryInfo() (*SMemory, error) {
 	return smem, nil
 }
 
-func (m *SMemory) GetHugepageTotal() (int, error) {
-	file, err := os.Open("/proc/meminfo")
-	if err != nil {
-		return 0, errors.Wrap(err, "open meminfo")
-	}
-	defer file.Close()
-	var (
-		nrHugePage   int
-		sizeHugePage int
-	)
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.HasPrefix(line, "Hugepagesize:") {
-			re := regexp.MustCompile(`\s+`)
-			segs := re.Split(line, -1)
-			v, err := strconv.Atoi(segs[1])
-			if err != nil {
-				return 0, errors.Wrap(err, "get hugepage size")
-			}
-			sizeHugePage = v / 1024 // MB
-			log.Debugf("Huge page size %v", sizeHugePage)
-		} else if strings.HasPrefix(line, "HugePages_Total:") {
-			re := regexp.MustCompile(`\s+`)
-			segs := re.Split(line, -1)
-			v, err := strconv.Atoi(segs[1])
-			if err != nil {
-				return 0, errors.Wrap(err, "get hugepages total")
-			}
-			nrHugePage = v
-			log.Debugf("Huge page number %v", nrHugePage)
-		}
-	}
-	return nrHugePage * sizeHugePage, nil
-}
-
-func (m *SMemory) GetHugepagesizeMb() int {
-	file, err := os.Open("/proc/meminfo")
-	if err != nil {
-		log.Errorln(err)
-		return 0
-	}
-	defer file.Close()
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.HasPrefix(line, "Hugepagesize:") {
-			re := regexp.MustCompile(`\s+`)
-			segs := re.Split(line, -1)
-			v, err := strconv.Atoi(segs[1])
-			if err != nil {
-				log.Errorln(err)
-				return 0
-			}
-			return int(v) / 1024
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		log.Errorln(err)
-	}
-	return 0
+func (m *SMemory) GetHugepages() (sysutils.THugepages, error) {
+	return sysutils.GetHugepages()
 }
 
 type SNIC struct {
@@ -381,6 +321,9 @@ type SSysInfo struct {
 	CpuMicrocode   string `json:"cpu_microcode"`
 
 	StorageType string `json:"storage_type"`
+
+	HugepagesOption string `json:"hugepages_option"`
+	HugepageSizeKb  int    `json:"hugepage_size_kb"`
 }
 
 func StartDetachStorages(hs []jsonutils.JSONObject) {

pkg/hostman/hostutils/hostutils.go

@@ -43,7 +43,9 @@ type IHost interface {
 	GetMasterIp() string
 	GetCpuArchitecture() string
 	IsAarch64() bool
+
 	IsHugepagesEnabled() bool
+	HugepageSizeKb() int
 
 	IsKvmSupport() bool
 	IsNestedVirtualization() bool

pkg/hostman/options/options.go

@@ -42,6 +42,8 @@ type SHostOptions struct {
 	RecycleDiskfile         bool `help:"Recycle instead of remove deleted disk file" default:"true"`
 	RecycleDiskfileKeepDays int  `help:"How long recycled files kept, default 28 days" default:"28"`
 
+	ZeroCleanDiskData bool `help:"Clean disk data by writing zeros" default:"false"`
+
 	EnableTemplateBacking    bool `help:"Use template as backing file"`
 	AutoMergeBackingTemplate bool `help:"Automatically stream merging backing file"`
 	AutoMergeDelaySeconds    int  `help:"Seconds to delay mergeing backing file after VM start, default 15 minutes" default:"900"`

pkg/hostman/storageman/core.go

@@ -37,6 +37,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
+	"yunion.io/x/onecloud/pkg/util/zeroclean"
 )
 
 const MINIMAL_FREE_SPACE = 128
@@ -354,6 +355,10 @@ func cleanDailyFiles(storagePath, subDir string, keepDay int) {
 		if date.Before(markTime) {
 			log.Infof("Cron Job Clean Recycle Bin: start delete %s", file.Name())
 			subDirPath := path.Join(recycleDir, file.Name())
+			if options.HostOptions.ZeroCleanDiskData {
+				// try to zero clean files in subdir
+				zeroclean.ZeroDir(subDirPath)
+			}
 			if output, err := procutils.NewCommand("rm", "-rf", subDirPath).Output(); err != nil {
 				log.Errorf("clean recycle dir %s error: %s, %s", subDirPath, err, output)
 			}

pkg/util/sysutils/hugepages.go

@@ -0,0 +1,109 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sysutils
+
+import (
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+)
+
+type SHugepageInfo struct {
+	SizeKb int
+	Total  int
+	Free   int
+}
+
+func (h SHugepageInfo) BytesMb() int64 {
+	return int64(h.Total) * int64(h.SizeKb) / 1024
+}
+
+type THugepages []SHugepageInfo
+
+func (a THugepages) Len() int           { return len(a) }
+func (a THugepages) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a THugepages) Less(i, j int) bool { return a[i].SizeKb < a[j].SizeKb }
+
+func (a THugepages) BytesMb() int64 {
+	ret := int64(0)
+	for _, h := range a {
+		ret += h.BytesMb()
+	}
+	return ret
+}
+
+func (a THugepages) PageSizes() []int {
+	ret := make([]int, 0)
+	for _, h := range a {
+		if h.Total > 0 {
+			ret = append(ret, h.SizeKb)
+		}
+	}
+	return ret
+}
+
+func fetchHugepageInfo(sizeKb int, dir string) (SHugepageInfo, error) {
+	info := SHugepageInfo{
+		SizeKb: sizeKb,
+	}
+	cont, err := ioutil.ReadFile(filepath.Join(dir, "nr_hugepages"))
+	if err != nil {
+		return info, errors.Wrap(err, "FileGetContents nr_hugepages")
+	}
+	total, _ := strconv.Atoi(strings.TrimSpace(string(cont)))
+	cont, err = ioutil.ReadFile(filepath.Join(dir, "free_hugepages"))
+	if err != nil {
+		return info, errors.Wrap(err, "FileGetContents free_hugepages")
+	}
+	free, _ := strconv.Atoi(strings.TrimSpace(string(cont)))
+	info.Total = int(total)
+	info.Free = int(free)
+	return info, nil
+}
+
+func GetHugepages() (THugepages, error) {
+	const hugepageDir = "/sys/kernel/mm/hugepages"
+	files, err := ioutil.ReadDir(hugepageDir)
+	if err != nil {
+		return nil, errors.Wrapf(err, "ReadDir %s", hugepageDir)
+	}
+	re := regexp.MustCompile(`hugepages-(\d+)kB`)
+	infos := make(THugepages, 0)
+	for _, dir := range files {
+		if !dir.IsDir() {
+			continue
+		}
+		ms := re.FindAllStringSubmatch(dir.Name(), -1)
+		if len(ms) > 0 && len(ms[0]) > 1 {
+			sizeKb, _ := strconv.Atoi(ms[0][1])
+			if sizeKb > 0 {
+				info, err := fetchHugepageInfo(sizeKb, filepath.Join(hugepageDir, dir.Name()))
+				if err != nil {
+					log.Errorf("fetchHugepageInfo %s fail %s", dir.Name(), err)
+				} else {
+					infos = append(infos, info)
+				}
+			}
+		}
+	}
+	sort.Sort(infos)
+	return infos, nil
+}

pkg/util/sysutils/hugepages_test.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sysutils
+
+import (
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestGetHugepages(t *testing.T) {
+	hp, err := GetHugepages()
+	if err != nil {
+		t.Errorf("GetHugepages fail %s", err)
+	} else {
+		t.Logf("%s: size: %dMb", jsonutils.Marshal(hp), hp.BytesMb())
+	}
+}

pkg/util/zeroclean/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zeroclean // import "yunion.io/x/onecloud/pkg/util/zeroclean"

pkg/util/zeroclean/zeroclean.go

@@ -0,0 +1,67 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package zeroclean
+
+import (
+	"os"
+	"path/filepath"
+
+	"yunion.io/x/pkg/errors"
+)
+
+func ZeroFile(filename string) error {
+	f, err := os.OpenFile(filename, os.O_RDWR, 0644)
+	if err != nil {
+		return errors.Wrap(err, "os.OpenFile")
+	}
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return errors.Wrap(err, "f.Stat")
+	}
+
+	zeroBuf := make([]byte, 4096)
+	offset := int64(0)
+
+	for offset < info.Size() {
+		if offset+int64(len(zeroBuf)) > info.Size() {
+			zeroBuf = zeroBuf[:info.Size()-offset]
+		}
+		n, err := f.WriteAt(zeroBuf, offset)
+		if err != nil {
+			return errors.Wrapf(err, "zero at %d", offset)
+		}
+		offset += int64(n)
+	}
+
+	return nil
+}
+
+func ZeroDir(dirname string) error {
+	err := filepath.Walk(dirname, func(path string, d os.FileInfo, err error) error {
+		if err != nil {
+			return errors.Wrapf(err, "WalkDIr %s", path)
+		}
+		if !d.IsDir() {
+			err := ZeroFile(path)
+			if err != nil {
+				return errors.Wrapf(err, "Zerofiles %s", path)
+			}
+		}
+		return nil
+	})
+	return errors.Wrap(err, "filepath.WalkDir")
+}