feat(region,host): mem clean after guest exited. (#14703)

Signed-off-by: wanyaoqi <d3lx.yq@gmail.com>

Co-authored-by: wanyaoqi <d3lx.yq@gmail.com>

cmd/climc/shell/compute/servers.go

@@ -106,6 +106,7 @@ func init() {
 	cmd.Perform("calculate-record-checksum", &options.ServerIdOptions{})
 	cmd.Perform("set-class-metadata", &baseoptions.ResourceMetadataOptions{})
 	cmd.Perform("monitor", &options.ServerMonitorOptions{})
+	cmd.BatchPerform("enable-memclean", new(options.ServerIdsOptions))
 
 	cmd.Get("vnc", new(options.ServerIdOptions))
 	cmd.Get("desc", new(options.ServerIdOptions))

cmd/memclean/main.go

@@ -0,0 +1,187 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"syscall"
+	"time"
+
+	"github.com/sevlyar/go-daemon"
+	"golang.org/x/sys/unix"
+
+	"yunion.io/x/log"
+	"yunion.io/x/log/hooks"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/signalutils"
+	"yunion.io/x/pkg/utils"
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+const (
+	MEM_BACKEND_FD = "/memfd:memory-backend-memfd (deleted)"
+	MMAP_SIZE      = 2 * 1024 * 1024 * 1024
+)
+
+type Options struct {
+	Pid        int    `help:"qemu process pid" required:"true"`
+	MemSize    int64  `help:"backend memory size" required:"true"`
+	Foreground bool   `help:"run in foreground"`
+	LogDir     string `help:"log dir" required:"true"`
+}
+
+var opt = &Options{}
+
+func main() {
+	procDir := fmt.Sprintf("/proc/%d", opt.Pid)
+	if !fileutils2.IsDir(procDir) {
+		log.Fatalf("Process %d not found", opt.Pid)
+	}
+
+	fdDir := fmt.Sprintf("%s/fd", procDir)
+	memBackendFd, err := findMemBackendFd(fdDir)
+	if err != nil {
+		log.Fatalf("findMemBackendFd: %s", err)
+	}
+	log.Infof("found mem backend fd: %s", memBackendFd)
+
+	f, err := os.OpenFile(memBackendFd, os.O_RDWR, 0644)
+	if err != nil {
+		log.Fatalf("failed open memfd: %s", err)
+	}
+	defer f.Close()
+
+	if !opt.Foreground {
+		cntxt := &daemon.Context{
+			WorkDir: "./",
+			Umask:   027,
+		}
+
+		d, err := cntxt.Reborn()
+		if err != nil {
+			log.Fatalf("Unable to run in background: %s", err)
+		}
+		if d != nil {
+			return
+		}
+		defer cntxt.Release()
+	}
+
+	log.Infof("start watch proc exit")
+	err = procutils.NewCommand("tail", fmt.Sprintf("--pid=%d", opt.Pid), "-f", "/dev/null").Run()
+	if err != nil {
+		log.Fatalf("failed watch process: %s", err)
+	}
+
+	log.Infof("watch proc exited, go to clean memory")
+	var (
+		start       = time.Now()
+		size  int64 = 0
+	)
+	for size < opt.MemSize {
+		mmapSize := MMAP_SIZE
+		if opt.MemSize-size < MMAP_SIZE {
+			mmapSize = int(opt.MemSize - size)
+		}
+
+		b, err := syscall.Mmap(
+			int(f.Fd()), size, mmapSize,
+			syscall.PROT_WRITE|syscall.PROT_READ,
+			syscall.MAP_SHARED,
+		)
+		if err != nil {
+			log.Fatalf("failed mmap mem backend fd: %s", err)
+		}
+		log.Infof("mmap memory offset %d, size %d", size, len(b))
+		size += int64(len(b))
+
+		// memsetRepeat(b, 0)
+		for i := 0; i < len(b); i++ {
+			b[i] = 0
+		}
+		unix.Msync(b, unix.MS_SYNC)
+		syscall.Munmap(b)
+	}
+
+	log.Infof(
+		"mem clean for process %d success, mem clean took %s",
+		opt.Pid, time.Since(start),
+	)
+}
+
+func findMemBackendFd(fdDir string) (string, error) {
+	files, err := ioutil.ReadDir(fdDir)
+	if err != nil {
+		return "", errors.Wrapf(err, "read dir %s", fdDir)
+	}
+
+	for _, f := range files {
+		p1 := path.Join(fdDir, f.Name())
+		p2, e := os.Readlink(p1)
+		if e != nil {
+			log.Errorf("os.readlink %s: %s", p1, e)
+			continue
+		}
+		log.Infof("os.readlink path %s", p2)
+		if p2 == MEM_BACKEND_FD {
+			return p1, nil
+		}
+	}
+
+	return "", errors.Errorf("no mem backend fd found")
+}
+
+func memsetRepeat(a []byte, v byte) {
+	if len(a) == 0 {
+		return
+	}
+	a[0] = v
+	for bp := 1; bp < len(a); bp *= 2 {
+		copy(a[bp:], a[:bp])
+	}
+}
+
+func init() {
+	signalutils.RegisterSignal(func() {
+		utils.DumpAllGoroutineStack(log.Logger().Out)
+	}, syscall.SIGUSR1)
+	signalutils.StartTrap()
+
+	parser, err := structarg.NewArgumentParser(opt, "", "", "")
+	if err != nil {
+		log.Fatalf("Error define argument parser: %v", err)
+	}
+	err = parser.ParseArgs2(os.Args[1:], true, true)
+	if err != nil {
+		log.Fatalf("Failed parse args %s", err)
+	}
+
+	logFileHook := hooks.LogFileRotateHook{
+		LogFileHook: hooks.LogFileHook{
+			FileDir:  opt.LogDir,
+			FileName: "memclean.log",
+		},
+		RotateNum:  10,
+		RotateSize: 100 * 1024 * 1024,
+	}
+	logFileHook.Init()
+	log.Logger().AddHook(&logFileHook)
+}

pkg/apis/compute/api.go

@@ -365,7 +365,8 @@ type ServerCreateInput struct {
 	apis.EncryptedResourceCreateInput
 
 	// 虚拟机内存大小,单位Mb,若未指定instance_type,此参数为必传项
-	VmemSize int `json:"vmem_size"`
+	VmemSize       int  `json:"vmem_size"`
+	EnableMemclean bool `json:"enable_memclean"`
 
 	// 虚拟机Cpu大小,若未指定instance_type,此参数为必传项
 	// default: 1

pkg/apis/compute/guest_const.go

@@ -336,6 +336,7 @@ const (
 	VM_METADATA_OS_NAME             = "os_name"
 	VM_METADATA_OS_VERSION          = "os_version"
 	VM_METADATA_CGROUP_CPUSET       = "cgroup_cpuset"
+	VM_METADATA_ENABLE_MEMCLEAN     = "enable_memclean"
 )
 
 func Hypervisors2HostTypes(hypervisors []string) []string {

pkg/compute/models/guest_actions.go

@@ -5600,3 +5600,7 @@ func (self *SGuest) PerformCalculateRecordChecksum(ctx context.Context, userCred
 		"checksum": checksum,
 	}), nil
 }
+
+func (self *SGuest) PerformEnableMemclean(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	return nil, self.SetMetadata(ctx, api.VM_METADATA_ENABLE_MEMCLEAN, "true", userCred)
+}

pkg/compute/models/guests.go

@@ -2021,6 +2021,10 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
 	if osProfileJson != nil {
 		guest.setOSProfile(ctx, userCred, osProfileJson)
 	}
+
+	if jsonutils.QueryBoolean(data, api.VM_METADATA_ENABLE_MEMCLEAN, false) {
+		guest.SetMetadata(ctx, api.VM_METADATA_ENABLE_MEMCLEAN, "true", userCred)
+	}
 	if jsonutils.QueryBoolean(data, imageapi.IMAGE_DISABLE_USB_KBD, false) {
 		guest.SetMetadata(ctx, imageapi.IMAGE_DISABLE_USB_KBD, "true", userCred)
 	}

pkg/hostman/guestman/guesttasks.go

@@ -933,7 +933,11 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) {
 		/* ref: qemu/src/qapi/run-state.json
 		 * prelaunch: QEMU was started with -S and guest has not started.
 		 * we need resume guest at state prelaunch */
-		s.onGuestPrelaunch()
+		if err := s.onGuestPrelaunch(); err != nil {
+			s.ForceStop()
+			s.taskFailed(err.Error())
+			return
+		}
 		s.resumeGuest()
 	} else if status == "running" || status == "paused (suspended)" {
 		s.onStartRunning()
@@ -941,7 +945,11 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) {
 		// handle error first, results may be 'paused (internal-error)'
 		s.taskFailed(status)
 	} else if strings.Contains(status, "paused") {
-		s.onGuestPrelaunch()
+		if err := s.onGuestPrelaunch(); err != nil {
+			s.ForceStop()
+			s.taskFailed(err.Error())
+			return
+		}
 		s.Monitor.GetBlocks(s.onGetBlockInfo)
 	} else if status == "postmigrate" {
 		s.resumeGuest()
@@ -954,6 +962,7 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) {
 		log.Infof("start guest timeout seconds: %d", migSeconds)
 		if s.isTimeout && time.Now().Sub(s.startTime) >= time.Second*time.Duration(migSeconds) {
 			s.taskFailed("Timeout")
+			return
 		} else {
 			time.Sleep(time.Second * 3)
 			s.confirmRunning()

pkg/hostman/guestman/qemu-kvm.go

@@ -1855,14 +1855,20 @@ func (s *SKVMGuestInstance) doBlockIoThrottle() {
 	}
 }
 
-func (s *SKVMGuestInstance) onGuestPrelaunch() {
+func (s *SKVMGuestInstance) onGuestPrelaunch() error {
 	if options.HostOptions.SetVncPassword {
 		s.SetVncPassword()
 	}
+	if s.isMemcleanEnabled() {
+		if err := s.startMemCleaner(); err != nil {
+			return err
+		}
+	}
 	s.OnResumeSyncMetadataInfo()
 	s.SetCgroup()
 	s.optimizeOom()
 	s.doBlockIoThrottle()
+	return nil
 }
 
 func (s *SKVMGuestInstance) CleanImportMetadata() *jsonutils.JSONDict {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"net"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -112,6 +113,11 @@ func (s *SKVMGuestInstance) isWindows10() bool {
 	return false
 }
 
+func (s *SKVMGuestInstance) isMemcleanEnabled() bool {
+	val, _ := s.Desc.GetString("metadata", "enable_memclean")
+	return val == "true"
+}
+
 func (s *SKVMGuestInstance) getMachine() string {
 	machine, err := s.Desc.GetString("machine")
 	if err != nil {
@@ -242,6 +248,7 @@ func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (strin
 			OVNIntegrationBridge: options.HostOptions.OvnIntegrationBridge,
 			HomeDir:              s.HomeDir(),
 			HugepagesEnabled:     s.manager.host.IsHugepagesEnabled(),
+			EnableMemfd:          s.isMemcleanEnabled(),
 			PidFilePath:          s.GetPidFilePath(),
 			BIOS:                 s.getBios(),
 		}
@@ -769,3 +776,18 @@ func (s *SKVMGuestInstance) WriteMigrateCerts(certs map[string]string) error {
 	}
 	return nil
 }
+
+func (s *SKVMGuestInstance) startMemCleaner() error {
+	mem, _ := s.Desc.Int("mem")
+	err := procutils.NewRemoteCommandAsFarAsPossible(
+		options.HostOptions.BinaryMemcleanPath,
+		"--pid", strconv.Itoa(s.GetPid()),
+		"--mem-size", strconv.FormatInt(mem*1024*1024, 10),
+		"--log-dir", s.HomeDir(),
+	).Run()
+	if err != nil {
+		log.Errorf("failed start memcleaner: %s", err)
+		return errors.Wrap(err, "start memclean")
+	}
+	return nil
+}

pkg/hostman/guestman/qemu/generate.go

@@ -46,6 +46,7 @@ type GenerateStartOptionsInput struct {
 	Name                  string
 	OsName                string
 	HugepagesEnabled      bool
+	EnableMemfd           bool
 	IsQ35                 bool
 	BootOrder             string
 	CdromPath             string
@@ -132,6 +133,8 @@ func GenerateStartOptions(
 	var memDev string
 	if input.HugepagesEnabled {
 		memDev = drvOpt.MemPath(input.Mem, fmt.Sprintf("/dev/hugepages/%s", input.UUID))
+	} else if input.EnableMemfd {
+		memDev = drvOpt.MemFd(input.Mem)
 	} else {
 		memDev = drvOpt.MemDev(input.Mem)
 	}

pkg/hostman/guestman/qemu/qemu.go

@@ -96,6 +96,7 @@ type QemuOptions interface {
 	Memory(sizeMB uint64) string
 	MemPath(sizeMB uint64, p string) string
 	MemDev(sizeMB uint64) string
+	MemFd(sizeMB uint64) string
 	Boot(order string, enableMenu bool) string
 	BIOS(file string) string
 	Device(devStr string) string
@@ -253,6 +254,10 @@ func (o baseOptions) MemDev(sizeMB uint64) string {
 	return fmt.Sprintf("-object memory-backend-ram,id=mem,size=%dM -numa node,memdev=mem", sizeMB)
 }
 
+func (o baseOptions) MemFd(sizeMB uint64) string {
+	return fmt.Sprintf("-object memory-backend-memfd,id=mem,size=%dM,share=on,prealloc=on -numa node,memdev=mem", sizeMB)
+}
+
 func (o baseOptions) Boot(order string, enableMenu bool) string {
 	opt := "-boot order=" + order
 	if enableMenu {

pkg/hostman/options/options.go

@@ -182,6 +182,8 @@ type SHostOptions struct {
 
 	LocalBackupStoragePath string `help:"path for mounting backup nfs storage" default:"/opt/cloud/workspace/backupstorage"`
 	LocalBackupTempPath    string `help:"the local temporary directory for backup" default:"/opt/cloud/workspace/run/backups"`
+
+	BinaryMemcleanPath string `help:"execute binary memclean path" default:"/opt/yunion/bin/memclean"`
 }
 
 var (

pkg/mcclient/options/compute/servers.go

@@ -391,7 +391,8 @@ type ServerCreateOptions struct {
 type ServerCreateOptionalOptions struct {
 	ServerConfigs
 
-	MemSpec string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"`
+	MemSpec        string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"`
+	EnableMemclean bool   `help:"clean guest memory after guest exit" json:"enable_memclean"`
 
 	Keypair          string   `help:"SSH Keypair"`
 	Password         string   `help:"Default user password"`
@@ -524,6 +525,7 @@ func (opts *ServerCreateOptionalOptions) OptionalParams() (*computeapi.ServerCre
 		OsType:             opts.OsType,
 		GuestImageID:       opts.GuestImageID,
 		Secgroups:          opts.Secgroups,
+		EnableMemclean:     opts.EnableMemclean,
 	}
 
 	if len(opts.EncryptKey) > 0 {