diff --git a/cmd/climc/shell/compute/servers.go b/cmd/climc/shell/compute/servers.go index b5664c80a3..8e0bc6827c 100644 --- a/cmd/climc/shell/compute/servers.go +++ b/cmd/climc/shell/compute/servers.go @@ -106,6 +106,7 @@ func init() { cmd.Perform("calculate-record-checksum", &options.ServerIdOptions{}) cmd.Perform("set-class-metadata", &baseoptions.ResourceMetadataOptions{}) cmd.Perform("monitor", &options.ServerMonitorOptions{}) + cmd.BatchPerform("enable-memclean", new(options.ServerIdsOptions)) cmd.Get("vnc", new(options.ServerIdOptions)) cmd.Get("desc", new(options.ServerIdOptions)) diff --git a/cmd/memclean/main.go b/cmd/memclean/main.go new file mode 100644 index 0000000000..72408c4acd --- /dev/null +++ b/cmd/memclean/main.go @@ -0,0 +1,187 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "syscall" + "time" + + "github.com/sevlyar/go-daemon" + "golang.org/x/sys/unix" + + "yunion.io/x/log" + "yunion.io/x/log/hooks" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/signalutils" + "yunion.io/x/pkg/utils" + "yunion.io/x/structarg" + + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +const ( + MEM_BACKEND_FD = "/memfd:memory-backend-memfd (deleted)" + MMAP_SIZE = 2 * 1024 * 1024 * 1024 +) + +type Options struct { + Pid int `help:"qemu process pid" required:"true"` + MemSize int64 `help:"backend memory size" required:"true"` + Foreground bool `help:"run in foreground"` + LogDir string `help:"log dir" required:"true"` +} + +var opt = &Options{} + +func main() { + procDir := fmt.Sprintf("/proc/%d", opt.Pid) + if !fileutils2.IsDir(procDir) { + log.Fatalf("Process %d not found", opt.Pid) + } + + fdDir := fmt.Sprintf("%s/fd", procDir) + memBackendFd, err := findMemBackendFd(fdDir) + if err != nil { + log.Fatalf("findMemBackendFd: %s", err) + } + log.Infof("found mem backend fd: %s", memBackendFd) + + f, err := os.OpenFile(memBackendFd, os.O_RDWR, 0644) + if err != nil { + log.Fatalf("failed open memfd: %s", err) + } + defer f.Close() + + if !opt.Foreground { + cntxt := &daemon.Context{ + WorkDir: "./", + Umask: 027, + } + + d, err := cntxt.Reborn() + if err != nil { + log.Fatalf("Unable to run in background: %s", err) + } + if d != nil { + return + } + defer cntxt.Release() + } + + log.Infof("start watch proc exit") + err = procutils.NewCommand("tail", fmt.Sprintf("--pid=%d", opt.Pid), "-f", "/dev/null").Run() + if err != nil { + log.Fatalf("failed watch process: %s", err) + } + + log.Infof("watch proc exited, go to clean memory") + var ( + start = time.Now() + size int64 = 0 + ) + for size < opt.MemSize { + mmapSize := MMAP_SIZE + if opt.MemSize-size < MMAP_SIZE { + mmapSize = int(opt.MemSize - size) + } + + b, err := syscall.Mmap( + int(f.Fd()), size, mmapSize, + syscall.PROT_WRITE|syscall.PROT_READ, + syscall.MAP_SHARED, + ) + if err != nil { + log.Fatalf("failed mmap mem backend fd: %s", err) + } + log.Infof("mmap memory offset %d, size %d", size, len(b)) + size += int64(len(b)) + + // memsetRepeat(b, 0) + for i := 0; i < len(b); i++ { + b[i] = 0 + } + unix.Msync(b, unix.MS_SYNC) + syscall.Munmap(b) + } + + log.Infof( + "mem clean for process %d success, mem clean took %s", + opt.Pid, time.Since(start), + ) +} + +func findMemBackendFd(fdDir string) (string, error) { + files, err := ioutil.ReadDir(fdDir) + if err != nil { + return "", errors.Wrapf(err, "read dir %s", fdDir) + } + + for _, f := range files { + p1 := path.Join(fdDir, f.Name()) + p2, e := os.Readlink(p1) + if e != nil { + log.Errorf("os.readlink %s: %s", p1, e) + continue + } + log.Infof("os.readlink path %s", p2) + if p2 == MEM_BACKEND_FD { + return p1, nil + } + } + + return "", errors.Errorf("no mem backend fd found") +} + +func memsetRepeat(a []byte, v byte) { + if len(a) == 0 { + return + } + a[0] = v + for bp := 1; bp < len(a); bp *= 2 { + copy(a[bp:], a[:bp]) + } +} + +func init() { + signalutils.RegisterSignal(func() { + utils.DumpAllGoroutineStack(log.Logger().Out) + }, syscall.SIGUSR1) + signalutils.StartTrap() + + parser, err := structarg.NewArgumentParser(opt, "", "", "") + if err != nil { + log.Fatalf("Error define argument parser: %v", err) + } + err = parser.ParseArgs2(os.Args[1:], true, true) + if err != nil { + log.Fatalf("Failed parse args %s", err) + } + + logFileHook := hooks.LogFileRotateHook{ + LogFileHook: hooks.LogFileHook{ + FileDir: opt.LogDir, + FileName: "memclean.log", + }, + RotateNum: 10, + RotateSize: 100 * 1024 * 1024, + } + logFileHook.Init() + log.Logger().AddHook(&logFileHook) +} diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 3a2b3aa70d..cdf96ebbe3 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -365,7 +365,8 @@ type ServerCreateInput struct { apis.EncryptedResourceCreateInput // 虚拟机内存大小,单位Mb,若未指定instance_type,此参数为必传项 - VmemSize int `json:"vmem_size"` + VmemSize int `json:"vmem_size"` + EnableMemclean bool `json:"enable_memclean"` // 虚拟机Cpu大小,若未指定instance_type,此参数为必传项 // default: 1 diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index c6010f1fd0..af101910e8 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -336,6 +336,7 @@ const ( VM_METADATA_OS_NAME = "os_name" VM_METADATA_OS_VERSION = "os_version" VM_METADATA_CGROUP_CPUSET = "cgroup_cpuset" + VM_METADATA_ENABLE_MEMCLEAN = "enable_memclean" ) func Hypervisors2HostTypes(hypervisors []string) []string { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index bd8c603aac..d27636afab 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -5600,3 +5600,7 @@ func (self *SGuest) PerformCalculateRecordChecksum(ctx context.Context, userCred "checksum": checksum, }), nil } + +func (self *SGuest) PerformEnableMemclean(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return nil, self.SetMetadata(ctx, api.VM_METADATA_ENABLE_MEMCLEAN, "true", userCred) +} diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d15e95b25e..6edfd1f233 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2021,6 +2021,10 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred if osProfileJson != nil { guest.setOSProfile(ctx, userCred, osProfileJson) } + + if jsonutils.QueryBoolean(data, api.VM_METADATA_ENABLE_MEMCLEAN, false) { + guest.SetMetadata(ctx, api.VM_METADATA_ENABLE_MEMCLEAN, "true", userCred) + } if jsonutils.QueryBoolean(data, imageapi.IMAGE_DISABLE_USB_KBD, false) { guest.SetMetadata(ctx, imageapi.IMAGE_DISABLE_USB_KBD, "true", userCred) } diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 0dca071d7f..c764b396ab 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -933,7 +933,11 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) { /* ref: qemu/src/qapi/run-state.json * prelaunch: QEMU was started with -S and guest has not started. * we need resume guest at state prelaunch */ - s.onGuestPrelaunch() + if err := s.onGuestPrelaunch(); err != nil { + s.ForceStop() + s.taskFailed(err.Error()) + return + } s.resumeGuest() } else if status == "running" || status == "paused (suspended)" { s.onStartRunning() @@ -941,7 +945,11 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) { // handle error first, results may be 'paused (internal-error)' s.taskFailed(status) } else if strings.Contains(status, "paused") { - s.onGuestPrelaunch() + if err := s.onGuestPrelaunch(); err != nil { + s.ForceStop() + s.taskFailed(err.Error()) + return + } s.Monitor.GetBlocks(s.onGetBlockInfo) } else if status == "postmigrate" { s.resumeGuest() @@ -954,6 +962,7 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) { log.Infof("start guest timeout seconds: %d", migSeconds) if s.isTimeout && time.Now().Sub(s.startTime) >= time.Second*time.Duration(migSeconds) { s.taskFailed("Timeout") + return } else { time.Sleep(time.Second * 3) s.confirmRunning() diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 04e7902b13..bc43a416d4 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -1855,14 +1855,20 @@ func (s *SKVMGuestInstance) doBlockIoThrottle() { } } -func (s *SKVMGuestInstance) onGuestPrelaunch() { +func (s *SKVMGuestInstance) onGuestPrelaunch() error { if options.HostOptions.SetVncPassword { s.SetVncPassword() } + if s.isMemcleanEnabled() { + if err := s.startMemCleaner(); err != nil { + return err + } + } s.OnResumeSyncMetadataInfo() s.SetCgroup() s.optimizeOom() s.doBlockIoThrottle() + return nil } func (s *SKVMGuestInstance) CleanImportMetadata() *jsonutils.JSONDict { diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index f0c1022a2d..17ed24091a 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -18,6 +18,7 @@ import ( "fmt" "net" "path" + "strconv" "strings" "time" @@ -112,6 +113,11 @@ func (s *SKVMGuestInstance) isWindows10() bool { return false } +func (s *SKVMGuestInstance) isMemcleanEnabled() bool { + val, _ := s.Desc.GetString("metadata", "enable_memclean") + return val == "true" +} + func (s *SKVMGuestInstance) getMachine() string { machine, err := s.Desc.GetString("machine") if err != nil { @@ -242,6 +248,7 @@ func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (strin OVNIntegrationBridge: options.HostOptions.OvnIntegrationBridge, HomeDir: s.HomeDir(), HugepagesEnabled: s.manager.host.IsHugepagesEnabled(), + EnableMemfd: s.isMemcleanEnabled(), PidFilePath: s.GetPidFilePath(), BIOS: s.getBios(), } @@ -769,3 +776,18 @@ func (s *SKVMGuestInstance) WriteMigrateCerts(certs map[string]string) error { } return nil } + +func (s *SKVMGuestInstance) startMemCleaner() error { + mem, _ := s.Desc.Int("mem") + err := procutils.NewRemoteCommandAsFarAsPossible( + options.HostOptions.BinaryMemcleanPath, + "--pid", strconv.Itoa(s.GetPid()), + "--mem-size", strconv.FormatInt(mem*1024*1024, 10), + "--log-dir", s.HomeDir(), + ).Run() + if err != nil { + log.Errorf("failed start memcleaner: %s", err) + return errors.Wrap(err, "start memclean") + } + return nil +} diff --git a/pkg/hostman/guestman/qemu/generate.go b/pkg/hostman/guestman/qemu/generate.go index 7d3d580ea8..cd5f0ddba9 100644 --- a/pkg/hostman/guestman/qemu/generate.go +++ b/pkg/hostman/guestman/qemu/generate.go @@ -46,6 +46,7 @@ type GenerateStartOptionsInput struct { Name string OsName string HugepagesEnabled bool + EnableMemfd bool IsQ35 bool BootOrder string CdromPath string @@ -132,6 +133,8 @@ func GenerateStartOptions( var memDev string if input.HugepagesEnabled { memDev = drvOpt.MemPath(input.Mem, fmt.Sprintf("/dev/hugepages/%s", input.UUID)) + } else if input.EnableMemfd { + memDev = drvOpt.MemFd(input.Mem) } else { memDev = drvOpt.MemDev(input.Mem) } diff --git a/pkg/hostman/guestman/qemu/qemu.go b/pkg/hostman/guestman/qemu/qemu.go index 8d45fd0008..4a15133ab8 100644 --- a/pkg/hostman/guestman/qemu/qemu.go +++ b/pkg/hostman/guestman/qemu/qemu.go @@ -96,6 +96,7 @@ type QemuOptions interface { Memory(sizeMB uint64) string MemPath(sizeMB uint64, p string) string MemDev(sizeMB uint64) string + MemFd(sizeMB uint64) string Boot(order string, enableMenu bool) string BIOS(file string) string Device(devStr string) string @@ -253,6 +254,10 @@ func (o baseOptions) MemDev(sizeMB uint64) string { return fmt.Sprintf("-object memory-backend-ram,id=mem,size=%dM -numa node,memdev=mem", sizeMB) } +func (o baseOptions) MemFd(sizeMB uint64) string { + return fmt.Sprintf("-object memory-backend-memfd,id=mem,size=%dM,share=on,prealloc=on -numa node,memdev=mem", sizeMB) +} + func (o baseOptions) Boot(order string, enableMenu bool) string { opt := "-boot order=" + order if enableMenu { diff --git a/pkg/hostman/options/options.go b/pkg/hostman/options/options.go index dc195cadd3..a9ffcb6346 100644 --- a/pkg/hostman/options/options.go +++ b/pkg/hostman/options/options.go @@ -182,6 +182,8 @@ type SHostOptions struct { LocalBackupStoragePath string `help:"path for mounting backup nfs storage" default:"/opt/cloud/workspace/backupstorage"` LocalBackupTempPath string `help:"the local temporary directory for backup" default:"/opt/cloud/workspace/run/backups"` + + BinaryMemcleanPath string `help:"execute binary memclean path" default:"/opt/yunion/bin/memclean"` } var ( diff --git a/pkg/mcclient/options/compute/servers.go b/pkg/mcclient/options/compute/servers.go index d15d42584d..785584bb64 100644 --- a/pkg/mcclient/options/compute/servers.go +++ b/pkg/mcclient/options/compute/servers.go @@ -391,7 +391,8 @@ type ServerCreateOptions struct { type ServerCreateOptionalOptions struct { ServerConfigs - MemSpec string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"` + MemSpec string `help:"Memory size Or Instance Type" metavar:"MEMSPEC" json:"-"` + EnableMemclean bool `help:"clean guest memory after guest exit" json:"enable_memclean"` Keypair string `help:"SSH Keypair"` Password string `help:"Default user password"` @@ -524,6 +525,7 @@ func (opts *ServerCreateOptionalOptions) OptionalParams() (*computeapi.ServerCre OsType: opts.OsType, GuestImageID: opts.GuestImageID, Secgroups: opts.Secgroups, + EnableMemclean: opts.EnableMemclean, } if len(opts.EncryptKey) > 0 {