opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-19 10:46:58 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13087 from swordqiu/hotfix/qj-migration-improvements-20211229

Browse Source 
fix: several improvements on qemu migration:
This commit is contained in:
Zexi Li
2021-12-30 10:04:01 +08:00
committed by GitHub
parent 56cf35fa4d 41ca44f02e
commit 0d00627bed
8 changed files with 86 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/compute/tasks/guest_live_migrate_task.go
+29 -1
View File
@@ -441,7 +441,31 @@ func (self *GuestLiveMigrateTask) OnResumeDestGuestCompleteFailed(ctx context.Co
guest *models.SGuest, data jsonutils.JSONObject) {
targetHostId, _ := self.Params.GetString("target_host_id")
self.markFailed(ctx, guest, data)
guest.StartUndeployGuestTask(ctx, self.UserCred, "", targetHostId)
self.SetStage("OnResumeSourceGuestComplete", nil)
sourceHost := models.HostManager.FetchHostById(guest.HostId)
headers := self.GetTaskRequestHeader()
body := jsonutils.NewDict()
url := fmt.Sprintf("%s/servers/%s/resume", sourceHost.ManagerUri, guest.Id)
_, _, err := httputils.JSONRequest(httputils.GetDefaultClient(),
ctx, "POST", url, headers, body, false)
if err != nil {
self.OnResumeSourceGuestCompleteFailed(ctx, guest, jsonutils.NewString(err.Error()))
}
}
func (self *GuestLiveMigrateTask) OnResumeSourceGuestCompleteFailed(ctx context.Context,
guest *models.SGuest, data jsonutils.JSONObject) {
db.OpsLog.LogEvent(guest, db.ACT_RESUME_FAIL, data, self.UserCred)
logclient.AddActionLogWithContext(ctx, guest, logclient.ACT_VM_RESUME, data, self.UserCred, false)
self.TaskFailed(ctx, guest, data)
}
func (self *GuestLiveMigrateTask) OnResumeSourceGuestComplete(ctx context.Context,
guest *models.SGuest, data jsonutils.JSONObject) {
self.TaskFailed(ctx, guest, data)
}
@@ -481,10 +505,14 @@ func (self *GuestMigrateTask) TaskComplete(ctx context.Context, guest *models.SG
}
func (self *GuestMigrateTask) TaskFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) {
self.markFailed(ctx, guest, reason)
self.SetStageFailed(ctx, reason)
}
func (self *GuestMigrateTask) markFailed(ctx context.Context, guest *models.SGuest, reason jsonutils.JSONObject) {
guest.SetStatus(self.UserCred, api.VM_MIGRATE_FAILED, reason.String())
db.OpsLog.LogEvent(guest, db.ACT_MIGRATE_FAIL, reason, self.UserCred)
logclient.AddActionLogWithContext(ctx, guest, logclient.ACT_MIGRATE, reason, self.UserCred, false)
self.SetStageFailed(ctx, reason)
notifyclient.NotifySystemErrorWithCtx(ctx, guest.Id, guest.Name, api.VM_MIGRATE_FAILED, reason.String())
notifyclient.EventNotify(ctx, self.GetUserCred(), notifyclient.SEventNotifyParam{
Obj: guest,
pkg/hostman/guestman/guestman.go
+10
View File
@@ -375,6 +375,9 @@ func (m *SGuestManager) PrepareDeploy(sid string) error {
func (m *SGuestManager) Monitor(sid, cmd string, callback func(string)) error {
if guest, ok := m.GetServer(sid); ok {
if guest.IsRunning() {
if guest.Monitor == nil {
return httperrors.NewBadRequestError("Monitor disconnected??")
}
guest.Monitor.HumanMonitorCommand(cmd, callback)
return nil
} else {
@@ -930,6 +933,13 @@ func (m *SGuestManager) DeleteSnapshot(ctx context.Context, params interface{})
func (m *SGuestManager) Resume(ctx context.Context, sid string, isLiveMigrate bool) (jsonutils.JSONObject, error) {
guest, _ := m.GetServer(sid)
if guest.IsStopping() || guest.IsStopped() {
return nil, httperrors.NewInvalidStatusError("resume stopped server???")
}
if guest.Monitor == nil {
guest.StartMonitor(ctx)
return nil, nil
}
resumeTask := NewGuestResumeTask(ctx, guest, !isLiveMigrate)
if isLiveMigrate {
guest.StartPresendArp()
pkg/hostman/guestman/guesttasks.go
+11 -1
View File
@@ -729,6 +729,7 @@ func (s *SGuestLiveMigrateTask) migrateComplete() {
s.migrateTask = nil
close(s.c)
s.Monitor.Disconnect()
s.Monitor = nil
hostutils.TaskComplete(s.ctx, nil)
}
@@ -754,6 +755,7 @@ func NewGuestResumeTask(ctx context.Context, s *SKVMGuestInstance, isTimeout boo
}
func (s *SGuestResumeTask) Start() {
log.Debugf("GuestResumeTask start")
s.startTime = time.Now()
s.confirmRunning()
}
@@ -761,10 +763,16 @@ func (s *SGuestResumeTask) Start() {
func (s *SGuestResumeTask) Stop() {
// TODO
// stop stream disk
s.taskFailed("qemu quit unexpectedly on resume")
}
func (s *SGuestResumeTask) confirmRunning() {
s.Monitor.QueryStatus(s.onConfirmRunning)
if s.Monitor != nil {
log.Debugf("GuestResumeTask QueryStatus")
s.Monitor.QueryStatus(s.onConfirmRunning)
} else {
s.taskFailed("qemu quit unexpectedly on resume confirmRunning")
}
}
func (s *SGuestResumeTask) onConfirmRunning(status string) {
@@ -776,6 +784,8 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) {
s.taskFailed(status)
} else if strings.Contains(status, "paused") {
s.Monitor.GetBlocks(s.onGetBlockInfo)
} else if status == "postmigrate" {
s.resumeGuest()
} else {
memMb, _ := s.Desc.Int("mem")
migSeconds := int(memMb) / options.HostOptions.MigrateExpectRate
pkg/hostman/guestman/qemu-arm.go
+5 -2
View File
@@ -22,7 +22,7 @@ import (
api "yunion.io/x/onecloud/pkg/apis/compute"
options "yunion.io/x/onecloud/pkg/hostman/options"
fileutils2 "yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/fileutils2"
"yunion.io/x/onecloud/pkg/util/qemutils"
)
@@ -153,6 +153,9 @@ function nic_mtu() {
}
cmd += fmt.Sprintf(" -cpu %s", cpuType)
if options.HostOptions.LogLevel == "debug" {
cmd += fmt.Sprintf(" -D %s -d all", s.getQemuLogPath())
}
cmd += s.getMonitorDesc("hmqmon", s.GetHmpMonitorPort(int(vncPort)), MODE_READLINE)
if options.HostOptions.EnableQmpMonitor {
@@ -253,7 +256,7 @@ function nic_mtu() {
cmd += s.extraOptions()
// cmd += s.getQgaDesc()
if fileutils2.Exists("/dev/random") {
if options.HostOptions.EnableVirtioRngDevice && fileutils2.Exists("/dev/random") {
cmd += " -object rng-random,filename=/dev/random,id=rng0"
cmd += " -device virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000"
}
pkg/hostman/guestman/qemu-kvm.go
+17 -6
View File
@@ -99,6 +99,10 @@ func (s *SKVMGuestInstance) getStateFilePathRootPrefix() string {
return path.Join(s.HomeDir(), STATE_FILE_PREFIX)
}
func (s *SKVMGuestInstance) getQemuLogPath() string {
return path.Join(s.HomeDir(), "qemu.log")
}
func (s *SKVMGuestInstance) IsLoaded() bool {
return s.Desc != nil
}
@@ -440,8 +444,11 @@ func (s *SKVMGuestInstance) StartMonitor(ctx context.Context) {
s.Id,
s.onMonitorDisConnect, // on monitor disconnect
func(err error) { s.onMonitorTimeout(ctx, err) }, // on monitor timeout
func() { s.onMonitorConnected(ctx) }, // on monitor connected
s.onReceiveQMPEvent, // on reveive qmp event
func() {
s.Monitor = mon
s.onMonitorConnected(ctx)
}, // on monitor connected
s.onReceiveQMPEvent, // on reveive qmp event
)
err := mon.Connect("127.0.0.1", s.GetQmpMonitorPort(-1))
if err != nil {
@@ -459,7 +466,6 @@ func (s *SKVMGuestInstance) StartMonitor(ctx context.Context) {
log.Errorf("Guest %s hmp monitor connect failed %s, something wrong", s.GetName(), err)
}
}
s.Monitor = mon
})
} else if monitorPath := s.GetMonitorPath(); len(monitorPath) > 0 {
s.StartMonitorWithImportGuestSocketFile(ctx, monitorPath)
@@ -689,10 +695,12 @@ func (s *SKVMGuestInstance) BlockJobsCount() int {
}
func (s *SKVMGuestInstance) CleanStartupTask() {
log.Infof("Clean startup task ...")
if s.startupTask != nil {
log.Infof("Clean startup task ... stop task ...")
s.startupTask.Stop()
s.startupTask = nil
} else {
log.Infof("Clean startup task ... no task")
}
}
@@ -963,10 +971,13 @@ func (s *SKVMGuestInstance) delFlatFiles(ctx context.Context) error {
func (s *SKVMGuestInstance) Delete(ctx context.Context, migrated bool) error {
if err := s.delTmpDisks(ctx, migrated); err != nil {
return err
return errors.Wrap(err, "delTmpDisks")
}
if err := s.delFlatFiles(ctx); err != nil {
return err
return errors.Wrap(err, "delFlatFiles")
}
if fileutils2.Exists(s.getQemuLogPath()) {
procutils.NewRemoteCommandAsFarAsPossible("mv", s.getQemuLogPath(), fmt.Sprintf("/tmp/%s-qemu.log", s.GetId())).Run()
}
output, err := procutils.NewCommand("rm", "-rf", s.HomeDir()).Output()
if err != nil {
pkg/hostman/guestman/qemu-kvmhelper.go
+11 -1
View File
@@ -543,6 +543,10 @@ function nic_mtu() {
cmd += fmt.Sprintf(" -cpu %s", cpuType)
if options.HostOptions.LogLevel == "debug" {
cmd += fmt.Sprintf(" -D %s -d all", s.getQemuLogPath())
}
// TODO hmp - -
cmd += s.getMonitorDesc("hmqmon", s.GetHmpMonitorPort(int(vncPort)), MODE_READLINE)
if options.HostOptions.EnableQmpMonitor {
@@ -695,7 +699,13 @@ function nic_mtu() {
cmd += s.extraOptions()
cmd += s.getQgaDesc()
if fileutils2.Exists("/dev/random") {
/*
QIU Jian
virtio-rng device may cause live migration failure
qemu-system-x86_64: Unknown savevm section or instance '0000:00:05.0/virtio-rng' 0
qemu-system-x86_64: load of migration failed: Invalid argument
*/
if options.HostOptions.EnableVirtioRngDevice && fileutils2.Exists("/dev/random") {
cmd += " -object rng-random,filename=/dev/random,id=rng0"
cmd += " -device virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000"
}
pkg/hostman/options/options.go
+2
View File
@@ -158,6 +158,8 @@ type SHostOptions struct {
EthtoolEnableGso bool `help:"use ethtool to turn on or off GSO(generic segment offloading)" default:"false" json:"ethtool_enable_gso"`
EnableVmUuid bool `help:"enable vm UUID" default:"true" json:"enable_vm_uuid"`
EnableVirtioRngDevice bool `help:"enable qemu virtio-rng device" default:"false"`
}
var (
pkg/util/logclient/consts.go
+1
View File
@@ -60,6 +60,7 @@ const (
ACT_VM_START = "vm_start"
ACT_VM_STOP = "vm_stop"
ACT_VM_RESTART = "vm_restart"
ACT_VM_RESUME = "vm_resume"
ACT_VM_SYNC_CONF = "vm_sync_conf"
ACT_VM_SYNC_STATUS = "vm_sync_status"
ACT_VM_UNBIND_KEYPAIR = "vm_unbind_keypair"