fix(region,host): live migrate quickly converge (#15419)

pkg/compute/tasks/guest_live_migrate_task.go

@@ -487,8 +487,10 @@ func (self *GuestLiveMigrateTask) OnStartDestComplete(ctx context.Context, guest
 }
 
 func (self *GuestLiveMigrateTask) OnStartDestCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
-	targetHostId, _ := self.Params.GetString("target_host_id")
-	guest.StartUndeployGuestTask(ctx, self.UserCred, "", targetHostId)
+	if !jsonutils.QueryBoolean(self.Params, "keep_dest_guest_on_failed", false) {
+		targetHostId, _ := self.Params.GetString("target_host_id")
+		guest.StartUndeployGuestTask(ctx, self.UserCred, "", targetHostId)
+	}
 	self.TaskFailed(ctx, guest, data)
 }
 
@@ -554,11 +556,12 @@ func (self *GuestLiveMigrateTask) OnLiveMigrateComplete(ctx context.Context, gue
 
 func (self *GuestLiveMigrateTask) OnResumeDestGuestCompleteFailed(ctx context.Context,
 	guest *models.SGuest, data jsonutils.JSONObject) {
-	targetHostId, _ := self.Params.GetString("target_host_id")
 
 	self.markFailed(ctx, guest, data)
-
-	guest.StartUndeployGuestTask(ctx, self.UserCred, "", targetHostId)
+	if !jsonutils.QueryBoolean(self.Params, "keep_dest_guest_on_failed", false) {
+		targetHostId, _ := self.Params.GetString("target_host_id")
+		guest.StartUndeployGuestTask(ctx, self.UserCred, "", targetHostId)
+	}
 
 	self.SetStage("OnResumeSourceGuestComplete", nil)
 	sourceHost := models.HostManager.FetchHostById(guest.HostId)

pkg/hostman/guestman/guesttasks.go

@@ -982,6 +982,7 @@ type SGuestLiveMigrateTask struct {
 	doTimeoutMigrate bool
 
 	expectDowntime int64
+	dirtySyncCount int64
 }
 
 func NewGuestLiveMigrateTask(
@@ -1012,6 +1013,7 @@ func (s *SGuestLiveMigrateTask) onSetAutoConverge(res string) {
 		s.migrateFailed(fmt.Sprintf("Migrate set capability auto-converge error: %s", res))
 		return
 	}
+
 	if version.LT(s.QemuVersion, "4.0.0") {
 		s.startMigrate()
 		return
@@ -1168,14 +1170,29 @@ func (s *SGuestLiveMigrateTask) onGetMigrateStatus(stats *monitor.MigrationInfo)
 		}
 		progress := (1 - float64(diskRemain+ramRemain)/float64(diskTotal+ramTotal)) * 100.0
 		hostutils.UpdateServerProgress(context.Background(), s.Id, progress, mbps)
-		if stats.Disk != nil && stats.Disk.Remaining > 0 {
-			// process disk copy
-			// do nothing, simply wait
-		} else if stats.RAM != nil && stats.RAM.Remaining > 0 {
-			if s.params.QuicklyFinish &&
-				stats.RAM.DirtySyncCount > 50 &&
-				stats.RAM.DirtyPagesRate > 0 &&
-				s.expectDowntime < *stats.ExpectedDowntime {
+
+		if s.params.QuicklyFinish && stats.RAM != nil && stats.RAM.Remaining > 0 {
+			if stats.CPUThrottlePercentage == nil {
+				// qemu do not enable cpu throttle, don't need set downtime
+				return
+			}
+
+			if *stats.CPUThrottlePercentage < 99 {
+				// TODO: configureable tolerate cpu throttle percentage
+				return
+			}
+
+			if stats.ExpectedDowntime != nil && *stats.ExpectedDowntime > s.expectDowntime {
+				if s.dirtySyncCount == 0 {
+					// record dirty sync count
+					s.dirtySyncCount = stats.RAM.DirtySyncCount
+					return
+				}
+				// run more than one round dirty ram sync after cpu throttle 99%
+				if stats.RAM.DirtySyncCount <= s.dirtySyncCount+1 {
+					return
+				}
+
 				cb := func(res string) {
 					if len(res) == 0 {
 						s.expectDowntime = *stats.ExpectedDowntime
@@ -1184,16 +1201,7 @@ func (s *SGuestLiveMigrateTask) onGetMigrateStatus(stats *monitor.MigrationInfo)
 						log.Errorf("failed set migrate downtime %s", res)
 					}
 				}
-				s.Monitor.MigrateSetDowntime(float32(*stats.ExpectedDowntime), cb)
-			} else if !s.doTimeoutMigrate &&
-				s.params.QuicklyFinish &&
-				stats.RAM.DirtySyncCount > 100 &&
-				stats.RAM.DirtyPagesRate > 0 {
-				log.Warningf("migrate timeout, force stop to finish migrate")
-				// timeout, start memory postcopy
-				// https://wiki.qemu.org/Features/PostCopyLiveMigration
-				s.Monitor.SimpleCommand("stop", s.onMigrateStartPostcopy)
-				s.doTimeoutMigrate = true
+				s.Monitor.MigrateSetDowntime(float32(*stats.ExpectedDowntime/1000.0), cb)
 			}
 		}
 	}

pkg/hostman/monitor/qmp.go

@@ -612,18 +612,7 @@ func (m *QmpMonitor) DeviceAdd(dev string, params map[string]string, callback St
 }
 
 func (m *QmpMonitor) MigrateSetDowntime(dtSec float32, callback StringCallback) {
-	var (
-		cb = func(res *Response) {
-			callback(m.actionResult(res))
-		}
-	)
-	cmd := &Command{
-		Execute: "migrate-set-downtime",
-		Args: map[string]interface{}{
-			"value": dtSec,
-		},
-	}
-	m.Query(cmd, cb)
+	m.HumanMonitorCommand(fmt.Sprintf("migrate_set_downtime %f", dtSec), callback)
 }
 
 func (m *QmpMonitor) MigrateSetCapability(capability, state string, callback StringCallback) {