From d9c042252ab7f8cb1512dd5ca07b8f369750dac4 Mon Sep 17 00:00:00 2001
From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com>
Date: Tue, 12 Jul 2022 10:38:17 +0800
Subject: [PATCH] fix(region,host): host health reconnect etcd add timeout
 (#14610)

- host: host health reconnect etcd add timeout
- region: request health status use 'GET' method
- region: only migrate running guests on host down
- etcd: keepalive use background context

Signed-off-by: wanyaoqi <wanyaoqi@yunion.cn>
---
 pkg/cloudcommon/etcd/etcd.go              |  2 +-
 pkg/compute/models/hosts.go               | 12 +++++++++-
 pkg/hostman/host_health/health_manager.go | 27 ++++++++++++-----------
 3 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/pkg/cloudcommon/etcd/etcd.go b/pkg/cloudcommon/etcd/etcd.go
index 965cca3e98..f974c10449 100644
--- a/pkg/cloudcommon/etcd/etcd.go
+++ b/pkg/cloudcommon/etcd/etcd.go
@@ -157,7 +157,7 @@ func (cli *SEtcdClient) startSession(ctx context.Context) error {
 	}
 	cli.leaseId = resp.ID
 
-	ch, err := cli.client.KeepAlive(ctx, cli.leaseId)
+	ch, err := cli.client.KeepAlive(context.Background(), cli.leaseId)
 	if err != nil {
 		return err
 	}
diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go
index 074544a7b5..4d41e39bd2 100644
--- a/pkg/compute/models/hosts.go
+++ b/pkg/compute/models/hosts.go
@@ -4001,6 +4001,11 @@ func (self *SHost) PerformOnline(ctx context.Context, userCred mcclient.TokenCre
 func (self *SHost) PerformAutoMigrateOnHostDown(
 	ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.HostAutoMigrateInput,
 ) (jsonutils.JSONObject, error) {
+	if input.AutoMigrateOnHostShutdown == "enable" &&
+		input.AutoMigrateOnHostDown != "enable" {
+		return nil, httperrors.NewBadRequestError("must enable auto_migrate_on_host_down at same time")
+	}
+
 	var meta = make(map[string]interface{})
 	if input.AutoMigrateOnHostShutdown == "enable" {
 		meta[api.HOSTMETA_AUTO_MIGRATE_ON_HOST_SHUTDOWN] = "enable"
@@ -5568,7 +5573,7 @@ func (host *SHost) RemoteHealthStatus(ctx context.Context) string {
 	var status = api.HOST_HEALTH_STATUS_UNKNOWN
 	userCred := auth.AdminCredential()
 	res, err := host.Request(
-		ctx, userCred, "POST", "/hosts/health-status",
+		ctx, userCred, "GET", "/hosts/health-status",
 		mcclient.GetTokenHeaders(userCred), nil,
 	)
 	if err != nil {
@@ -5705,6 +5710,11 @@ func (host *SHost) MigrateSharedStorageServers(ctx context.Context, userCred mcc
 	hostGuests := []*api.GuestBatchMigrateParams{}
 
 	for i := 0; i < len(guests); i++ {
+		if guests[i].isNotRunningStatus(guests[i].Status) {
+			// skip not running guests
+			continue
+		}
+
 		lockman.LockObject(ctx, &guests[i])
 		defer lockman.ReleaseObject(ctx, &guests[i])
 		_, err := guests[i].validateForBatchMigrate(ctx, true)
diff --git a/pkg/hostman/host_health/health_manager.go b/pkg/hostman/host_health/health_manager.go
index 0831737dfa..1d3904c80f 100644
--- a/pkg/hostman/host_health/health_manager.go
+++ b/pkg/hostman/host_health/health_manager.go
@@ -211,26 +211,27 @@ func (m *SHostHealthManager) Reconnect() {
 	if m.cli.SessionLiving() {
 		return
 	}
-	for {
-		if err := m.cli.RestartSession(); err != nil && !m.cli.SessionLiving() {
-			log.Errorf("restart session failed %s", err)
-			time.Sleep(1 * time.Second)
-		} else {
-			log.Infof("restart ression success")
-			break
-		}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer cancel()
+
+	if err := m.cli.RestartSessionWithContext(ctx); err != nil && !m.cli.SessionLiving() {
+		log.Errorf("restart session failed %s", err)
+		go m.Reconnect()
+		return
 	}
-	if err := m.cli.PutSession(context.Background(),
-		fmt.Sprintf("%s/%s", api.HOST_HEALTH_PREFIX, m.hostId),
+	log.Infof("restart ression success")
+
+	if err := m.cli.PutSession(
+		context.Background(), fmt.Sprintf("%s/%s", api.HOST_HEALTH_PREFIX, m.hostId),
 		api.HOST_HEALTH_STATUS_RUNNING,
 	); err != nil {
 		log.Errorf("put host key failed %s", err)
 		go m.Reconnect()
-	} else {
-		m.status = api.HOST_HEALTH_STATUS_RUNNING
-		log.Infof("put key %s/%s success", api.HOST_HEALTH_PREFIX, m.hostId)
 		return
 	}
+	log.Infof("put key %s/%s success", api.HOST_HEALTH_PREFIX, m.hostId)
+	m.status = api.HOST_HEALTH_STATUS_RUNNING
 }
 
 func (m *SHostHealthManager) SetOnHostDown(onHostDown string) {