opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-01 15:07:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(region,host): host health reconnect etcd add timeout (#14610)

Browse Source 
- host: host health reconnect etcd add timeout
- region: request health status use 'GET' method
- region: only migrate running guests on host down
- etcd: keepalive use background context

Signed-off-by: wanyaoqi <wanyaoqi@yunion.cn>
This commit is contained in:
wanyaoqi
2022-07-12 10:38:17 +08:00
committed by GitHub
parent 9fbbe05c6a
commit d9c042252a
3 changed files with 26 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cloudcommon/etcd/etcd.go
+1 -1
View File
@@ -157,7 +157,7 @@ func (cli *SEtcdClient) startSession(ctx context.Context) error {
}
cli.leaseId = resp.ID
ch, err := cli.client.KeepAlive(ctx, cli.leaseId)
ch, err := cli.client.KeepAlive(context.Background(), cli.leaseId)
if err != nil {
return err
}
pkg/compute/models/hosts.go
+11 -1
View File
@@ -4001,6 +4001,11 @@ func (self *SHost) PerformOnline(ctx context.Context, userCred mcclient.TokenCre
func (self *SHost) PerformAutoMigrateOnHostDown(
ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.HostAutoMigrateInput,
) (jsonutils.JSONObject, error) {
if input.AutoMigrateOnHostShutdown == "enable" &&
input.AutoMigrateOnHostDown != "enable" {
return nil, httperrors.NewBadRequestError("must enable auto_migrate_on_host_down at same time")
}
var meta = make(map[string]interface{})
if input.AutoMigrateOnHostShutdown == "enable" {
meta[api.HOSTMETA_AUTO_MIGRATE_ON_HOST_SHUTDOWN] = "enable"
@@ -5568,7 +5573,7 @@ func (host *SHost) RemoteHealthStatus(ctx context.Context) string {
var status = api.HOST_HEALTH_STATUS_UNKNOWN
userCred := auth.AdminCredential()
res, err := host.Request(
ctx, userCred, "POST", "/hosts/health-status",
ctx, userCred, "GET", "/hosts/health-status",
mcclient.GetTokenHeaders(userCred), nil,
)
if err != nil {
@@ -5705,6 +5710,11 @@ func (host *SHost) MigrateSharedStorageServers(ctx context.Context, userCred mcc
hostGuests := []*api.GuestBatchMigrateParams{}
for i := 0; i < len(guests); i++ {
if guests[i].isNotRunningStatus(guests[i].Status) {
// skip not running guests
continue
}
lockman.LockObject(ctx, &guests[i])
defer lockman.ReleaseObject(ctx, &guests[i])
_, err := guests[i].validateForBatchMigrate(ctx, true)
pkg/hostman/host_health/health_manager.go
+14 -13
View File
@@ -211,26 +211,27 @@ func (m *SHostHealthManager) Reconnect() {
if m.cli.SessionLiving() {
return
}
for {
if err := m.cli.RestartSession(); err != nil && !m.cli.SessionLiving() {
log.Errorf("restart session failed %s", err)
time.Sleep(1 * time.Second)
} else {
log.Infof("restart ression success")
break
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
if err := m.cli.RestartSessionWithContext(ctx); err != nil && !m.cli.SessionLiving() {
log.Errorf("restart session failed %s", err)
go m.Reconnect()
return
}
if err := m.cli.PutSession(context.Background(),
fmt.Sprintf("%s/%s", api.HOST_HEALTH_PREFIX, m.hostId),
log.Infof("restart ression success")
if err := m.cli.PutSession(
context.Background(), fmt.Sprintf("%s/%s", api.HOST_HEALTH_PREFIX, m.hostId),
api.HOST_HEALTH_STATUS_RUNNING,
); err != nil {
log.Errorf("put host key failed %s", err)
go m.Reconnect()
} else {
m.status = api.HOST_HEALTH_STATUS_RUNNING
log.Infof("put key %s/%s success", api.HOST_HEALTH_PREFIX, m.hostId)
return
}
log.Infof("put key %s/%s success", api.HOST_HEALTH_PREFIX, m.hostId)
m.status = api.HOST_HEALTH_STATUS_RUNNING
}
func (m *SHostHealthManager) SetOnHostDown(onHostDown string) {