diff --git a/pkg/hostman/host_health/etcd.go b/pkg/hostman/host_health/etcd.go index b50fe4130c..d885bc22bc 100644 --- a/pkg/hostman/host_health/etcd.go +++ b/pkg/hostman/host_health/etcd.go @@ -17,6 +17,7 @@ package host_health import ( "context" "fmt" + "time" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -87,7 +88,7 @@ func (c *SEtcdClient) OnKeepaliveFailure() { for timeout > 0 { timeout -= c.requestExpend if err := c.cli.RestartSession(); err != nil { - log.Errorf("restart session failed %s", err) + log.Errorf("etcd restart session failed %s", err) } else { break } @@ -99,6 +100,7 @@ func (c *SEtcdClient) OnKeepaliveFailure() { ); err != nil { log.Errorf("put host key failed %s", err) } else { + log.Infof("etcd client restart session success") return } } @@ -106,6 +108,27 @@ func (c *SEtcdClient) OnKeepaliveFailure() { if c.onUnhealthy != nil { c.onUnhealthy() } + go c.Reconnect() +} + +func (c *SEtcdClient) Reconnect() { + for { + if err := c.cli.RestartSession(); err != nil { + log.Errorf("restart session failed %s", err) + time.Sleep(1 * time.Second) + } else { + break + } + } + if err := c.cli.PutSession(context.Background(), + fmt.Sprintf("%s/%s", api.HOST_HEALTH_PREFIX, c.hostId), + api.HOST_HEALTH_STATUS_RUNNING, + ); err != nil { + log.Errorf("put host key failed %s", err) + go c.Reconnect() + } else { + return + } } func (c *SEtcdClient) Stop() error {