From 1365007ed172b42b2ec1a848d934f76f2a0b8fe5 Mon Sep 17 00:00:00 2001 From: gouqi11 <66834753+gouqi11@users.noreply.github.com> Date: Mon, 28 Nov 2022 23:57:43 +0800 Subject: [PATCH] fix(region): add guest power states (#15454) --- go.mod | 2 +- go.sum | 4 +- pkg/compute/models/guests.go | 65 +++++++++++++++++++ vendor/modules.txt | 2 +- .../x/cloudmux/pkg/cloudprovider/resources.go | 1 + .../hcso/client/modules/manager_base.go | 40 ++++++++++++ .../pkg/multicloud/remotefile/instance.go | 7 ++ 7 files changed, 117 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index a788b11900..d69333c84d 100644 --- a/go.mod +++ b/go.mod @@ -84,7 +84,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221124020449-e64c2aa36146 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221128022002-9a9d6ff89e79 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd yunion.io/x/log v1.0.0 diff --git a/go.sum b/go.sum index b236ac07eb..09fc5e3ee7 100644 --- a/go.sum +++ b/go.sum @@ -1176,8 +1176,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221124020449-e64c2aa36146 h1:24Tb9M9wrCepLrkeBh9Gh+1IdZbe0zhyY6UUTifMXVg= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221124020449-e64c2aa36146/go.mod h1:drc/WDD65r6wI1tSejk19xcl9QwmzZlgJGXN7a9o2KQ= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221128022002-9a9d6ff89e79 h1:PaZdEBQxu/Ag5twEecpTvQGZ6YR+OBHhf6/tZluTVoY= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221128022002-9a9d6ff89e79/go.mod h1:drc/WDD65r6wI1tSejk19xcl9QwmzZlgJGXN7a9o2KQ= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 35dd8dcf57..41dd50ee01 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -2726,6 +2726,8 @@ func (self *SGuest) syncRemoveCloudVM(ctx context.Context, userCred mcclient.Tok _, err = db.Update(self, func() error { self.HostId = host.GetId() self.Status = iVM.GetStatus() + self.PowerStates = iVM.GetPowerStates() + self.inferPowerStates() return nil }) return err @@ -2830,6 +2832,8 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token } if !self.IsFailureStatus() && syncStatus { self.Status = extVM.GetStatus() + self.PowerStates = extVM.GetPowerStates() + self.inferPowerStates() } self.VcpuCount = extVM.GetVcpuCount() @@ -2928,6 +2932,8 @@ func (manager *SGuestManager) newCloudVM(ctx context.Context, userCred mcclient. guest.SetModelManager(manager, &guest) guest.Status = extVM.GetStatus() + guest.PowerStates = extVM.GetPowerStates() + guest.inferPowerStates() guest.ExternalId = extVM.GetGlobalId() guest.VcpuCount = extVM.GetVcpuCount() guest.BootOrder = extVM.GetBootOrder() @@ -6325,3 +6331,62 @@ func (self *SGuest) GetAddress() (string, error) { } return "", errors.Wrapf(cloudprovider.ErrNotFound, "guest %s address", self.Name) } + +func (guest *SGuest) inferPowerStates() { + if len(guest.PowerStates) == 0 { + switch guest.Status { + case api.VM_READY: + guest.PowerStates = api.VM_POWER_STATES_OFF + case api.VM_UNKNOWN: + guest.PowerStates = api.VM_POWER_STATES_UNKNOWN + case api.VM_INIT: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_SCHEDULE: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_SCHEDULE_FAILED: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_CREATE_NETWORK: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_NETWORK_FAILED: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_DEVICE_FAILED: + guest.PowerStates = api.VM_POWER_STATES_UNKNOWN + case api.VM_CREATE_FAILED: + guest.PowerStates = api.VM_POWER_STATES_OFF + case api.VM_CREATE_DISK: + guest.PowerStates = api.VM_POWER_STATES_OFF + case api.VM_DISK_FAILED: + guest.PowerStates = api.VM_POWER_STATES_OFF + case api.VM_IMAGE_CACHING: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_START_DEPLOY: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_DEPLOYING: + guest.PowerStates = api.VM_POWER_STATES_OFF + case api.VM_START_START: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_STARTING: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_START_FAILED: + guest.PowerStates = api.VM_POWER_STATES_OFF + case api.VM_RUNNING: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_START_STOP: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_STOPPING: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_STOP_FAILED: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_RENEWING: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_RENEW_FAILED: + guest.PowerStates = api.VM_POWER_STATES_ON + case api.VM_ATTACH_DISK: + guest.PowerStates = api.VM_POWER_STATES_UNKNOWN + case api.VM_DETACH_DISK: + guest.PowerStates = api.VM_POWER_STATES_UNKNOWN + default: + guest.PowerStates = api.VM_POWER_STATES_UNKNOWN + } + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6229f4c3d8..df2bbd9224 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1439,7 +1439,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221124020449-e64c2aa36146 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20221128022002-9a9d6ff89e79 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go index 71f252e152..d775d7a286 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/resources.go @@ -400,6 +400,7 @@ type ICloudVM interface { SaveImage(opts *SaveImageOptions) (ICloudImage, error) AllocatePublicIpAddress() (string, error) + GetPowerStates() string } type ICloudNic interface { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/modules/manager_base.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/modules/manager_base.go index d956613545..2834a0758c 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/modules/manager_base.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/client/modules/manager_base.go @@ -17,7 +17,9 @@ package modules import ( "context" "fmt" + "net" "net/http" + "net/url" "strconv" "strings" "time" @@ -187,6 +189,39 @@ func (ce *HuaweiClientError) ParseErrorFromJsonResponse(statusCode int, body jso return ce } +func isHTTPReqErrorRetryable(err error) bool { + if err == nil { + return false + } + ce := errors.Cause(err) + switch e := ce.(type) { + case *url.Error: + switch e.Err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: + return true + } + if strings.Contains(err.Error(), "Connection closed by foreign host") { + return true + } else if strings.Contains(err.Error(), "net/http: TLS handshake timeout") { + // If error is - tlsHandshakeTimeoutError, retry. + return true + } else if strings.Contains(err.Error(), "i/o timeout") { + // If error is - tcp timeoutError, retry. + return true + } else if strings.Contains(err.Error(), "connection timed out") { + // If err is a net.Dial timeout, retry. + return true + } else if strings.Contains(err.Error(), "net/http: HTTP/1.x transport connection broken") { + // If error is transport connection broken, retry. + return true + } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { + // Retry errors due to server not sending the response before timeout + return true + } + } + return false +} + func (self *SBaseManager) jsonRequest(request requests.IRequest) (http.Header, jsonutils.JSONObject, error) { ThrottlingLock.CheckingLock() ctx := context.Background() @@ -224,6 +259,11 @@ func (self *SBaseManager) jsonRequest(request requests.IRequest) (http.Header, j if e == nil { return h, b, nil } + if isHTTPReqErrorRetryable(e) && retry > 0 { + retry -= 1 + time.Sleep(3 * time.Second * time.Duration(MAX_RETRY-retry)) + continue + } log.Errorf("[%s] %s body: %v error: %v", req.GetHttpMethod(), req.GetUrl(), jsonBody, e) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/instance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/instance.go index 432ed5493a..ca8a62a67f 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/instance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/remotefile/instance.go @@ -262,3 +262,10 @@ func (self *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) { } return ret, nil } + +func (self *SInstance) GetPowerStates() string { + if self.Status == api.VM_RUNNING { + return api.VM_POWER_STATES_ON + } + return api.VM_POWER_STATES_OFF +}