From e11af2d012db5247fd2a40e9f3650be9ccbeadc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Tue, 19 Dec 2023 20:26:13 +0800 Subject: [PATCH] fix(region): ignore empty certificate error for ssl sync (#19054) --- go.mod | 2 +- go.sum | 4 +- vendor/modules.txt | 2 +- .../pkg/multicloud/huawei/client/client.go | 5 -- .../huawei/client/modules/mod_keypairs.go | 37 --------- .../huawei/client/modules/mod_quotas.go | 37 --------- .../cloudmux/pkg/multicloud/huawei/huawei.go | 22 +++++ .../cloudmux/pkg/multicloud/huawei/keypair.go | 61 ++++++++------ .../x/cloudmux/pkg/multicloud/huawei/quota.go | 80 ------------------- .../pkg/multicloud/huawei/sslcertificate.go | 76 ++++++------------ 10 files changed, 87 insertions(+), 239 deletions(-) delete mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go delete mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go delete mode 100644 vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go diff --git a/go.mod b/go.mod index 6ec2f734c6..a31c74be8b 100644 --- a/go.mod +++ b/go.mod @@ -88,7 +88,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072118-36b6dd2a146b + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063429-4e0c70548d7b yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19 yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361 diff --git a/go.sum b/go.sum index a38e994b32..aa7c91a954 100644 --- a/go.sum +++ b/go.sum @@ -1201,8 +1201,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072118-36b6dd2a146b h1:qeI8NpnqmOea7UFYTgy+hiOvyfykOKzVoIBfeV1nM6o= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072118-36b6dd2a146b/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063429-4e0c70548d7b h1:NH2QqBfEwpjxbS5Yvtnh/hTyGzVkMafDyiwJ0QX1ry0= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063429-4e0c70548d7b/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/vendor/modules.txt b/vendor/modules.txt index 1c254438ee..6b45b0593a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1465,7 +1465,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231218072118-36b6dd2a146b +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231219063429-4e0c70548d7b ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go index 60963fd57f..d6dc91b9c2 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/client.go @@ -37,7 +37,6 @@ type Client struct { OpenStackImages *modules.SImageManager Interface *modules.SInterfaceManager Jobs *modules.SJobManager - Keypairs *modules.SKeypairManager Orders *modules.SOrderManager Port *modules.SPortManager Projects *modules.SProjectManager @@ -62,7 +61,6 @@ type Client struct { DBInstanceJob *modules.SDBInstanceJobManager Traces *modules.STraceManager CloudEye *modules.SCloudEyeManager - Quotas *modules.SQuotaManager EnterpriseProjects *modules.SEnterpriseProjectManager Roles *modules.SRoleManager Groups *modules.SGroupManager @@ -123,7 +121,6 @@ func (self *Client) SetHttpClient(httpClient *http.Client) { self.DcsAvailableZone.SetHttpClient(httpClient) self.Disks.SetHttpClient(httpClient) self.Domains.SetHttpClient(httpClient) - self.Keypairs.SetHttpClient(httpClient) self.Orders.SetHttpClient(httpClient) self.Subnets.SetHttpClient(httpClient) self.Users.SetHttpClient(httpClient) @@ -193,7 +190,6 @@ func (self *Client) initManagers() { self.DcsAvailableZone = modules.NewDcsAvailableZoneManager(self.cfg) self.Disks = modules.NewDiskManager(self.cfg) self.Domains = modules.NewDomainManager(self.cfg) - self.Keypairs = modules.NewKeypairManager(self.cfg) self.Orders = modules.NewOrderManager(self.cfg) self.Subnets = modules.NewSubnetManager(self.cfg) self.Users = modules.NewUserManager(self.cfg) @@ -213,7 +209,6 @@ func (self *Client) initManagers() { self.DBInstanceJob = modules.NewDBInstanceJobManager(self.cfg) self.Traces = modules.NewTraceManager(self.cfg) self.CloudEye = modules.NewCloudEyeManager(self.cfg) - self.Quotas = modules.NewQuotaManager(self.cfg) self.EnterpriseProjects = modules.NewEnterpriseProjectManager(self.cfg) self.Roles = modules.NewRoleManager(self.cfg) self.Groups = modules.NewGroupManager(self.cfg) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go deleted file mode 100644 index b25b4e33c8..0000000000 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_keypairs.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package modules - -import ( - "yunion.io/x/cloudmux/pkg/multicloud/huawei/client/manager" -) - -type SKeypairManager struct { - SResourceManager -} - -func NewKeypairManager(cfg manager.IManagerConfig) *SKeypairManager { - return &SKeypairManager{SResourceManager: SResourceManager{ - SBaseManager: NewBaseManager(cfg), - ServiceName: ServiceNameECS, - Region: cfg.GetRegionId(), - ProjectId: cfg.GetProjectId(), - version: "v2", - Keyword: "keypair", - KeywordPlural: "keypairs", - - ResourceKeyword: "os-keypairs", - }} -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go deleted file mode 100644 index e7a488a828..0000000000 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/client/modules/mod_quotas.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package modules - -import ( - "yunion.io/x/cloudmux/pkg/multicloud/huawei/client/manager" -) - -type SQuotaManager struct { - SResourceManager -} - -func NewQuotaManager(cfg manager.IManagerConfig) *SQuotaManager { - return &SQuotaManager{SResourceManager: SResourceManager{ - SBaseManager: NewBaseManager(cfg), - ServiceName: ServiceNameEVS, - Region: cfg.GetRegionId(), - ProjectId: cfg.GetProjectId(), - version: "v1", - Keyword: "quotas", - KeywordPlural: "quotas", - - ResourceKeyword: "quotas", - }} -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go index 23ab4ec026..8b36d1884d 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/huawei.go @@ -627,6 +627,20 @@ func (self *SHuaweiClient) QueryAccountBalance() (*SBalance, error) { return &SBalance{Currency: "CYN"}, nil } +func (self *SHuaweiClient) GetISSLCertificates() ([]cloudprovider.ICloudSSLCertificate, error) { + ret, err := self.GetSSLCertificates() + if err != nil { + return nil, errors.Wrapf(err, "GetSSLCertificates") + } + + result := make([]cloudprovider.ICloudSSLCertificate, 0) + for i := range ret { + ret[i].client = self + result = append(result, &ret[i]) + } + return result, nil +} + func (self *SHuaweiClient) GetVersion() string { return HUAWEI_API_VERSION } @@ -865,6 +879,14 @@ func (self *SHuaweiClient) getUrl(service, regionId, resource string, method htt url = fmt.Sprintf("https://cts.%s.myhuaweicloud.com/v3/%s/%s", regionId, self.projectId, resource) case SERVICE_NAT: url = fmt.Sprintf("https://nat.%s.myhuaweicloud.com/v2/%s/%s", regionId, self.projectId, resource) + case SERVICE_SCM: + url = fmt.Sprintf("https://scm.cn-north-4.myhuaweicloud.com/v3/%s", resource) + case SERVICE_CDN: + url = fmt.Sprintf("https://cdn.myhuaweicloud.com/v1.0/%s", resource) + case SERVICE_GAUSSDB, SERVICE_GAUSSDB_NOSQL: + url = fmt.Sprintf("https://%s.%s.myhuaweicloud.com/v3/%s/%s", service, regionId, self.projectId, resource) + case SERVICE_FUNCTIONGRAPH: + url = fmt.Sprintf("https://%s.%s.myhuaweicloud.com/v2/%s/%s", service, regionId, self.projectId, resource) default: return "", fmt.Errorf("invalid service %s", service) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go index f0d4db7f14..65c551ab32 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/keypair.go @@ -16,6 +16,7 @@ package huawei import ( "fmt" + "net/url" "strconv" "strings" "time" @@ -23,14 +24,15 @@ import ( "github.com/aokoli/goutils" "golang.org/x/crypto/ssh" - "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" ) -// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html type SKeypair struct { - Fingerprint string `json:"fingerprint"` - Name string `json:"name"` - PublicKey string `json:"public_key"` + Keypair struct { + Fingerprint string `json:"fingerprint"` + Name string `json:"name"` + PublicKey string `json:"public_key"` + } } func (self *SRegion) getFingerprint(publicKey string) (string, error) { @@ -43,15 +45,22 @@ func (self *SRegion) getFingerprint(publicKey string) (string, error) { return fingerprint, nil } -// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212676.html -func (self *SRegion) GetKeypairs() ([]SKeypair, int, error) { - keypairs := make([]SKeypair, 0) - err := doListAll(self.ecsClient.Keypairs.List, nil, &keypairs) - return keypairs, len(keypairs), err +func (self *SRegion) GetKeypairs() ([]SKeypair, error) { + ret := []SKeypair{} + query := url.Values{} + resp, err := self.list(SERVICE_ECS, "os-keypairs", query) + if err != nil { + return nil, errors.Wrapf(err, "list os-keypairs") + } + err = resp.Unmarshal(&ret, "keypairs") + if err != nil { + return nil, err + } + return ret, nil } func (self *SRegion) lookUpKeypair(publicKey string) (string, error) { - keypairs, _, err := self.GetKeypairs() + keypairs, err := self.GetKeypairs() if err != nil { return "", err } @@ -62,24 +71,30 @@ func (self *SRegion) lookUpKeypair(publicKey string) (string, error) { } for _, keypair := range keypairs { - if keypair.Fingerprint == fingerprint { - return keypair.Name, nil + if keypair.Keypair.Fingerprint == fingerprint { + return keypair.Keypair.Name, nil } } return "", fmt.Errorf("keypair not found %s", err) } -// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212678.html +// https://console.huaweicloud.com/apiexplorer/#/openapi/ECS/doc?api=NovaCreateKeypair func (self *SRegion) ImportKeypair(name, publicKey string) (*SKeypair, error) { - keypairObj := jsonutils.NewDict() - keypairObj.Add(jsonutils.NewString(name), "name") - keypairObj.Add(jsonutils.NewString(publicKey), "public_key") - params := jsonutils.NewDict() - params.Set("keypair", keypairObj) - ret := SKeypair{} - err := DoCreate(self.ecsClient.Keypairs.Create, params, &ret) - return &ret, err + params := map[string]interface{}{ + "name": name, + "public_key": publicKey, + } + resp, err := self.post(SERVICE_ECS, "os-keypairs", map[string]interface{}{"keypair": params}) + if err != nil { + return nil, errors.Wrapf(err, "create os-keypairs") + } + ret := &SKeypair{} + err = resp.Unmarshal(ret) + if err != nil { + return nil, errors.Wrapf(err, "Unmarshal") + } + return ret, nil } func (self *SRegion) importKeypair(publicKey string) (string, error) { @@ -92,7 +107,7 @@ func (self *SRegion) importKeypair(publicKey string) (string, error) { if k, e := self.ImportKeypair(name, publicKey); e != nil { return "", fmt.Errorf("keypair import error %s", e) } else { - return k.Name, nil + return k.Keypair.Name, nil } } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go deleted file mode 100644 index a26fc137b7..0000000000 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/quota.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package huawei - -import ( - "yunion.io/x/pkg/errors" - - "yunion.io/x/cloudmux/pkg/cloudprovider" -) - -type SQuota struct { - Min int - Quota int - Type string - Used int -} - -func (q *SQuota) GetGlobalId() string { - return q.Type -} - -func (q *SQuota) GetQuotaType() string { - return q.Type -} - -func (q *SQuota) GetName() string { - return q.Type -} - -func (q *SQuota) GetDesc() string { - return "" -} - -func (q *SQuota) GetMaxQuotaCount() int { - return q.Quota -} - -func (q *SQuota) GetCurrentQuotaUsedCount() int { - return q.Used -} - -func (self *SRegion) GetQuotas() ([]SQuota, error) { - quotas := []SQuota{} - params := map[string]string{} - result, err := self.ecsClient.Quotas.Get("", params) - if err != nil { - return nil, errors.Wrap(err, "Quotas.List") - } - - err = result.Unmarshal("as, "resources") - if err != nil { - return nil, errors.Wrap(err, "result.Unmarshal") - } - - return quotas, nil -} - -func (region *SRegion) GetICloudQuotas() ([]cloudprovider.ICloudQuota, error) { - quotas, err := region.GetQuotas() - if err != nil { - return nil, errors.Wrap(err, "GetQuotas") - } - ret := []cloudprovider.ICloudQuota{} - for i := range quotas { - ret = append(ret, "as[i]) - } - return ret, nil -} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go index 2adc99d628..2062709184 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/sslcertificate.go @@ -9,7 +9,6 @@ import ( "net/url" "time" - "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/cloudmux/pkg/multicloud" "yunion.io/x/pkg/errors" ) @@ -17,7 +16,7 @@ import ( type SSSLCertificate struct { multicloud.SVirtualResourceBase HuaweiTags - region *SRegion + client *SHuaweiClient Id string // 证书ID Name string // 证书名称 @@ -138,7 +137,7 @@ func (s *SSSLCertificate) GetKey() string { func (s *SSSLCertificate) GetDetails() (*SSSLCertificate, error) { if !s.detailsInitd { - cert, err := s.region.GetCertificate(s.GetId()) + cert, err := s.client.GetCertificate(s.GetId()) if err != nil { return nil, err } @@ -149,70 +148,41 @@ func (s *SSSLCertificate) GetDetails() (*SSSLCertificate, error) { return s, nil } -// GetISSLCertificates 获取证书资源列表 -func (r *SRegion) GetISSLCertificates() ([]cloudprovider.ICloudSSLCertificate, error) { +func (r *SHuaweiClient) GetSSLCertificates() ([]SSSLCertificate, error) { + params := url.Values{} + params.Set("sort_key", "certExpiredTime") + params.Set("sort_dir", "DESC") ret := make([]SSSLCertificate, 0) - offset := 0 - for { - part, total, err := r.GetSSLCertificates(50, offset) + resp, err := r.list(SERVICE_SCM, "", "scm/certificates", params) if err != nil { - return nil, errors.Wrapf(err, "GetSSLCertificates") + return nil, errors.Wrapf(err, "list certificates") } - - ret = append(ret, part...) - if len(ret) >= total { + part := struct { + Certificates []SSSLCertificate + TotalCount int + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, err + } + ret = append(ret, part.Certificates...) + if len(ret) >= part.TotalCount || len(part.Certificates) == 0 { break } - - offset += 50 + params.Set("offset", fmt.Sprintf("%d", len(ret))) } - - result := make([]cloudprovider.ICloudSSLCertificate, 0) - for i := range ret { - ret[i].region = r - result = append(result, &ret[i]) - } - return result, nil + return ret, nil } -func (r *SRegion) GetSSLCertificates(size, offset int) ([]SSSLCertificate, int, error) { - if size < 1 || size > 50 { - size = 50 - } - if offset < 0 { - offset = 0 - } - - params := url.Values{ - "limit": []string{fmt.Sprintf("%d", size)}, - "offset": []string{fmt.Sprintf("%d", offset)}, - "sort_key": []string{"certExpiredTime"}, - "sort_dir": []string{"DESC"}, - } - resp, err := r.list(SERVICE_SCM, "scm/certificates", params) - if err != nil { - return nil, 0, errors.Wrapf(err, "CertificateList") - } - - ret := make([]SSSLCertificate, 0) - err = resp.Unmarshal(&ret, "certificates") - if err != nil { - return nil, 0, errors.Wrapf(err, "resp.Unmarshal") - } - - totalCount, _ := resp.Int("total_count") - return ret, int(totalCount), nil -} - -func (r *SRegion) GetCertificate(certId string) (*SSSLCertificate, error) { +func (cli *SHuaweiClient) GetCertificate(certId string) (*SSSLCertificate, error) { resource := fmt.Sprintf("scm/certificates/%s/export", certId) - resp, err := r.post(SERVICE_SCM, resource, nil) + resp, err := cli.post(SERVICE_SCM, "", resource, nil) if err != nil { return nil, errors.Wrap(err, "DescribeCertificateDetail") } - cert := &SSSLCertificate{region: r} + cert := &SSSLCertificate{client: cli} err = resp.Unmarshal(cert) if err != nil { return nil, errors.Wrap(err, "Unmarshal")