From e2826b1338ee807c64929e05983a228d0ead5f28 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Mon, 16 Mar 2020 18:22:21 +0800 Subject: [PATCH 01/42] cloudcommon: add ProxySetting --- pkg/apis/cloudcommon/proxy/doc.go | 1 + pkg/apis/cloudcommon/proxy/proxy.go | 36 ++++++++ .../cloudcommon/proxy/zz_generated.model.go | 27 ++++++ pkg/cloudcommon/db/proxy/doc.go | 1 + pkg/cloudcommon/db/proxy/proxysetting.go | 90 +++++++++++++++++++ scripts/codegen.py | 1 + 6 files changed, 156 insertions(+) create mode 100644 pkg/apis/cloudcommon/proxy/doc.go create mode 100644 pkg/apis/cloudcommon/proxy/proxy.go create mode 100644 pkg/apis/cloudcommon/proxy/zz_generated.model.go create mode 100644 pkg/cloudcommon/db/proxy/doc.go create mode 100644 pkg/cloudcommon/db/proxy/proxysetting.go diff --git a/pkg/apis/cloudcommon/proxy/doc.go b/pkg/apis/cloudcommon/proxy/doc.go new file mode 100644 index 0000000000..5ed87c1edc --- /dev/null +++ b/pkg/apis/cloudcommon/proxy/doc.go @@ -0,0 +1 @@ +package proxy // import "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy" diff --git a/pkg/apis/cloudcommon/proxy/proxy.go b/pkg/apis/cloudcommon/proxy/proxy.go new file mode 100644 index 0000000000..6f4f619b86 --- /dev/null +++ b/pkg/apis/cloudcommon/proxy/proxy.go @@ -0,0 +1,36 @@ +package proxy + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis" +) + +const ( + ProxySettingId_DIRECT = "DIRECT" +) + +type ProxySettingCreateInput struct { + apis.VirtualResourceCreateInput + + HttpProxy string + HttpsProxy string + NoProxy string +} + +type ProxySettingUpdateInput ProxySettingCreateInput + +// String implements ISerializable interface +func (ps *SProxySetting) String() string { + return jsonutils.Marshal(ps).String() +} + +// IsZero implements ISerializable interface +func (ps *SProxySetting) IsZero() bool { + if ps.HTTPProxy == "" && + ps.HTTPSProxy == "" && + ps.NoProxy == "" { + return true + } + return false +} diff --git a/pkg/apis/cloudcommon/proxy/zz_generated.model.go b/pkg/apis/cloudcommon/proxy/zz_generated.model.go new file mode 100644 index 0000000000..ee143b6f38 --- /dev/null +++ b/pkg/apis/cloudcommon/proxy/zz_generated.model.go @@ -0,0 +1,27 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by model-api-gen. DO NOT EDIT. + +package proxy + +import ( + "yunion.io/x/onecloud/pkg/apis" +) + +// SProxySetting is an autogenerated struct via yunion.io/x/onecloud/pkg/cloudcommon/db/proxy.SProxySetting. +type SProxySetting struct { + apis.SStandaloneResourceBase + HTTPProxy string `json:"http_proxy"` + HTTPSProxy string `json:"https_proxy"` + NoProxy string `json:"no_proxy"` +} diff --git a/pkg/cloudcommon/db/proxy/doc.go b/pkg/cloudcommon/db/proxy/doc.go new file mode 100644 index 0000000000..b5fd7e2ca8 --- /dev/null +++ b/pkg/cloudcommon/db/proxy/doc.go @@ -0,0 +1 @@ +package proxy // import "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy" diff --git a/pkg/cloudcommon/db/proxy/proxysetting.go b/pkg/cloudcommon/db/proxy/proxysetting.go new file mode 100644 index 0000000000..e466c4389b --- /dev/null +++ b/pkg/cloudcommon/db/proxy/proxysetting.go @@ -0,0 +1,90 @@ +package proxy + +import ( + "context" + "database/sql" + "net/http" + "net/url" + + "golang.org/x/net/http/httpproxy" + + "yunion.io/x/jsonutils" + + proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type SProxySettingManager struct { + db.SStandaloneResourceBaseManager +} + +var ProxySettingManager *SProxySettingManager + +func init() { + ProxySettingManager = &SProxySettingManager{ + SStandaloneResourceBaseManager: db.NewStandaloneResourceBaseManager( + SProxySetting{}, + "proxysettings_tbl", + "proxysetting", + "proxysettings", + ), + } + ProxySettingManager.SetVirtualObject(ProxySettingManager) +} + +type SProxySetting struct { + db.SStandaloneResourceBase + + HTTPProxy string `create:"admin_optional" list:"admin" update:"admin"` + HTTPSProxy string `create:"admin_optional" list:"admin" update:"admin"` + NoProxy string `create:"admin_optional" list:"admin" update:"admin"` +} + +func (man *SProxySettingManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data proxyapi.ProxySettingCreateInput) (proxyapi.ProxySettingCreateInput, error) { + return data, nil +} + +func (ps *SProxySetting) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data proxyapi.ProxySettingUpdateInput) (proxyapi.ProxySettingUpdateInput, error) { + if ps.Id == proxyapi.ProxySettingId_DIRECT { + return data, httperrors.NewConflictError("DIRECT setting cannot be changed") + } + return data, nil +} + +func (ps *SProxySetting) HttpTransportProxyFunc() httputils.TransportProxyFunc { + cfg := &httpproxy.Config{ + HTTPProxy: ps.HTTPProxy, + HTTPSProxy: ps.HTTPSProxy, + NoProxy: ps.NoProxy, + } + proxyFunc := cfg.ProxyFunc() + return func(req *http.Request) (*url.URL, error) { + return proxyFunc(req.URL) + } +} + +func (man *SProxySettingManager) InitializeData() error { + _, err := man.FetchById(proxyapi.ProxySettingId_DIRECT) + if err == nil { + return nil + } + if err != sql.ErrNoRows { + return err + } + + m, err := db.NewModelObject(man) + if err != nil { + return err + } + ps := m.(*SProxySetting) + ps.Id = proxyapi.ProxySettingId_DIRECT + ps.Name = proxyapi.ProxySettingId_DIRECT + ps.Description = "Connect directly" + if err := man.TableSpec().Insert(ps); err != nil { + return err + } + return nil +} diff --git a/scripts/codegen.py b/scripts/codegen.py index b42b72ea01..b90eabd8de 100755 --- a/scripts/codegen.py +++ b/scripts/codegen.py @@ -124,6 +124,7 @@ class ModelAPI(FuncDispatcher): def gen_cloudcommon(self): self.run(pkg=["cloudcommon", "db"]) + self.run(pkg=["cloudcommon", "db", "proxy"], out=["cloudcommon", "proxy"]) def gen_cloudprovider(self): self.run_same("cloudprovider") From 8009857c8a8c060693eb33391658ccb8eba2fb03 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 25 Mar 2020 17:16:35 +0800 Subject: [PATCH 02/42] cloudprovider: use ProviderConfig --- pkg/cloudevent/models/cloudproviders.go | 11 ++++++- pkg/cloudprovider/cloudprovider.go | 29 ++++++++++++++----- pkg/compute/models/cloudaccounts.go | 25 ++++++++++++++-- pkg/compute/models/cloudproviders.go | 9 +++++- pkg/multicloud/aliyun/provider/provider.go | 6 ++-- pkg/multicloud/aws/provider/provider.go | 4 +-- pkg/multicloud/azure/provider/provider.go | 6 ++-- pkg/multicloud/ctyun/provider/provider.go | 7 +++-- pkg/multicloud/esxi/provider/provider.go | 6 ++-- pkg/multicloud/google/provider/provider.go | 10 +++---- pkg/multicloud/huawei/provider/provider.go | 6 ++-- .../objectstore/ceph/provider/provider.go | 4 +-- .../objectstore/provider/provider.go | 7 +++-- .../objectstore/xsky/provider/provider.go | 4 +-- pkg/multicloud/openstack/provider/provider.go | 8 ++--- pkg/multicloud/qcloud/provider/provider.go | 8 ++--- pkg/multicloud/ucloud/provider/provider.go | 6 ++-- pkg/multicloud/zstack/provider/provider.go | 4 +-- pkg/s3gateway/models/cloudproviders.go | 9 +++++- 19 files changed, 116 insertions(+), 53 deletions(-) diff --git a/pkg/cloudevent/models/cloudproviders.go b/pkg/cloudevent/models/cloudproviders.go index 2779d57b69..34817f340e 100644 --- a/pkg/cloudevent/models/cloudproviders.go +++ b/pkg/cloudevent/models/cloudproviders.go @@ -299,7 +299,16 @@ func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, err if err != nil { return nil, err } - return cloudprovider.GetProvider(provider.Id, provider.Name, accessUrl, provider.Account, passwd, provider.Provider) + return cloudprovider.GetProvider( + cloudprovider.ProviderConfig{ + Id: provider.Id, + Name: provider.Name, + Vendor: provider.Provider, + URL: accessUrl, + Account: provider.Account, + Secret: passwd, + }, + ) } func (manager *SCloudproviderManager) InitializeData() error { diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 2b0abd97b0..8349af9de9 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -140,8 +141,22 @@ type SCloudaccount struct { AccessUrl string `json:"access_url"` } +type ProviderConfig struct { + // Id, Name are properties of Cloudprovider object + Id string + Name string + + // Vendor are names like Aliyun, OpenStack, etc. + Vendor string + URL string + Account string + Secret string + + ProxyFunc httputils.TransportProxyFunc +} + type ICloudProviderFactory interface { - GetProvider(providerId, providerName, url, account, secret string) (ICloudProvider, error) + GetProvider(cfg ProviderConfig) (ICloudProvider, error) GetClientRC(url, account, secret string) (map[string]string, error) @@ -240,12 +255,12 @@ func GetRegistedProviderIds() []string { return providers } -func GetProvider(providerId, providerName, accessUrl, account, secret, provider string) (ICloudProvider, error) { - driver, err := GetProviderFactory(provider) +func GetProvider(cfg ProviderConfig) (ICloudProvider, error) { + driver, err := GetProviderFactory(cfg.Vendor) if err != nil { return nil, errors.Wrap(err, "GetProviderFactory") } - return driver.GetProvider(providerId, providerName, accessUrl, account, secret) + return driver.GetProvider(cfg) } func GetClientRC(accessUrl, account, secret, provider string) (map[string]string, error) { @@ -261,10 +276,10 @@ func IsSupported(provider string) bool { return ok } -func IsValidCloudAccount(accessUrl, account, secret, provider string) (string, error) { - factory, ok := providerTable[provider] +func IsValidCloudAccount(cfg ProviderConfig) (string, error) { + factory, ok := providerTable[cfg.Vendor] if ok { - provider, err := factory.GetProvider("", "", accessUrl, account, secret) + provider, err := factory.GetProvider(cfg) if err != nil { return "", err } diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index d65b8b1fe0..622b793b55 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -148,6 +148,8 @@ type SCloudaccount struct { IsPublic bool `default:"false" nullable:"false"` // add share_mode field to indicate the share range of this account ShareMode string `width:"32" charset:"ascii" nullable:"true" list:"domain"` + + ProxySettingId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" default:"default"` } func (self *SCloudaccountManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -362,7 +364,12 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use return input, httperrors.NewConflictError("The account has been registered") } - accountId, err := cloudprovider.IsValidCloudAccount(input.AccessUrl, input.Account, input.Secret, input.Provider) + accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{ + Vendor: input.Provider, + URL: input.AccessUrl, + Account: input.Account, + Secret: input.Secret, + }) if err != nil { if err == cloudprovider.ErrNoSuchProvder { return input, httperrors.NewResourceNotFoundError("no such provider %s", input.Provider) @@ -516,7 +523,12 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred originSecret, _ := self.getPassword() - accountId, err := cloudprovider.IsValidCloudAccount(self.AccessUrl, account.Account, account.Secret, self.Provider) + accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{ + Vendor: self.Provider, + URL: self.AccessUrl, + Account: account.Account, + Secret: account.Secret, + }) if err != nil { return nil, httperrors.NewInputParameterError("invalid cloud account info error: %s", err.Error()) } @@ -673,7 +685,14 @@ func (self *SCloudaccount) getProviderInternal() (cloudprovider.ICloudProvider, if err != nil { return nil, fmt.Errorf("Invalid password %s", err) } - return cloudprovider.GetProvider(self.Id, self.Name, self.AccessUrl, self.Account, secret, self.Provider) + return cloudprovider.GetProvider(cloudprovider.ProviderConfig{ + Id: self.Id, + Name: self.Name, + Vendor: self.Provider, + URL: self.AccessUrl, + Account: self.Account, + Secret: secret, + }) } func (self *SCloudaccount) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 5bfa2592e7..4b642a619c 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -737,7 +737,14 @@ func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) if err != nil { return nil, err } - return cloudprovider.GetProvider(self.Id, self.Name, accessUrl, self.Account, passwd, self.Provider) + return cloudprovider.GetProvider(cloudprovider.ProviderConfig{ + Id: self.Id, + Name: self.Name, + Vendor: self.Provider, + URL: accessUrl, + Account: self.Account, + Secret: passwd, + }) } func (self *SCloudprovider) savePassword(secret string) error { diff --git a/pkg/multicloud/aliyun/provider/provider.go b/pkg/multicloud/aliyun/provider/provider.go index 2a14e0c99e..7476ba9fbc 100644 --- a/pkg/multicloud/aliyun/provider/provider.go +++ b/pkg/multicloud/aliyun/provider/provider.go @@ -71,8 +71,10 @@ func (self *SAliyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx con return output, nil } -func (self *SAliyunProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - client, err := aliyun.NewAliyunClient(providerId, providerName, account, secret, true) +func (self *SAliyunProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + client, err := aliyun.NewAliyunClient( + cfg.Id, cfg.Name, cfg.Account, cfg.Secret, true, + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/aws/provider/provider.go b/pkg/multicloud/aws/provider/provider.go index 9e09fb73e4..bd63ccec9e 100644 --- a/pkg/multicloud/aws/provider/provider.go +++ b/pkg/multicloud/aws/provider/provider.go @@ -75,8 +75,8 @@ func (self *SAwsProviderFactory) ValidateUpdateCloudaccountCredential(ctx contex return output, nil } -func (self *SAwsProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - client, err := aws.NewAwsClient(providerId, providerName, url, account, secret, false) +func (self *SAwsProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + client, err := aws.NewAwsClient(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/azure/provider/provider.go b/pkg/multicloud/azure/provider/provider.go index b78468fa22..eceab6daea 100644 --- a/pkg/multicloud/azure/provider/provider.go +++ b/pkg/multicloud/azure/provider/provider.go @@ -111,9 +111,9 @@ func parseAccount(account, secret string) (tenantId string, appId string, appKey return } -func (self *SAzureProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - tenantId, appId, appKey, subId := parseAccount(account, secret) - if client, err := azure.NewAzureClient(providerId, providerName, url, tenantId, appId, appKey, subId, false); err != nil { +func (self *SAzureProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + tenantId, appId, appKey, subId := parseAccount(cfg.Account, cfg.Secret) + if client, err := azure.NewAzureClient(cfg.Id, cfg.Name, cfg.URL, tenantId, appId, appKey, subId, false); err != nil { return nil, err } else { return &SAzureProvider{ diff --git a/pkg/multicloud/ctyun/provider/provider.go b/pkg/multicloud/ctyun/provider/provider.go index 0708dc7d9a..7ad6fb5287 100644 --- a/pkg/multicloud/ctyun/provider/provider.go +++ b/pkg/multicloud/ctyun/provider/provider.go @@ -76,15 +76,16 @@ func (self *SCtyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx cont return output, nil } -func (self *SCtyunProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - segs := strings.Split(account, "/") +func (self *SCtyunProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + segs := strings.Split(cfg.Account, "/") projectId := "" + account := cfg.Account if len(segs) == 2 { projectId = segs[1] account = segs[0] } - client, err := ctyun.NewSCtyunClient(providerId, providerName, projectId, account, secret, false) + client, err := ctyun.NewSCtyunClient(cfg.Id, cfg.Name, projectId, account, cfg.Secret, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/esxi/provider/provider.go b/pkg/multicloud/esxi/provider/provider.go index 9c28a8da40..5b0175f13f 100644 --- a/pkg/multicloud/esxi/provider/provider.go +++ b/pkg/multicloud/esxi/provider/provider.go @@ -101,8 +101,8 @@ func parseHostPort(host string, defPort int) (string, int, error) { } } -func (self *SESXiProviderFactory) GetProvider(providerId, providerName, urlStr, account, secret string) (cloudprovider.ICloudProvider, error) { - parts, err := url.Parse(urlStr) +func (self *SESXiProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + parts, err := url.Parse(cfg.URL) if err != nil { return nil, err } @@ -111,7 +111,7 @@ func (self *SESXiProviderFactory) GetProvider(providerId, providerName, urlStr, return nil, err } - client, err := esxi.NewESXiClient(providerId, providerName, host, port, account, secret) + client, err := esxi.NewESXiClient(cfg.Id, cfg.Name, host, port, cfg.Account, cfg.Secret) if err != nil { return nil, err } diff --git a/pkg/multicloud/google/provider/provider.go b/pkg/multicloud/google/provider/provider.go index 8b533d9e77..5b5b07372a 100644 --- a/pkg/multicloud/google/provider/provider.go +++ b/pkg/multicloud/google/provider/provider.go @@ -101,21 +101,21 @@ func (self *SGoogleProviderFactory) ValidateUpdateCloudaccountCredential(ctx con return output, nil } -func (self *SGoogleProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { +func (self *SGoogleProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { privateKeyID, privateKey := "", "" - privateKeyInfo := strings.Split(secret, "/") + privateKeyInfo := strings.Split(cfg.Secret, "/") if len(privateKeyInfo) < 2 { return nil, fmt.Errorf("Missing privateKeyID or privateKey for google cloud") } privateKeyID = privateKeyInfo[0] privateKey = strings.Join(privateKeyInfo[1:], "/") projectID, clientEmail := "", "" - accountInfo := strings.Split(account, "/") + accountInfo := strings.Split(cfg.Account, "/") if len(accountInfo) < 2 { - return nil, fmt.Errorf("Invalid projectID or client email for google cloud %s", account) + return nil, fmt.Errorf("Invalid projectID or client email for google cloud %s", cfg.Account) } projectID, clientEmail = accountInfo[0], accountInfo[1] - client, err := google.NewGoogleClient(providerId, providerName, projectID, clientEmail, privateKeyID, privateKey, false) + client, err := google.NewGoogleClient(cfg.Id, cfg.Name, projectID, clientEmail, privateKeyID, privateKey, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/huawei/provider/provider.go b/pkg/multicloud/huawei/provider/provider.go index 77bfa89682..3551e03ed6 100644 --- a/pkg/multicloud/huawei/provider/provider.go +++ b/pkg/multicloud/huawei/provider/provider.go @@ -99,9 +99,9 @@ func parseAccount(account string) (accessKey string, projectId string) { return } -func (self *SHuaweiProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - accessKey, projectId := parseAccount(account) - client, err := huawei.NewHuaweiClient(providerId, providerName, url, accessKey, secret, projectId, false) +func (self *SHuaweiProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + accessKey, projectId := parseAccount(cfg.Account) + client, err := huawei.NewHuaweiClient(cfg.Id, cfg.Name, cfg.URL, accessKey, cfg.Secret, projectId, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/objectstore/ceph/provider/provider.go b/pkg/multicloud/objectstore/ceph/provider/provider.go index af9e17865c..c4c60439ce 100644 --- a/pkg/multicloud/objectstore/ceph/provider/provider.go +++ b/pkg/multicloud/objectstore/ceph/provider/provider.go @@ -33,8 +33,8 @@ func (self *SCephRadosProviderFactory) GetName() string { return api.CLOUD_PROVIDER_CEPH } -func (self *SCephRadosProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - client, err := ceph.NewCephRados(providerId, providerName, url, account, secret, false) +func (self *SCephRadosProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + client, err := ceph.NewCephRados(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/objectstore/provider/provider.go b/pkg/multicloud/objectstore/provider/provider.go index 146f9a30f8..cf3bb7eaf0 100644 --- a/pkg/multicloud/objectstore/provider/provider.go +++ b/pkg/multicloud/objectstore/provider/provider.go @@ -71,8 +71,11 @@ func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ct return output, nil } -func (self *SObjectStoreProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - client, err := objectstore.NewObjectStoreClient(providerId, providerName, url, account, secret, false) +func (self *SObjectStoreProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + dbg := false + client, err := objectstore.NewObjectStoreClient( + cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, dbg, + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/objectstore/xsky/provider/provider.go b/pkg/multicloud/objectstore/xsky/provider/provider.go index fc88bd6dcc..dfa7e4e3c7 100644 --- a/pkg/multicloud/objectstore/xsky/provider/provider.go +++ b/pkg/multicloud/objectstore/xsky/provider/provider.go @@ -33,8 +33,8 @@ func (self *SXskyProviderFactory) GetName() string { return api.CLOUD_PROVIDER_XSKY } -func (self *SXskyProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - client, err := xsky.NewXskyClient(providerId, providerName, url, account, secret, false) +func (self *SXskyProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + client, err := xsky.NewXskyClient(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/openstack/provider/provider.go b/pkg/multicloud/openstack/provider/provider.go index 227439859e..fd7c335d3a 100644 --- a/pkg/multicloud/openstack/provider/provider.go +++ b/pkg/multicloud/openstack/provider/provider.go @@ -102,16 +102,16 @@ func (self *SOpenStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx return output, nil } -func (self *SOpenStackProviderFactory) GetProvider(providerId, providerName, url, account, password string) (cloudprovider.ICloudProvider, error) { - accountInfo := strings.Split(account, "/") +func (self *SOpenStackProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + accountInfo := strings.Split(cfg.Account, "/") if len(accountInfo) < 2 { - return nil, fmt.Errorf("Missing username or project name %s", account) + return nil, fmt.Errorf("Missing username or project name %s", cfg.Account) } project, username, endpointType, domainName, projectDomainName := accountInfo[0], accountInfo[1], "internal", "Default", "Default" if len(accountInfo) == 3 { domainName, projectDomainName = accountInfo[2], accountInfo[2] } - client, err := openstack.NewOpenStackClient(providerId, providerName, url, username, password, project, endpointType, domainName, projectDomainName, false) + client, err := openstack.NewOpenStackClient(cfg.Id, cfg.Name, cfg.URL, username, cfg.Secret, project, endpointType, domainName, projectDomainName, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/qcloud/provider/provider.go b/pkg/multicloud/qcloud/provider/provider.go index 6478fa19dc..564237ddaf 100644 --- a/pkg/multicloud/qcloud/provider/provider.go +++ b/pkg/multicloud/qcloud/provider/provider.go @@ -98,14 +98,14 @@ func (self *SQcloudProviderFactory) ValidateUpdateCloudaccountCredential(ctx con return output, nil } -func (self *SQcloudProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - secretId := account +func (self *SQcloudProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + secretId := cfg.Account appId := "" - if tmp := strings.Split(account, "/"); len(tmp) == 2 { + if tmp := strings.Split(cfg.Account, "/"); len(tmp) == 2 { secretId = tmp[0] appId = tmp[1] } - client, err := qcloud.NewQcloudClient(providerId, providerName, secretId, secret, appId, false) + client, err := qcloud.NewQcloudClient(cfg.Id, cfg.Name, secretId, cfg.Secret, appId, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/ucloud/provider/provider.go b/pkg/multicloud/ucloud/provider/provider.go index ae9b17daec..d8433b35de 100644 --- a/pkg/multicloud/ucloud/provider/provider.go +++ b/pkg/multicloud/ucloud/provider/provider.go @@ -82,9 +82,9 @@ func parseAccount(account string) (accessKey string, projectId string) { return } -func (self *SUcloudProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) { - accessKey, projectId := parseAccount(account) - client, err := ucloud.NewUcloudClient(providerId, providerName, accessKey, secret, projectId, false) +func (self *SUcloudProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + accessKey, projectId := parseAccount(cfg.Account) + client, err := ucloud.NewUcloudClient(cfg.Id, cfg.Name, accessKey, cfg.Secret, projectId, false) if err != nil { return nil, err } diff --git a/pkg/multicloud/zstack/provider/provider.go b/pkg/multicloud/zstack/provider/provider.go index 507c7705e0..c82ba70615 100644 --- a/pkg/multicloud/zstack/provider/provider.go +++ b/pkg/multicloud/zstack/provider/provider.go @@ -76,8 +76,8 @@ func (self *SZStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx con return output, nil } -func (self *SZStackProviderFactory) GetProvider(providerId, providerName, url, username, password string) (cloudprovider.ICloudProvider, error) { - client, err := zstack.NewZStackClient(providerId, providerName, url, username, password, false) +func (self *SZStackProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { + client, err := zstack.NewZStackClient(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) if err != nil { return nil, err } diff --git a/pkg/s3gateway/models/cloudproviders.go b/pkg/s3gateway/models/cloudproviders.go index 3b1e2b3931..866fe331a0 100644 --- a/pkg/s3gateway/models/cloudproviders.go +++ b/pkg/s3gateway/models/cloudproviders.go @@ -136,5 +136,12 @@ func (provider *SCloudproviderDelegate) GetProvider() (cloudprovider.ICloudProvi if err != nil { return nil, err } - return cloudprovider.GetProvider(provider.Id, provider.Name, accessUrl, provider.Account, passwd, provider.Provider) + return cloudprovider.GetProvider(cloudprovider.ProviderConfig{ + Id: provider.Id, + Name: provider.Name, + Vendor: provider.Provider, + URL: accessUrl, + Account: provider.Account, + Secret: passwd, + }) } From 85a9d51e5866c18931576c72c9e3e25948c88f5b Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 14:51:21 +0800 Subject: [PATCH 03/42] httputils: add SetClientProxyFunc(client, proxyFunc) --- pkg/util/httputils/httputils.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/util/httputils/httputils.go b/pkg/util/httputils/httputils.go index a20d15a1ff..cbd6fc013e 100644 --- a/pkg/util/httputils/httputils.go +++ b/pkg/util/httputils/httputils.go @@ -248,6 +248,20 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client { } } +type TransportProxyFunc func(*http.Request) (*url.URL, error) + +func SetClientProxyFunc( + client *http.Client, + proxyFunc TransportProxyFunc, +) bool { + set := false + if transport, ok := client.Transport.(*http.Transport); ok { + transport.Proxy = proxyFunc + set = true + } + return set +} + func GetTimeoutClient(timeout time.Duration) *http.Client { return GetClient(true, timeout) } From 4c50f4077169e1b89843f83549e2ece8f4e772e9 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 12:22:56 +0800 Subject: [PATCH 04/42] aliyun clientcfg --- cmd/aliyuncli/main.go | 6 +- pkg/multicloud/aliyun/aliyun.go | 67 +++++++++++++--------- pkg/multicloud/aliyun/bucket.go | 4 +- pkg/multicloud/aliyun/business.go | 2 +- pkg/multicloud/aliyun/host.go | 6 +- pkg/multicloud/aliyun/monitor.go | 2 +- pkg/multicloud/aliyun/natgateway.go | 2 +- pkg/multicloud/aliyun/natstable.go | 4 +- pkg/multicloud/aliyun/provider/provider.go | 5 +- pkg/multicloud/aliyun/ram.go | 2 +- pkg/multicloud/aliyun/region.go | 14 ++--- pkg/multicloud/aliyun/storage.go | 6 +- pkg/multicloud/aliyun/storagecache.go | 6 +- pkg/multicloud/aliyun/vswitch.go | 2 +- 14 files changed, 72 insertions(+), 56 deletions(-) diff --git a/cmd/aliyuncli/main.go b/cmd/aliyuncli/main.go index 456b8dd4d4..53344fe3ad 100644 --- a/cmd/aliyuncli/main.go +++ b/cmd/aliyuncli/main.go @@ -84,7 +84,11 @@ func newClient(options *BaseOptions) (*aliyun.SRegion, error) { return nil, fmt.Errorf("Missing secret") } - cli, err := aliyun.NewAliyunClient("", "", options.AccessKey, options.Secret, options.Debug) + cli, err := aliyun.NewAliyunClient( + aliyun.NewAliyunClientConfig( + options.AccessKey, options.Secret, + ).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/aliyun/aliyun.go b/pkg/multicloud/aliyun/aliyun.go index 9cc0080876..2bb6b29d56 100644 --- a/pkg/multicloud/aliyun/aliyun.go +++ b/pkg/multicloud/aliyun/aliyun.go @@ -53,28 +53,48 @@ const ( ALIYUN_API_VERION_RDS = "2014-08-15" ) -type SAliyunClient struct { - providerId string - providerName string +type AliyunClientConfig struct { + cpcfg cloudprovider.ProviderConfig accessKey string - secret string + accessSecret string + debug bool +} + +func NewAliyunClientConfig(accessKey, accessSecret string) *AliyunClientConfig { + cfg := &AliyunClientConfig{ + accessKey: accessKey, + accessSecret: accessSecret, + } + return cfg +} + +func (cfg *AliyunClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *AliyunClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *AliyunClientConfig) Debug(debug bool) *AliyunClientConfig { + cfg.debug = debug + return cfg +} + +func (cfg AliyunClientConfig) Copy() AliyunClientConfig { + return cfg +} + +type SAliyunClient struct { + *AliyunClientConfig ownerId string ownerName string iregions []cloudprovider.ICloudRegion iBuckets []cloudprovider.ICloudBucket - - Debug bool } -func NewAliyunClient(providerId string, providerName string, accessKey string, secret string, isDebug bool) (*SAliyunClient, error) { +func NewAliyunClient(cfg *AliyunClientConfig) (*SAliyunClient, error) { client := SAliyunClient{ - providerId: providerId, - providerName: providerName, - accessKey: accessKey, - secret: secret, - Debug: isDebug, + AliyunClientConfig: cfg, } err := client.fetchRegions() if err != nil { @@ -84,7 +104,7 @@ func NewAliyunClient(providerId string, providerName string, accessKey string, s if err != nil { return nil, errors.Wrap(err, "fetchBuckets") } - if client.Debug { + if client.debug { log.Debugf("ClientID: %s ClientName: %s", client.ownerId, client.ownerName) } return &client, nil @@ -165,18 +185,9 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str return body, nil } -func (self *SAliyunClient) UpdateAccount(accessKey, secret string) error { - if self.accessKey != accessKey || self.secret != secret { - self.accessKey = accessKey - self.secret = secret - return self.fetchRegions() - } else { - return nil - } -} - func (self *SAliyunClient) getDefaultClient() (*sdk.Client, error) { - return sdk.NewClientWithAccessKey(ALIYUN_DEFAULT_REGION, self.accessKey, self.secret) + return sdk.NewClientWithAccessKey(ALIYUN_DEFAULT_REGION, + self.accessKey, self.accessSecret) } func (self *SAliyunClient) ecsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -184,7 +195,7 @@ func (self *SAliyunClient) ecsRequest(apiName string, params map[string]string) if err != nil { return nil, err } - return jsonRequest(cli, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params, self.Debug) + return jsonRequest(cli, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params, self.debug) } func (self *SAliyunClient) trialRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -192,7 +203,7 @@ func (self *SAliyunClient) trialRequest(apiName string, params map[string]string if err != nil { return nil, err } - return jsonRequest(cli, "actiontrail.cn-hangzhou.aliyuncs.com", ALIYUN_API_VERSION_TRIAL, apiName, params, self.Debug) + return jsonRequest(cli, "actiontrail.cn-hangzhou.aliyuncs.com", ALIYUN_API_VERSION_TRIAL, apiName, params, self.debug) } func (self *SAliyunClient) fetchRegions() error { @@ -241,7 +252,7 @@ func (client *SAliyunClient) getOssClient(regionId string) (*oss.Client, error) oss.HTTPClient(httputils.GetAdaptiveTimeoutClient()), } ep := getOSSExternalDomain(regionId) - cli, err := oss.New(ep, client.accessKey, client.secret, cliOpts...) + cli, err := oss.New(ep, client.accessKey, client.accessSecret, cliOpts...) if err != nil { return nil, errors.Wrap(err, "oss.New") } @@ -320,7 +331,7 @@ func (self *SAliyunClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) return nil, err } subAccount := cloudprovider.SSubAccount{} - subAccount.Name = self.providerName + subAccount.Name = self.cpcfg.Name subAccount.Account = self.accessKey subAccount.HealthStatus = api.CLOUD_PROVIDER_HEALTH_NORMAL return []cloudprovider.SSubAccount{subAccount}, nil diff --git a/pkg/multicloud/aliyun/bucket.go b/pkg/multicloud/aliyun/bucket.go index e3833538ed..678a3b8297 100644 --- a/pkg/multicloud/aliyun/bucket.go +++ b/pkg/multicloud/aliyun/bucket.go @@ -291,7 +291,7 @@ func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, p if err != nil { return "", errors.Wrap(err, "bucket.UploadPart") } - if b.region.client.Debug { + if b.region.client.debug { log.Debugf("upload part key:%s uploadId:%s partIndex:%d etag:%s", key, uploadId, partIndex, part.ETag) } return part.ETag, nil @@ -322,7 +322,7 @@ func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploa if err != nil { return errors.Wrap(err, "bucket.CompleteMultipartUpload") } - if b.region.client.Debug { + if b.region.client.debug { log.Debugf("CompleteMultipartUpload bucket:%s key:%s etag:%s location:%s", result.Bucket, result.Key, result.ETag, result.Location) } return nil diff --git a/pkg/multicloud/aliyun/business.go b/pkg/multicloud/aliyun/business.go index 8aa7d62340..e25e0f55d4 100644 --- a/pkg/multicloud/aliyun/business.go +++ b/pkg/multicloud/aliyun/business.go @@ -29,7 +29,7 @@ func (self *SAliyunClient) businessRequest(apiName string, params map[string]str if err != nil { return nil, err } - return jsonRequest(cli, "business.aliyuncs.com", ALIYUN_BSS_API_VERSION, apiName, params, self.Debug) + return jsonRequest(cli, "business.aliyuncs.com", ALIYUN_BSS_API_VERSION, apiName, params, self.debug) } type SAccountBalance struct { diff --git a/pkg/multicloud/aliyun/host.go b/pkg/multicloud/aliyun/host.go index 8dcc0d4bb8..d29ecb2eda 100644 --- a/pkg/multicloud/aliyun/host.go +++ b/pkg/multicloud/aliyun/host.go @@ -88,15 +88,15 @@ func (self *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) } func (self *SHost) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) IsEmulated() bool { diff --git a/pkg/multicloud/aliyun/monitor.go b/pkg/multicloud/aliyun/monitor.go index f4883b6c95..a45c81a498 100644 --- a/pkg/multicloud/aliyun/monitor.go +++ b/pkg/multicloud/aliyun/monitor.go @@ -31,7 +31,7 @@ func (r *SRegion) metricsRequest(action string, params map[string]string) (jsonu if err != nil { return nil, errors.Wrap(err, "r.getSdkClient") } - return jsonRequest(client, "metrics.aliyuncs.com", ALIYUN_API_VERSION_METRICS, action, params, r.client.Debug) + return jsonRequest(client, "metrics.aliyuncs.com", ALIYUN_API_VERSION_METRICS, action, params, r.client.debug) } type SResourceLabel struct { diff --git a/pkg/multicloud/aliyun/natgateway.go b/pkg/multicloud/aliyun/natgateway.go index 7326199ca9..1985cb3cd4 100644 --- a/pkg/multicloud/aliyun/natgateway.go +++ b/pkg/multicloud/aliyun/natgateway.go @@ -207,7 +207,7 @@ func (self *SRegion) GetNatGateways(vpcId string, natGwId string, offset, limit return nil, 0, err } - if self.client.Debug { + if self.client.debug { log.Debugf("%s", body.PrettyString()) } diff --git a/pkg/multicloud/aliyun/natstable.go b/pkg/multicloud/aliyun/natstable.go index 4347da8610..f6e60d893e 100644 --- a/pkg/multicloud/aliyun/natstable.go +++ b/pkg/multicloud/aliyun/natstable.go @@ -94,7 +94,7 @@ func (self *SRegion) GetSNATEntries(tableId string, offset, limit int) ([]SSNATT return nil, 0, err } - if self.client.Debug { + if self.client.debug { log.Debugf("%s", body.PrettyString()) } @@ -120,7 +120,7 @@ func (self *SRegion) GetSNATEntry(tableID, SNATEntryID string) (SSNATTableEntry, return SSNATTableEntry{}, err } - if self.client.Debug { + if self.client.debug { log.Debugf("%s", body.PrettyString()) } diff --git a/pkg/multicloud/aliyun/provider/provider.go b/pkg/multicloud/aliyun/provider/provider.go index 7476ba9fbc..c6823cd06c 100644 --- a/pkg/multicloud/aliyun/provider/provider.go +++ b/pkg/multicloud/aliyun/provider/provider.go @@ -73,7 +73,10 @@ func (self *SAliyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx con func (self *SAliyunProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { client, err := aliyun.NewAliyunClient( - cfg.Id, cfg.Name, cfg.Account, cfg.Secret, true, + aliyun.NewAliyunClientConfig( + cfg.Account, + cfg.Secret, + ).CloudproviderConfig(cfg), ) if err != nil { return nil, err diff --git a/pkg/multicloud/aliyun/ram.go b/pkg/multicloud/aliyun/ram.go index 203217c287..608bb57f5b 100644 --- a/pkg/multicloud/aliyun/ram.go +++ b/pkg/multicloud/aliyun/ram.go @@ -28,7 +28,7 @@ func (self *SAliyunClient) ramRequest(apiName string, params map[string]string) if err != nil { return nil, err } - return jsonRequest(cli, "ram.aliyuncs.com", ALIYUN_RAM_API_VERSION, apiName, params, self.Debug) + return jsonRequest(cli, "ram.aliyuncs.com", ALIYUN_RAM_API_VERSION, apiName, params, self.debug) } type SRole struct { diff --git a/pkg/multicloud/aliyun/region.go b/pkg/multicloud/aliyun/region.go index afe99aa3c2..3e6e831b1f 100644 --- a/pkg/multicloud/aliyun/region.go +++ b/pkg/multicloud/aliyun/region.go @@ -41,8 +41,6 @@ type SRegion struct { sdkClient *sdk.Client ossClient *oss.Client - Debug bool - RegionId string LocalName string @@ -75,7 +73,7 @@ func (self *SRegion) GetMetadata() *jsonutils.JSONDict { func (self *SRegion) getSdkClient() (*sdk.Client, error) { if self.sdkClient == nil { - cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.secret) + cli, err := sdk.NewClientWithAccessKey(self.RegionId, self.client.accessKey, self.client.accessSecret) if err != nil { return nil, err } @@ -108,7 +106,7 @@ func (self *SRegion) ecsRequest(apiName string, params map[string]string) (jsonu if err != nil { return nil, err } - return jsonRequest(client, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params, self.client.Debug) + return jsonRequest(client, "ecs.aliyuncs.com", ALIYUN_API_VERSION, apiName, params, self.client.debug) } func (self *SRegion) rdsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -116,7 +114,7 @@ func (self *SRegion) rdsRequest(apiName string, params map[string]string) (jsonu if err != nil { return nil, err } - return jsonRequest(client, "rds.aliyuncs.com", ALIYUN_API_VERION_RDS, apiName, params, self.client.Debug) + return jsonRequest(client, "rds.aliyuncs.com", ALIYUN_API_VERION_RDS, apiName, params, self.client.debug) } func (self *SRegion) vpcRequest(action string, params map[string]string) (jsonutils.JSONObject, error) { @@ -124,7 +122,7 @@ func (self *SRegion) vpcRequest(action string, params map[string]string) (jsonut if err != nil { return nil, err } - return jsonRequest(client, "vpc.aliyuncs.com", ALIYUN_API_VERSION_VPC, action, params, self.client.Debug) + return jsonRequest(client, "vpc.aliyuncs.com", ALIYUN_API_VERSION_VPC, action, params, self.client.debug) } func (self *SRegion) kvsRequest(action string, params map[string]string) (jsonutils.JSONObject, error) { @@ -132,7 +130,7 @@ func (self *SRegion) kvsRequest(action string, params map[string]string) (jsonut if err != nil { return nil, err } - return jsonRequest(client, "r-kvstore.aliyuncs.com", ALIYUN_API_VERSION_KVS, action, params, self.client.Debug) + return jsonRequest(client, "r-kvstore.aliyuncs.com", ALIYUN_API_VERSION_KVS, action, params, self.client.debug) } type LBRegion struct { @@ -182,7 +180,7 @@ func (self *SRegion) lbRequest(apiName string, params map[string]string) (jsonut } func (self *SRegion) _lbRequest(client *sdk.Client, apiName string, domain string, params map[string]string) (jsonutils.JSONObject, error) { - return jsonRequest(client, domain, ALIYUN_API_VERSION_LB, apiName, params, self.Debug) + return jsonRequest(client, domain, ALIYUN_API_VERSION_LB, apiName, params, self.client.debug) } ///////////////////////////////////////////////////////////////////////////// diff --git a/pkg/multicloud/aliyun/storage.go b/pkg/multicloud/aliyun/storage.go index b8cd089c6e..a4dc0d8b17 100644 --- a/pkg/multicloud/aliyun/storage.go +++ b/pkg/multicloud/aliyun/storage.go @@ -49,15 +49,15 @@ func (self *SStorage) GetMetadata() *jsonutils.JSONDict { } func (self *SStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), self.storageType) } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) } func (self *SStorage) GetGlobalId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) } func (self *SStorage) IsEmulated() bool { diff --git a/pkg/multicloud/aliyun/storagecache.go b/pkg/multicloud/aliyun/storagecache.go index 26cdb9a55d..6298cd3faa 100644 --- a/pkg/multicloud/aliyun/storagecache.go +++ b/pkg/multicloud/aliyun/storagecache.go @@ -46,11 +46,11 @@ func (self *SStoragecache) GetMetadata() *jsonutils.JSONDict { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetStatus() string { @@ -62,7 +62,7 @@ func (self *SStoragecache) Refresh() error { } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) IsEmulated() bool { diff --git a/pkg/multicloud/aliyun/vswitch.go b/pkg/multicloud/aliyun/vswitch.go index 546bda11e5..d09ca46af7 100644 --- a/pkg/multicloud/aliyun/vswitch.go +++ b/pkg/multicloud/aliyun/vswitch.go @@ -246,7 +246,7 @@ func (self *SRegion) GetVSwitchAttributes(idstr string) (*SVSwitch, error) { log.Errorf("DescribeVSwitchAttributes fail %s", err) return nil, err } - if self.client.Debug { + if self.client.debug { log.Debugf("%s", body.PrettyString()) } switches := SVSwitch{} From 7da2fa556ca0131f9ceea71211d11d924add97d8 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 14:45:06 +0800 Subject: [PATCH 05/42] aliyun: use cpcfg.ProxyFunc --- pkg/multicloud/aliyun/aliyun.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pkg/multicloud/aliyun/aliyun.go b/pkg/multicloud/aliyun/aliyun.go index 2bb6b29d56..de760dde8a 100644 --- a/pkg/multicloud/aliyun/aliyun.go +++ b/pkg/multicloud/aliyun/aliyun.go @@ -20,6 +20,7 @@ import ( "time" "github.com/aliyun/alibaba-cloud-sdk-go/sdk" + "github.com/aliyun/alibaba-cloud-sdk-go/sdk/auth/credentials" "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/pkg/errors" @@ -186,8 +187,19 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str } func (self *SAliyunClient) getDefaultClient() (*sdk.Client, error) { - return sdk.NewClientWithAccessKey(ALIYUN_DEFAULT_REGION, - self.accessKey, self.accessSecret) + transport := httputils.GetTransport(true) + transport.Proxy = self.cpcfg.ProxyFunc + client, err := sdk.NewClientWithOptions( + ALIYUN_DEFAULT_REGION, + &sdk.Config{ + HttpTransport: transport, + }, + &credentials.BaseCredential{ + AccessKeyId: self.accessKey, + AccessKeySecret: self.accessSecret, + }, + ) + return client, err } func (self *SAliyunClient) ecsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -248,8 +260,10 @@ func (client *SAliyunClient) getOssClient(regionId string) (*oss.Client, error) // which can be used to whitelist ips, domains from http_proxy, // https_proxy setting // oss use no timeout client so as to send/download large files + httpClient := httputils.GetAdaptiveTimeoutClient() + httputils.SetClientProxyFunc(httpClient, client.cpcfg.ProxyFunc) cliOpts := []oss.ClientOption{ - oss.HTTPClient(httputils.GetAdaptiveTimeoutClient()), + oss.HTTPClient(httpClient), } ep := getOSSExternalDomain(regionId) cli, err := oss.New(ep, client.accessKey, client.accessSecret, cliOpts...) From 01ca9543990942e244b4a1a6e54251914828cd54 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 25 Mar 2020 17:19:52 +0800 Subject: [PATCH 06/42] google: use GoogleClientConfig --- cmd/googlecli/main.go | 9 +- pkg/multicloud/google/google.go | 96 +++++++++++++--------- pkg/multicloud/google/host.go | 2 +- pkg/multicloud/google/provider/provider.go | 7 +- pkg/multicloud/google/storagecache.go | 6 +- 5 files changed, 76 insertions(+), 44 deletions(-) diff --git a/cmd/googlecli/main.go b/cmd/googlecli/main.go index e4821c5272..1d3fc0df0d 100644 --- a/cmd/googlecli/main.go +++ b/cmd/googlecli/main.go @@ -94,7 +94,14 @@ func newClient(options *BaseOptions) (*google.SRegion, error) { return nil, fmt.Errorf("Missing ProjectID") } - cli, err := google.NewGoogleClient("", "", options.ProjectID, options.ClientEmail, options.PrivateKeyID, options.PrivateKey, options.Debug) + cli, err := google.NewGoogleClient( + google.NewGoogleClientConfig( + options.ProjectID, + options.ClientEmail, + options.PrivateKeyID, + options.PrivateKey, + ).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/google/google.go b/pkg/multicloud/google/google.go index 134d32b680..cbc3836a59 100644 --- a/pkg/multicloud/google/google.go +++ b/pkg/multicloud/google/google.go @@ -61,13 +61,41 @@ const ( MAX_RETRY = 3 ) +type GoogleClientConfig struct { + cpcfg cloudprovider.ProviderConfig + + projectId string + clientEmail string + privateKeyId string + privateKey string + + debug bool +} + +func NewGoogleClientConfig(projectId, clientEmail, privateKeyId, privateKey string) *GoogleClientConfig { + privateKey = strings.Replace(privateKey, "\\n", "\n", -1) + cfg := &GoogleClientConfig{ + projectId: projectId, + clientEmail: clientEmail, + privateKeyId: privateKeyId, + privateKey: privateKey, + } + return cfg +} + +func (cfg *GoogleClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *GoogleClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *GoogleClientConfig) Debug(debug bool) *GoogleClientConfig { + cfg.debug = debug + return cfg +} + type SGoogleClient struct { - providerId string - providerName string - projectId string - privateKey string - privateKeyId string - clientEmail string + *GoogleClientConfig + iregions []cloudprovider.ICloudRegion images []SImage snapshots map[string][]SSnapshot @@ -75,24 +103,16 @@ type SGoogleClient struct { resourcepolices []SResourcePolicy client *http.Client - - Debug bool } -func NewGoogleClient(providerId string, providerName string, projectId, clientEmail, privateKeyId, privateKey string, isDebug bool) (*SGoogleClient, error) { +func NewGoogleClient(cfg *GoogleClientConfig) (*SGoogleClient, error) { client := SGoogleClient{ - providerId: providerId, - providerName: providerName, - projectId: projectId, - privateKey: strings.Replace(privateKey, "\\n", "\n", -1), - privateKeyId: privateKeyId, - clientEmail: clientEmail, - Debug: isDebug, + GoogleClientConfig: cfg, } conf := &jwt.Config{ - Email: clientEmail, - PrivateKeyID: privateKeyId, - PrivateKey: []byte(client.privateKey), + Email: cfg.clientEmail, + PrivateKeyID: cfg.privateKeyId, + PrivateKey: []byte(cfg.privateKey), Scopes: []string{ "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/compute", @@ -146,7 +166,7 @@ func jsonRequest(client *http.Client, method httputils.THttpMethod, domain, apiV } func (self *SGoogleClient) ecsGet(resource string, retval interface{}) error { - resp, err := jsonRequest(self.client, "GET", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, nil, nil, self.Debug) + resp, err := jsonRequest(self.client, "GET", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, nil, nil, self.debug) if err != nil { return err } @@ -161,15 +181,15 @@ func (self *SGoogleClient) ecsGet(resource string, retval interface{}) error { func (self *SGoogleClient) ecsList(resource string, params map[string]string) (jsonutils.JSONObject, error) { resource = fmt.Sprintf("projects/%s/%s", self.projectId, resource) - return jsonRequest(self.client, "GET", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, nil, self.Debug) + return jsonRequest(self.client, "GET", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, nil, self.debug) } func (self *SGoogleClient) managerList(resource string, params map[string]string) (jsonutils.JSONObject, error) { - return jsonRequest(self.client, "GET", GOOGLE_MANAGER_DOMAIN, GOOGLE_MANAGER_API_VERSION, resource, params, nil, self.Debug) + return jsonRequest(self.client, "GET", GOOGLE_MANAGER_DOMAIN, GOOGLE_MANAGER_API_VERSION, resource, params, nil, self.debug) } func (self *SGoogleClient) managerGet(resource string) (jsonutils.JSONObject, error) { - return jsonRequest(self.client, "GET", GOOGLE_MANAGER_DOMAIN, GOOGLE_MANAGER_API_VERSION, resource, nil, nil, self.Debug) + return jsonRequest(self.client, "GET", GOOGLE_MANAGER_DOMAIN, GOOGLE_MANAGER_API_VERSION, resource, nil, nil, self.debug) } func (self *SGoogleClient) ecsListAll(resource string, params map[string]string, retval interface{}) error { @@ -201,7 +221,7 @@ func (self *SGoogleClient) ecsListAll(resource string, params map[string]string, } func (self *SGoogleClient) ecsDelete(id string, retval interface{}) error { - resp, err := jsonRequest(self.client, "DELETE", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, id, nil, nil, self.Debug) + resp, err := jsonRequest(self.client, "DELETE", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, id, nil, nil, self.debug) if err != nil { return err } @@ -215,7 +235,7 @@ func (self *SGoogleClient) ecsPatch(resource string, action string, params map[s if len(action) > 0 { resource = fmt.Sprintf("%s/%s", resource, action) } - resp, err := jsonRequest(self.client, "PATCH", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, body, self.Debug) + resp, err := jsonRequest(self.client, "PATCH", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, body, self.debug) if err != nil { return "", err } @@ -225,7 +245,7 @@ func (self *SGoogleClient) ecsPatch(resource string, action string, params map[s func (self *SGoogleClient) ecsDo(resource string, action string, params map[string]string, body jsonutils.JSONObject) (string, error) { resource = fmt.Sprintf("%s/%s", resource, action) - resp, err := jsonRequest(self.client, "POST", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, body, self.Debug) + resp, err := jsonRequest(self.client, "POST", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, params, body, self.debug) if err != nil { return "", err } @@ -251,7 +271,7 @@ func (self *SGoogleClient) ecsInsert(resource string, body jsonutils.JSONObject, } } } - resp, err := jsonRequest(self.client, "POST", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, nil, body, self.Debug) + resp, err := jsonRequest(self.client, "POST", GOOGLE_COMPUTE_DOMAIN, GOOGLE_API_VERSION, resource, nil, body, self.debug) if err != nil { return err } @@ -262,7 +282,7 @@ func (self *SGoogleClient) ecsInsert(resource string, body jsonutils.JSONObject, } func (self *SGoogleClient) storageInsert(resource string, body jsonutils.JSONObject, retval interface{}) error { - resp, err := jsonRequest(self.client, "POST", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, nil, body, self.Debug) + resp, err := jsonRequest(self.client, "POST", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, nil, body, self.debug) if err != nil { return err } @@ -273,11 +293,11 @@ func (self *SGoogleClient) storageInsert(resource string, body jsonutils.JSONObj } func (self *SGoogleClient) storageUpload(resource string, header http.Header, body io.Reader) error { - return rawRequest(self.client, "POST", GOOGLE_STORAGE_UPLOAD_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, header, body, self.Debug) + return rawRequest(self.client, "POST", GOOGLE_STORAGE_UPLOAD_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, header, body, self.debug) } func (self *SGoogleClient) storageList(resource string, params map[string]string) (jsonutils.JSONObject, error) { - return jsonRequest(self.client, "GET", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, params, nil, self.Debug) + return jsonRequest(self.client, "GET", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, params, nil, self.debug) } func (self *SGoogleClient) storageListAll(resource string, params map[string]string, retval interface{}) error { @@ -309,7 +329,7 @@ func (self *SGoogleClient) storageListAll(resource string, params map[string]str } func (self *SGoogleClient) storageGet(resource string, retval interface{}) error { - resp, err := jsonRequest(self.client, "GET", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, nil, nil, self.Debug) + resp, err := jsonRequest(self.client, "GET", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, nil, nil, self.debug) if err != nil { return err } @@ -323,7 +343,7 @@ func (self *SGoogleClient) storageGet(resource string, retval interface{}) error } func (self *SGoogleClient) storageDelete(id string, retval interface{}) error { - resp, err := jsonRequest(self.client, "DELETE", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, id, nil, nil, self.Debug) + resp, err := jsonRequest(self.client, "DELETE", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, id, nil, nil, self.debug) if err != nil { return err } @@ -335,7 +355,7 @@ func (self *SGoogleClient) storageDelete(id string, retval interface{}) error { func (self *SGoogleClient) storageDo(resource string, action string, params map[string]string, body jsonutils.JSONObject) (string, error) { resource = fmt.Sprintf("%s/%s", resource, action) - resp, err := jsonRequest(self.client, "POST", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, params, body, self.Debug) + resp, err := jsonRequest(self.client, "POST", GOOGLE_STORAGE_DOMAIN, GOOGLE_STORAGE_API_VERSION, resource, params, body, self.debug) if err != nil { return "", err } @@ -344,7 +364,7 @@ func (self *SGoogleClient) storageDo(resource string, action string, params map[ } func (self *SGoogleClient) cloudbuildGet(resource string, retval interface{}) error { - resp, err := jsonRequest(self.client, "GET", GOOGLE_CLOUDBUILD_DOMAIN, GOOGLE_CLOUDBUILD_API_VERSION, resource, nil, nil, self.Debug) + resp, err := jsonRequest(self.client, "GET", GOOGLE_CLOUDBUILD_DOMAIN, GOOGLE_CLOUDBUILD_API_VERSION, resource, nil, nil, self.debug) if err != nil { return err } @@ -358,7 +378,7 @@ func (self *SGoogleClient) cloudbuildGet(resource string, retval interface{}) er } func (self *SGoogleClient) cloudbuildInsert(resource string, body jsonutils.JSONObject, retval interface{}) error { - resp, err := jsonRequest(self.client, "POST", GOOGLE_CLOUDBUILD_DOMAIN, GOOGLE_CLOUDBUILD_API_VERSION, resource, nil, body, self.Debug) + resp, err := jsonRequest(self.client, "POST", GOOGLE_CLOUDBUILD_DOMAIN, GOOGLE_CLOUDBUILD_API_VERSION, resource, nil, body, self.debug) if err != nil { return err } @@ -369,7 +389,7 @@ func (self *SGoogleClient) cloudbuildInsert(resource string, body jsonutils.JSON } func (self *SGoogleClient) billingList(resource string, params map[string]string) (jsonutils.JSONObject, error) { - return jsonRequest(self.client, "GET", GOOGLE_BILLING_DOMAIN, GOOGLE_BILLING_API_VERSION, resource, params, nil, self.Debug) + return jsonRequest(self.client, "GET", GOOGLE_BILLING_DOMAIN, GOOGLE_BILLING_API_VERSION, resource, params, nil, self.debug) } func (self *SGoogleClient) billingListAll(resource string, params map[string]string, retval interface{}) error { @@ -401,7 +421,7 @@ func (self *SGoogleClient) billingListAll(resource string, params map[string]str } func (self *SGoogleClient) monitorList(resource string, params map[string]string) (jsonutils.JSONObject, error) { - return jsonRequest(self.client, "GET", GOOGLE_MONITOR_DOMAIN, GOOGLE_MONITOR_API_VERSION, resource, params, nil, self.Debug) + return jsonRequest(self.client, "GET", GOOGLE_MONITOR_DOMAIN, GOOGLE_MONITOR_API_VERSION, resource, params, nil, self.debug) } func (self *SGoogleClient) monitorListAll(resource string, params map[string]string, retval interface{}) error { @@ -500,7 +520,7 @@ func (client *SGoogleClient) GetSubAccounts() ([]cloudprovider.SSubAccount, erro accounts := []cloudprovider.SSubAccount{} for _, project := range projects { subAccount := cloudprovider.SSubAccount{} - subAccount.Name = client.providerName + subAccount.Name = client.cpcfg.Name subAccount.Account = fmt.Sprintf("%s/%s", project.ProjectId, client.clientEmail) if project.LifecycleState == "ACTIVE" { subAccount.HealthStatus = api.CLOUD_PROVIDER_HEALTH_NORMAL diff --git a/pkg/multicloud/google/host.go b/pkg/multicloud/google/host.go index dc77e01678..dd5b5e46a8 100644 --- a/pkg/multicloud/google/host.go +++ b/pkg/multicloud/google/host.go @@ -44,7 +44,7 @@ func (host *SHost) GetGlobalId() string { } func (host *SHost) GetName() string { - return fmt.Sprintf("%s-%s", host.zone.region.client.providerName, host.zone.GetName()) + return fmt.Sprintf("%s-%s", host.zone.region.client.cpcfg.Name, host.zone.GetName()) } func (host *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { diff --git a/pkg/multicloud/google/provider/provider.go b/pkg/multicloud/google/provider/provider.go index 5b5b07372a..f9d3988ea5 100644 --- a/pkg/multicloud/google/provider/provider.go +++ b/pkg/multicloud/google/provider/provider.go @@ -115,7 +115,12 @@ func (self *SGoogleProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig return nil, fmt.Errorf("Invalid projectID or client email for google cloud %s", cfg.Account) } projectID, clientEmail = accountInfo[0], accountInfo[1] - client, err := google.NewGoogleClient(cfg.Id, cfg.Name, projectID, clientEmail, privateKeyID, privateKey, false) + + client, err := google.NewGoogleClient( + google.NewGoogleClientConfig( + projectID, clientEmail, privateKeyID, privateKey, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/google/storagecache.go b/pkg/multicloud/google/storagecache.go index 30dce73fa4..6f89b84c09 100644 --- a/pkg/multicloud/google/storagecache.go +++ b/pkg/multicloud/google/storagecache.go @@ -45,11 +45,11 @@ func (cache *SStoragecache) GetMetadata() *jsonutils.JSONDict { } func (cache *SStoragecache) GetId() string { - return cache.region.client.providerId + return cache.region.client.cpcfg.Id } func (cache *SStoragecache) GetName() string { - return cache.region.client.providerName + return cache.region.client.cpcfg.Name } func (cache *SStoragecache) GetStatus() string { @@ -61,7 +61,7 @@ func (cache *SStoragecache) Refresh() error { } func (cache *SStoragecache) GetGlobalId() string { - return cache.region.client.providerId + return cache.region.client.cpcfg.Id } func (cache *SStoragecache) IsEmulated() bool { From e57dcee3e48464d1eaa8e8d23210cbb262771a8f Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 17:12:28 +0800 Subject: [PATCH 07/42] google: use cpcfg.ProxyFunc --- pkg/multicloud/google/google.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/multicloud/google/google.go b/pkg/multicloud/google/google.go index cbc3836a59..5c853d34f5 100644 --- a/pkg/multicloud/google/google.go +++ b/pkg/multicloud/google/google.go @@ -126,7 +126,13 @@ func NewGoogleClient(cfg *GoogleClientConfig) (*SGoogleClient, error) { }, TokenURL: google.JWTTokenURL, } - client.client = conf.Client(oauth2.NoContext) + + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, cfg.cpcfg.ProxyFunc) + ctx := context.Background() + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + + client.client = conf.Client(ctx) return &client, client.fetchRegions() } From 374d15b701e37488969fe4e6797391070dce9e4b Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 17:32:05 +0800 Subject: [PATCH 08/42] ctyun: use CtyunClientConfig --- cmd/ctyuncli/main.go | 7 ++- pkg/multicloud/ctyun/ctyun.go | 54 ++++++++++++++++++----- pkg/multicloud/ctyun/host.go | 4 +- pkg/multicloud/ctyun/project.go | 6 ++- pkg/multicloud/ctyun/provider/provider.go | 6 ++- pkg/multicloud/ctyun/storage.go | 4 +- pkg/multicloud/ctyun/storagecache.go | 6 +-- 7 files changed, 64 insertions(+), 23 deletions(-) diff --git a/cmd/ctyuncli/main.go b/cmd/ctyuncli/main.go index 6b188a5db5..12938dc3b3 100644 --- a/cmd/ctyuncli/main.go +++ b/cmd/ctyuncli/main.go @@ -84,8 +84,11 @@ func newClient(options *BaseOptions) (*ctyun.SRegion, error) { return nil, fmt.Errorf("Missing secret") } - cli, err := ctyun.NewSCtyunClient("", "", "", - options.AccessKey, options.Secret, options.Debug) + cli, err := ctyun.NewSCtyunClient( + ctyun.NewSCtyunClientConfig( + options.AccessKey, options.Secret, + ).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/ctyun/ctyun.go b/pkg/multicloud/ctyun/ctyun.go index ce120904c6..d1dcdecf44 100644 --- a/pkg/multicloud/ctyun/ctyun.go +++ b/pkg/multicloud/ctyun/ctyun.go @@ -44,21 +44,51 @@ const ( CTYUN_API_VERSION = "2019-11-22" ) -type SCtyunClient struct { - httpClient *http.Client - debug bool +type CtyunClientConfig struct { + cpcfg cloudprovider.ProviderConfig - providerId string - providerName string - projectId string // 项目ID. + projectId string accessKey string - secret string + accessSecret string - iregions []cloudprovider.ICloudRegion + debug bool } -func NewSCtyunClient(providerId string, providerName string, projectId string, accessKey string, secret string, debug bool) (*SCtyunClient, error) { - client := &SCtyunClient{httpClient: http.DefaultClient, providerId: providerId, providerName: providerName, projectId: projectId, accessKey: accessKey, secret: secret, debug: debug} +func NewSCtyunClientConfig(accessKey, accessSecret string) *CtyunClientConfig { + cfg := &CtyunClientConfig{ + accessKey: accessKey, + accessSecret: accessSecret, + } + return cfg +} + +func (cfg *CtyunClientConfig) ProjectId(projectId string) *CtyunClientConfig { + cfg.projectId = projectId + return cfg +} + +func (cfg *CtyunClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *CtyunClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *CtyunClientConfig) Debug(debug bool) *CtyunClientConfig { + cfg.debug = debug + return cfg +} + +type SCtyunClient struct { + *CtyunClientConfig + + httpClient *http.Client + iregions []cloudprovider.ICloudRegion +} + +func NewSCtyunClient(cfg *CtyunClientConfig) (*SCtyunClient, error) { + client := &SCtyunClient{ + CtyunClientConfig: cfg, + httpClient: http.DefaultClient, + } err := client.init() if err != nil { @@ -156,7 +186,7 @@ func formRequest(client *SCtyunClient, method httputils.THttpMethod, apiName str // EEE, d MMM yyyy HH:mm:ss z // Mon, 2 Jan 2006 15:04:05 MST requestDate := time.Now().Format("Mon, 2 Jan 2006 15:04:05 MST") - hashMac := hmac.New(sha1.New, []byte(client.secret)) + hashMac := hmac.New(sha1.New, []byte(client.accessSecret)) hashRawString := strings.Join([]string{contentMd5, requestDate, apiName}, "\n") hashMac.Write([]byte(hashRawString)) hsum := base64.StdEncoding.EncodeToString(hashMac.Sum(nil)) @@ -242,7 +272,7 @@ func (self *SCtyunClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) iregion := self.iregions[i] s := cloudprovider.SSubAccount{ - Name: fmt.Sprintf("%s-%s", self.providerName, iregion.GetId()), + Name: fmt.Sprintf("%s-%s", self.cpcfg.Name, iregion.GetId()), State: api.CLOUD_PROVIDER_CONNECTED, Account: fmt.Sprintf("%s/%s", self.accessKey, iregion.GetId()), HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL, diff --git a/pkg/multicloud/ctyun/host.go b/pkg/multicloud/ctyun/host.go index b1ebbd21ec..3bbaf1eb1d 100644 --- a/pkg/multicloud/ctyun/host.go +++ b/pkg/multicloud/ctyun/host.go @@ -35,11 +35,11 @@ type SHost struct { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) } func (self *SHost) GetGlobalId() string { diff --git a/pkg/multicloud/ctyun/project.go b/pkg/multicloud/ctyun/project.go index 2b89be2cd9..5e981642b4 100644 --- a/pkg/multicloud/ctyun/project.go +++ b/pkg/multicloud/ctyun/project.go @@ -43,7 +43,11 @@ func (self *SProject) GetHealthStatus() string { } func (self *SCtyunClient) FetchProjects() ([]SProject, error) { - client, err := NewSCtyunClient("", "", "", self.accessKey, self.secret, self.debug) + client, err := NewSCtyunClient( + NewSCtyunClientConfig( + self.accessKey, self.accessSecret, + ).Debug(self.debug), + ) if err != nil { return nil, errors.Wrap(err, "CtyunClient.FetchProjects") } diff --git a/pkg/multicloud/ctyun/provider/provider.go b/pkg/multicloud/ctyun/provider/provider.go index 7ad6fb5287..4adeaa2b93 100644 --- a/pkg/multicloud/ctyun/provider/provider.go +++ b/pkg/multicloud/ctyun/provider/provider.go @@ -85,7 +85,11 @@ func (self *SCtyunProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) account = segs[0] } - client, err := ctyun.NewSCtyunClient(cfg.Id, cfg.Name, projectId, account, cfg.Secret, false) + client, err := ctyun.NewSCtyunClient( + ctyun.NewSCtyunClientConfig( + account, cfg.Secret, + ).ProjectId(projectId).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/ctyun/storage.go b/pkg/multicloud/ctyun/storage.go index 4856f62bcf..8cde883245 100644 --- a/pkg/multicloud/ctyun/storage.go +++ b/pkg/multicloud/ctyun/storage.go @@ -38,11 +38,11 @@ type SStorage struct { } func (self *SStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) } func (self *SStorage) GetGlobalId() string { diff --git a/pkg/multicloud/ctyun/storagecache.go b/pkg/multicloud/ctyun/storagecache.go index c00b2c613c..35ea132d5d 100644 --- a/pkg/multicloud/ctyun/storagecache.go +++ b/pkg/multicloud/ctyun/storagecache.go @@ -77,15 +77,15 @@ func (self *SStoragecache) fetchImages() error { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) GetStatus() string { From 5ab5fbdf9288ac425920df3d2f89642ccbe069c7 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 17:33:45 +0800 Subject: [PATCH 09/42] ctyun: use cpcfg.ProxyFunc --- pkg/multicloud/ctyun/ctyun.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/multicloud/ctyun/ctyun.go b/pkg/multicloud/ctyun/ctyun.go index d1dcdecf44..5edfc32970 100644 --- a/pkg/multicloud/ctyun/ctyun.go +++ b/pkg/multicloud/ctyun/ctyun.go @@ -85,9 +85,11 @@ type SCtyunClient struct { } func NewSCtyunClient(cfg *CtyunClientConfig) (*SCtyunClient, error) { + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, cfg.cpcfg.ProxyFunc) client := &SCtyunClient{ CtyunClientConfig: cfg, - httpClient: http.DefaultClient, + httpClient: httpClient, } err := client.init() From 58f6f37891b6ead411c20f3042ba7f523f70d877 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 18:05:36 +0800 Subject: [PATCH 10/42] aws: use AwsClientConfig --- cmd/awscli/main.go | 8 +++- pkg/multicloud/aws/aws.go | 55 ++++++++++++++++--------- pkg/multicloud/aws/host.go | 6 +-- pkg/multicloud/aws/provider/provider.go | 6 ++- pkg/multicloud/aws/storage.go | 6 +-- pkg/multicloud/aws/storagecache.go | 6 +-- 6 files changed, 57 insertions(+), 30 deletions(-) diff --git a/cmd/awscli/main.go b/cmd/awscli/main.go index d811f0b72d..1217f56dd8 100644 --- a/cmd/awscli/main.go +++ b/cmd/awscli/main.go @@ -85,7 +85,13 @@ func newClient(options *BaseOptions) (*aws.SRegion, error) { return nil, fmt.Errorf("Missing secret") } - cli, err := aws.NewAwsClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug) + cli, err := aws.NewAwsClient( + aws.NewAwsClientConfig( + options.AccessUrl, + options.AccessKey, + options.Secret, + ).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/aws/aws.go b/pkg/multicloud/aws/aws.go index b6122a206c..e6ec528c44 100644 --- a/pkg/multicloud/aws/aws.go +++ b/pkg/multicloud/aws/aws.go @@ -47,32 +47,49 @@ var ( DEBUG = false ) -type SAwsClient struct { - providerId string - providerName string +type AwsClientConfig struct { + cpcfg cloudprovider.ProviderConfig + accessUrl string // 服务区域 ChinaCloud | InternationalCloud accessKey string - secret string + accessSecret string + + debug bool +} + +func NewAwsClientConfig(accessUrl, accessKey, accessSecret string) *AwsClientConfig { + cfg := &AwsClientConfig{ + accessUrl: accessUrl, + accessKey: accessKey, + accessSecret: accessSecret, + } + return cfg +} + +func (cfg *AwsClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *AwsClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *AwsClientConfig) Debug(debug bool) *AwsClientConfig { + cfg.debug = debug + return cfg +} + +type SAwsClient struct { + *AwsClientConfig ownerId string ownerName string iregions []cloudprovider.ICloudRegion iBuckets []cloudprovider.ICloudBucket - - debug bool } -func NewAwsClient(providerId string, providerName string, accessUrl string, accessKey string, secret string, debug bool) (*SAwsClient, error) { +func NewAwsClient(cfg *AwsClientConfig) (*SAwsClient, error) { client := SAwsClient{ - providerId: providerId, - providerName: providerName, - accessUrl: accessUrl, - accessKey: accessKey, - secret: secret, - debug: debug, + AwsClientConfig: cfg, } - DEBUG = debug err := client.fetchRegions() if err != nil { return nil, errors.Wrap(err, "fetchRegions") @@ -81,7 +98,7 @@ func NewAwsClient(providerId string, providerName string, accessUrl string, acce if err != nil { return nil, errors.Wrap(err, "fetchBuckets") } - if debug { + if client.debug { log.Debugf("ownerId: %s ownerName: %s", client.ownerId, client.ownerName) } return &client, nil @@ -114,7 +131,7 @@ func (self *SAwsClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { return nil, err } subAccount := cloudprovider.SSubAccount{} - subAccount.Name = self.providerName + subAccount.Name = self.cpcfg.Name subAccount.Account = self.accessKey subAccount.HealthStatus = api.CLOUD_PROVIDER_HEALTH_NORMAL return []cloudprovider.SSubAccount{subAccount}, nil @@ -125,9 +142,9 @@ func (client *SAwsClient) GetAccountId() string { } func (self *SAwsClient) UpdateAccount(accessKey, secret string) error { - if self.accessKey != accessKey || self.secret != secret { + if self.accessKey != accessKey || self.accessSecret != secret { self.accessKey = accessKey - self.secret = secret + self.accessSecret = secret self.iregions = nil return self.fetchRegions() } else { @@ -175,7 +192,7 @@ func (client *SAwsClient) getAwsSession(regionId string) (*session.Session, erro chainVerboseErrors := true return session.NewSession(&sdk.Config{ Region: sdk.String(regionId), - Credentials: credentials.NewStaticCredentials(client.accessKey, client.secret, ""), + Credentials: credentials.NewStaticCredentials(client.accessKey, client.accessSecret, ""), DisableParamValidation: &disableParamValidation, CredentialsChainVerboseErrors: &chainVerboseErrors, diff --git a/pkg/multicloud/aws/host.go b/pkg/multicloud/aws/host.go index 4a9ef6e7ad..d2977acefa 100644 --- a/pkg/multicloud/aws/host.go +++ b/pkg/multicloud/aws/host.go @@ -34,15 +34,15 @@ type SHost struct { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) } func (self *SHost) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetStatus() string { diff --git a/pkg/multicloud/aws/provider/provider.go b/pkg/multicloud/aws/provider/provider.go index bd63ccec9e..69f4e05c97 100644 --- a/pkg/multicloud/aws/provider/provider.go +++ b/pkg/multicloud/aws/provider/provider.go @@ -76,7 +76,11 @@ func (self *SAwsProviderFactory) ValidateUpdateCloudaccountCredential(ctx contex } func (self *SAwsProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { - client, err := aws.NewAwsClient(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) + client, err := aws.NewAwsClient( + aws.NewAwsClientConfig( + cfg.URL, cfg.Account, cfg.Secret, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/aws/storage.go b/pkg/multicloud/aws/storage.go index 01a81b405e..9a9b3572bf 100644 --- a/pkg/multicloud/aws/storage.go +++ b/pkg/multicloud/aws/storage.go @@ -30,15 +30,15 @@ type SStorage struct { } func (self *SStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), self.storageType) } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) } func (self *SStorage) GetGlobalId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) } func (self *SStorage) GetStatus() string { diff --git a/pkg/multicloud/aws/storagecache.go b/pkg/multicloud/aws/storagecache.go index 7090a421cc..59866d8ee9 100644 --- a/pkg/multicloud/aws/storagecache.go +++ b/pkg/multicloud/aws/storagecache.go @@ -43,15 +43,15 @@ type SStoragecache struct { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) GetStatus() string { From cfe5b990f9c9b4b59be47bc84b10a0fc7027dfff Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 18:09:57 +0800 Subject: [PATCH 11/42] aws: use cpcfg.ProxyFunc --- pkg/multicloud/aws/aws.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/pkg/multicloud/aws/aws.go b/pkg/multicloud/aws/aws.go index e6ec528c44..41d3d421e8 100644 --- a/pkg/multicloud/aws/aws.go +++ b/pkg/multicloud/aws/aws.go @@ -29,6 +29,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -188,14 +189,16 @@ func (self *SAwsClient) fetchRegions() error { } func (client *SAwsClient) getAwsSession(regionId string) (*session.Session, error) { - disableParamValidation := true - chainVerboseErrors := true + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, client.cpcfg.ProxyFunc) return session.NewSession(&sdk.Config{ - Region: sdk.String(regionId), - Credentials: credentials.NewStaticCredentials(client.accessKey, client.accessSecret, ""), - DisableParamValidation: &disableParamValidation, - - CredentialsChainVerboseErrors: &chainVerboseErrors, + Region: sdk.String(regionId), + Credentials: credentials.NewStaticCredentials( + client.accessKey, client.accessSecret, "", + ), + HTTPClient: httpClient, + DisableParamValidation: sdk.Bool(true), + CredentialsChainVerboseErrors: sdk.Bool(true), }) } From 23bc58240797ab36ac0e2f79d894506ca2ed2f1f Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 18:23:06 +0800 Subject: [PATCH 12/42] azure: use AzureClientConfig --- cmd/azurecli/main.go | 16 +++-- pkg/multicloud/azure/azure.go | 74 +++++++++++++++-------- pkg/multicloud/azure/classic_host.go | 2 +- pkg/multicloud/azure/classic_wire.go | 2 +- pkg/multicloud/azure/host.go | 2 +- pkg/multicloud/azure/provider/provider.go | 6 +- pkg/multicloud/azure/storage.go | 2 +- pkg/multicloud/azure/storagecache.go | 6 +- pkg/multicloud/azure/wire.go | 2 +- pkg/multicloud/azure/zone.go | 2 +- 10 files changed, 75 insertions(+), 39 deletions(-) diff --git a/cmd/azurecli/main.go b/cmd/azurecli/main.go index 6303e2e9ef..4c9f4a3dad 100644 --- a/cmd/azurecli/main.go +++ b/cmd/azurecli/main.go @@ -100,11 +100,17 @@ func newClient(options *BaseOptions) (*azure.SRegion, error) { return nil, fmt.Errorf("Missing Cloud Environment") } - cli, err := azure.NewAzureClient("", "", options.CloudEnv, - options.DirectoryID, - options.ApplicationID, options.ApplicationKey, - options.SubscriptionID, - options.Debug) + cli, err := azure.NewAzureClient( + azure.NewAzureClientConfig( + options.CloudEnv, + options.DirectoryID, + options.ApplicationID, + options.ApplicationKey, + ). + SubscriptionId(options.SubscriptionID). + Debug(options.Debug), + ) + if err != nil { return nil, err } diff --git a/pkg/multicloud/azure/azure.go b/pkg/multicloud/azure/azure.go index 7b1716bb5e..eb622dd9cd 100644 --- a/pkg/multicloud/azure/azure.go +++ b/pkg/multicloud/azure/azure.go @@ -43,17 +43,12 @@ const ( ) type SAzureClient struct { - client autorest.Client - providerId string - providerName string - subscriptionId string - tenantId string - clientId string - clientScret string - domain string - baseUrl string - // secret string - envName string + *AzureClientConfig + + client autorest.Client + domain string + baseUrl string + ressourceGroups []SResourceGroup fetchResourceGroups bool env azureenv.Environment @@ -91,22 +86,53 @@ var DEFAULT_API_VERSION = map[string]string{ "microsoft.insights/eventtypes/management/values": "2017-03-01-preview", } -func NewAzureClient(providerId string, providerName string, envName, tenantId, clientId, clientSecret, subscriptionId string, debug bool) (*SAzureClient, error) { +type AzureClientConfig struct { + cpcfg cloudprovider.ProviderConfig + + envName string + tenantId string + clientId string + clientSecret string + + subscriptionId string + + debug bool +} + +func NewAzureClientConfig(envName, tenantId, clientId, clientSecret string) *AzureClientConfig { + cfg := &AzureClientConfig{ + envName: envName, + tenantId: tenantId, + clientId: clientId, + clientSecret: clientSecret, + } + return cfg +} + +func (cfg *AzureClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *AzureClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *AzureClientConfig) SubscriptionId(id string) *AzureClientConfig { + cfg.subscriptionId = id + return cfg +} + +func (cfg *AzureClientConfig) Debug(debug bool) *AzureClientConfig { + cfg.debug = debug + return cfg +} + +func NewAzureClient(cfg *AzureClientConfig) (*SAzureClient, error) { client := SAzureClient{ - providerId: providerId, - providerName: providerName, - envName: envName, - tenantId: tenantId, - clientId: clientId, - clientScret: clientSecret, - subscriptionId: subscriptionId, - debug: debug, + AzureClientConfig: cfg, } err := client.fetchRegions() if err != nil { return nil, errors.Wrap(err, "fetchRegions") } - if len(subscriptionId) > 0 { + if len(cfg.subscriptionId) > 0 { err = client.fetchBuckets() if err != nil { return nil, errors.Wrap(err, "fetchBuckets") @@ -117,7 +143,7 @@ func NewAzureClient(providerId string, providerName string, envName, tenantId, c func (self *SAzureClient) getDefaultClient() (*autorest.Client, error) { client := autorest.NewClientWithUserAgent("Yunion API") - conf := auth.NewClientCredentialsConfig(self.clientId, self.clientScret, self.tenantId) + conf := auth.NewClientCredentialsConfig(self.clientId, self.clientSecret, self.tenantId) env, err := azureenv.EnvironmentFromName(self.envName) if err != nil { return nil, err @@ -730,7 +756,7 @@ func _jsonRequest(client *autorest.Client, method, domain, baseURL, body string) /*func (self *SAzureClient) UpdateAccount(envName, tenantId, appId, appKey, subscriptionId string) error { if self.tenantId != tenantId || self.secret != secret || self.envName != envName { if clientInfo, accountInfo := strings.Split(secret, "/"), strings.Split(tenantId, "/"); len(clientInfo) >= 2 && len(accountInfo) >= 1 { - self.clientId, self.clientScret = clientInfo[0], strings.Join(clientInfo[1:], "/") + self.clientId, self.clientSecret = clientInfo[0], strings.Join(clientInfo[1:], "/") self.tenantId = accountInfo[0] if len(accountInfo) == 2 { self.subscriptionId = accountInfo[1] @@ -741,7 +767,7 @@ func _jsonRequest(client *autorest.Client, method, domain, baseURL, body string) } return nil } else { - return httperrors.NewUnauthorizedError("clientId、clientScret or subscriptId input error") + return httperrors.NewUnauthorizedError("clientId、clientSecret or subscriptId input error") } } return nil diff --git a/pkg/multicloud/azure/classic_host.go b/pkg/multicloud/azure/classic_host.go index 26b742246b..1b8f4d7b11 100644 --- a/pkg/multicloud/azure/classic_host.go +++ b/pkg/multicloud/azure/classic_host.go @@ -35,7 +35,7 @@ func (self *SClassicHost) GetMetadata() *jsonutils.JSONDict { } func (self *SClassicHost) GetId() string { - return fmt.Sprintf("%s-%s-classic", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s-classic", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SClassicHost) GetName() string { diff --git a/pkg/multicloud/azure/classic_wire.go b/pkg/multicloud/azure/classic_wire.go index bf37c922b6..7a7c2f5f69 100644 --- a/pkg/multicloud/azure/classic_wire.go +++ b/pkg/multicloud/azure/classic_wire.go @@ -43,7 +43,7 @@ func (self *SClassicWire) GetGlobalId() string { } func (self *SClassicWire) GetName() string { - return fmt.Sprintf("%s-%s-classic", self.zone.region.client.providerName, self.vpc.GetName()) + return fmt.Sprintf("%s-%s-classic", self.zone.region.client.cpcfg.Name, self.vpc.GetName()) } func (self *SClassicWire) IsEmulated() bool { diff --git a/pkg/multicloud/azure/host.go b/pkg/multicloud/azure/host.go index 63d352daae..ad8bf8cef0 100644 --- a/pkg/multicloud/azure/host.go +++ b/pkg/multicloud/azure/host.go @@ -40,7 +40,7 @@ func (self *SHost) GetMetadata() *jsonutils.JSONDict { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { diff --git a/pkg/multicloud/azure/provider/provider.go b/pkg/multicloud/azure/provider/provider.go index eceab6daea..5d9eda1b65 100644 --- a/pkg/multicloud/azure/provider/provider.go +++ b/pkg/multicloud/azure/provider/provider.go @@ -113,7 +113,11 @@ func parseAccount(account, secret string) (tenantId string, appId string, appKey func (self *SAzureProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { tenantId, appId, appKey, subId := parseAccount(cfg.Account, cfg.Secret) - if client, err := azure.NewAzureClient(cfg.Id, cfg.Name, cfg.URL, tenantId, appId, appKey, subId, false); err != nil { + if client, err := azure.NewAzureClient( + azure.NewAzureClientConfig( + cfg.URL, tenantId, appId, appKey, + ).SubscriptionId(subId).CloudproviderConfig(cfg), + ); err != nil { return nil, err } else { return &SAzureProvider{ diff --git a/pkg/multicloud/azure/storage.go b/pkg/multicloud/azure/storage.go index 2821bb0b1a..f8856fcce0 100644 --- a/pkg/multicloud/azure/storage.go +++ b/pkg/multicloud/azure/storage.go @@ -52,7 +52,7 @@ func (self *SStorage) GetId() string { } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, strings.ToLower(self.storageType)) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, strings.ToLower(self.storageType)) } func (self *SStorage) GetGlobalId() string { diff --git a/pkg/multicloud/azure/storagecache.go b/pkg/multicloud/azure/storagecache.go index 15f53eb6de..83a67b1411 100644 --- a/pkg/multicloud/azure/storagecache.go +++ b/pkg/multicloud/azure/storagecache.go @@ -54,11 +54,11 @@ func (self *SStoragecache) GetMetadata() *jsonutils.JSONDict { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetStatus() string { @@ -70,7 +70,7 @@ func (self *SStoragecache) Refresh() error { } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) IsEmulated() bool { diff --git a/pkg/multicloud/azure/wire.go b/pkg/multicloud/azure/wire.go index f2823864e5..0eb5e28f4a 100644 --- a/pkg/multicloud/azure/wire.go +++ b/pkg/multicloud/azure/wire.go @@ -43,7 +43,7 @@ func (self *SWire) GetGlobalId() string { } func (self *SWire) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.vpc.GetName()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.vpc.GetName()) } func (self *SWire) IsEmulated() bool { diff --git a/pkg/multicloud/azure/zone.go b/pkg/multicloud/azure/zone.go index f00d8722a5..975fc7dc2e 100644 --- a/pkg/multicloud/azure/zone.go +++ b/pkg/multicloud/azure/zone.go @@ -43,7 +43,7 @@ func (self *SZone) GetMetadata() *jsonutils.JSONDict { } func (self *SZone) GetId() string { - return self.region.client.providerId + return self.region.client.cpcfg.Id } func (self *SZone) GetName() string { From a1575969737eb625ac6743c06148207184bb8560 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 18:42:04 +0800 Subject: [PATCH 13/42] azure: use cpcfg.ProxyFunc --- pkg/multicloud/azure/azure.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/multicloud/azure/azure.go b/pkg/multicloud/azure/azure.go index eb622dd9cd..48b1906358 100644 --- a/pkg/multicloud/azure/azure.go +++ b/pkg/multicloud/azure/azure.go @@ -33,6 +33,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -148,19 +149,28 @@ func (self *SAzureClient) getDefaultClient() (*autorest.Client, error) { if err != nil { return nil, err } + + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, self.cpcfg.ProxyFunc) + client.Sender = httpClient + self.env = env self.domain = env.ResourceManagerEndpoint conf.Resource = env.ResourceManagerEndpoint conf.AADEndpoint = env.ActiveDirectoryEndpoint - authorizer, err := conf.Authorizer() - if err != nil { - return nil, err + { + spt, err := conf.ServicePrincipalToken() + if err != nil { + return nil, err + } + spt.SetSender(httpClient) + client.Authorizer = autorest.NewBearerAuthorizer(spt) } - client.Authorizer = authorizer if self.debug { client.RequestInspector = LogRequest() client.ResponseInspector = LogResponse() } + return &client, nil } From 292ca54153956aa94ee0e8ff5601d1333b6fcd31 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 18:47:48 +0800 Subject: [PATCH 14/42] huawei: use HuaweiClientConfig --- cmd/huaweicli/main.go | 11 ++-- pkg/multicloud/huawei/domain.go | 2 +- pkg/multicloud/huawei/host.go | 6 +-- pkg/multicloud/huawei/huawei.go | 60 ++++++++++++++-------- pkg/multicloud/huawei/project.go | 2 +- pkg/multicloud/huawei/provider/provider.go | 6 ++- pkg/multicloud/huawei/region.go | 2 +- pkg/multicloud/huawei/storage.go | 6 +-- pkg/multicloud/huawei/storagecache.go | 6 +-- 9 files changed, 64 insertions(+), 37 deletions(-) diff --git a/cmd/huaweicli/main.go b/cmd/huaweicli/main.go index 9c7dee4de6..ba0778920f 100644 --- a/cmd/huaweicli/main.go +++ b/cmd/huaweicli/main.go @@ -86,9 +86,14 @@ func newClient(options *BaseOptions) (*huawei.SRegion, error) { return nil, fmt.Errorf("Missing secret") } - cli, err := huawei.NewHuaweiClient("", "", options.CloudEnv, - options.AccessKey, options.Secret, options.ProjectId, - options.Debug) + cli, err := huawei.NewHuaweiClient( + huawei.NewHuaweiClientConfig( + options.CloudEnv, + options.AccessKey, + options.Secret, + options.ProjectId, + ).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/huawei/domain.go b/pkg/multicloud/huawei/domain.go index a2cf083d03..eb533e4bab 100644 --- a/pkg/multicloud/huawei/domain.go +++ b/pkg/multicloud/huawei/domain.go @@ -29,7 +29,7 @@ type SDomain struct { } func (self *SHuaweiClient) GetDomains() ([]SDomain, error) { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret, self.debug) + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) domains := make([]SDomain, 0) err := doListAll(huawei.Domains.List, nil, &domains) return domains, err diff --git a/pkg/multicloud/huawei/host.go b/pkg/multicloud/huawei/host.go index 99bbf362d6..e1ccb36fd5 100644 --- a/pkg/multicloud/huawei/host.go +++ b/pkg/multicloud/huawei/host.go @@ -37,15 +37,15 @@ type SHost struct { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) } func (self *SHost) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetStatus() string { diff --git a/pkg/multicloud/huawei/huawei.go b/pkg/multicloud/huawei/huawei.go index cd6886676f..06901b5851 100644 --- a/pkg/multicloud/huawei/huawei.go +++ b/pkg/multicloud/huawei/huawei.go @@ -48,17 +48,41 @@ const ( HUAWEI_API_VERSION = "2018-12-25" ) -type SHuaweiClient struct { - signer auth.Signer +type HuaweiClientConfig struct { + cpcfg cloudprovider.ProviderConfig - debug bool - - providerId string - providerName string projectId string // 华为云项目ID. cloudEnv string // 服务区域 ChinaCloud | InternationalCloud accessKey string - secret string + accessSecret string + + debug bool +} + +func NewHuaweiClientConfig(cloudEnv, accessKey, accessSecret, projectId string) *HuaweiClientConfig { + cfg := &HuaweiClientConfig{ + projectId: projectId, + cloudEnv: cloudEnv, + accessKey: accessKey, + accessSecret: accessSecret, + } + return cfg +} + +func (cfg *HuaweiClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *HuaweiClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *HuaweiClientConfig) Debug(debug bool) *HuaweiClientConfig { + cfg.debug = debug + return cfg +} + +type SHuaweiClient struct { + *HuaweiClientConfig + + signer auth.Signer isMainProject bool // whether the project is the main project in the region @@ -73,15 +97,9 @@ type SHuaweiClient struct { // 初次导入Subaccount时，参数account对应cloudaccounts表中的account字段，即accesskey。此时projectID为空， // 只能进行同步子账号、查询region列表等projectId无关的操作。 // todo: 通过accessurl支持国际站。目前暂时未支持国际站 -func NewHuaweiClient(providerId, providerName, cloudEnv, accessKey, secret, projectId string, debug bool) (*SHuaweiClient, error) { +func NewHuaweiClient(cfg *HuaweiClientConfig) (*SHuaweiClient, error) { client := SHuaweiClient{ - providerId: providerId, - providerName: providerName, - projectId: projectId, - cloudEnv: cloudEnv, - accessKey: accessKey, - secret: secret, - debug: debug, + HuaweiClientConfig: cfg, } err := client.init() if err != nil { @@ -120,7 +138,7 @@ func (self *SHuaweiClient) initSigner() error { } func (self *SHuaweiClient) fetchRegions() error { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret, self.debug) + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) regions := make([]SRegion, 0) err := doListAll(huawei.Regions.List, nil, ®ions) if err != nil { @@ -179,7 +197,7 @@ func getOBSEndpoint(regionId string) string { func (client *SHuaweiClient) getOBSClient(regionId string) (*obs.ObsClient, error) { endpoint := getOBSEndpoint(regionId) - return obs.New(client.accessKey, client.secret, endpoint) + return obs.New(client.accessKey, client.accessSecret, endpoint) } func (self *SHuaweiClient) fetchBuckets() error { @@ -225,9 +243,9 @@ func (self *SHuaweiClient) GetCloudRegionExternalIdPrefix() string { } func (self *SHuaweiClient) UpdateAccount(accessKey, secret string) error { - if self.accessKey != accessKey || self.secret != secret { + if self.accessKey != accessKey || self.accessSecret != secret { self.accessKey = accessKey - self.secret = secret + self.accessSecret = secret return self.fetchRegions() } else { return nil @@ -258,7 +276,7 @@ func (self *SHuaweiClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) continue } s := cloudprovider.SSubAccount{ - Name: fmt.Sprintf("%s-%s", self.providerName, project.Name), + Name: fmt.Sprintf("%s-%s", self.cpcfg.Name, project.Name), State: api.CLOUD_PROVIDER_CONNECTED, Account: fmt.Sprintf("%s/%s", self.accessKey, project.ID), HealthStatus: project.GetHealthStatus(), @@ -384,7 +402,7 @@ func (self *SHuaweiClient) QueryAccountBalance() (*SAccountBalance, error) { // https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0075213309.html func (self *SHuaweiClient) queryDomainBalance(domainId string) (float64, error) { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret, self.debug) + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) huawei.Balances.SetDomainId(domainId) balances := make([]SBalance, 0) err := doListAll(huawei.Balances.List, nil, &balances) diff --git a/pkg/multicloud/huawei/project.go b/pkg/multicloud/huawei/project.go index 28053fdf09..bf32d75ac6 100644 --- a/pkg/multicloud/huawei/project.go +++ b/pkg/multicloud/huawei/project.go @@ -49,7 +49,7 @@ func (self *SProject) GetHealthStatus() string { } func (self *SHuaweiClient) fetchProjects() ([]SProject, error) { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.secret, self.debug) + huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) projects := make([]SProject, 0) err := doListAll(huawei.Projects.List, nil, &projects) return projects, err diff --git a/pkg/multicloud/huawei/provider/provider.go b/pkg/multicloud/huawei/provider/provider.go index 3551e03ed6..c8c0b64260 100644 --- a/pkg/multicloud/huawei/provider/provider.go +++ b/pkg/multicloud/huawei/provider/provider.go @@ -101,7 +101,11 @@ func parseAccount(account string) (accessKey string, projectId string) { func (self *SHuaweiProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { accessKey, projectId := parseAccount(cfg.Account) - client, err := huawei.NewHuaweiClient(cfg.Id, cfg.Name, cfg.URL, accessKey, cfg.Secret, projectId, false) + client, err := huawei.NewHuaweiClient( + huawei.NewHuaweiClientConfig( + cfg.URL, accessKey, cfg.Secret, projectId, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/huawei/region.go b/pkg/multicloud/huawei/region.go index 59b3cff24c..9aba228e9c 100644 --- a/pkg/multicloud/huawei/region.go +++ b/pkg/multicloud/huawei/region.go @@ -82,7 +82,7 @@ func (self *SRegion) getECSClient() (*client.Client, error) { } if self.ecsClient == nil { - self.ecsClient, err = client.NewClientWithAccessKey(self.ID, self.client.projectId, self.client.accessKey, self.client.secret, self.client.debug) + self.ecsClient, err = client.NewClientWithAccessKey(self.ID, self.client.projectId, self.client.accessKey, self.client.accessSecret, self.client.debug) if err != nil { return nil, err } diff --git a/pkg/multicloud/huawei/storage.go b/pkg/multicloud/huawei/storage.go index 5132c9de4a..2bfa04f8ee 100644 --- a/pkg/multicloud/huawei/storage.go +++ b/pkg/multicloud/huawei/storage.go @@ -31,15 +31,15 @@ type SStorage struct { } func (self *SStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), self.storageType) } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) } func (self *SStorage) GetGlobalId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) } func (self *SStorage) GetStatus() string { diff --git a/pkg/multicloud/huawei/storagecache.go b/pkg/multicloud/huawei/storagecache.go index 5269089db5..fd1d32c6a4 100644 --- a/pkg/multicloud/huawei/storagecache.go +++ b/pkg/multicloud/huawei/storagecache.go @@ -68,15 +68,15 @@ func (self *SStoragecache) fetchImages() error { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) GetStatus() string { From 608a18eed4d8f6b6ff89a7586cd4f3c09ac32fc4 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:11:29 +0800 Subject: [PATCH 15/42] huawei: client: add SBaseManager.SetHttpClient() --- pkg/multicloud/huawei/client/client.go | 59 ++++++++++++++++--- .../huawei/client/modules/manager_base.go | 4 ++ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/pkg/multicloud/huawei/client/client.go b/pkg/multicloud/huawei/client/client.go index b88e6fa5ed..08c005cbce 100644 --- a/pkg/multicloud/huawei/client/client.go +++ b/pkg/multicloud/huawei/client/client.go @@ -15,6 +15,8 @@ package client import ( + "net/http" + "yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth" "yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth/credentials" "yunion.io/x/onecloud/pkg/multicloud/huawei/client/modules" @@ -81,9 +83,55 @@ type Client struct { CloudEye *modules.SCloudEyeManager } -func (self *Client) Init() error { - // 从环境变量中初始化client - return nil +func (self *Client) SetHttpClient(httpClient *http.Client) { + self.Servers.SetHttpClient(httpClient) + self.ServersV2.SetHttpClient(httpClient) + self.NovaServers.SetHttpClient(httpClient) + self.Snapshots.SetHttpClient(httpClient) + self.OsSnapshots.SetHttpClient(httpClient) + self.Images.SetHttpClient(httpClient) + self.OpenStackImages.SetHttpClient(httpClient) + self.Projects.SetHttpClient(httpClient) + self.Regions.SetHttpClient(httpClient) + self.Zones.SetHttpClient(httpClient) + self.Vpcs.SetHttpClient(httpClient) + self.Eips.SetHttpClient(httpClient) + self.Elasticcache.SetHttpClient(httpClient) + self.DcsAvailableZone.SetHttpClient(httpClient) + self.Disks.SetHttpClient(httpClient) + self.Domains.SetHttpClient(httpClient) + self.Keypairs.SetHttpClient(httpClient) + self.Elb.SetHttpClient(httpClient) + self.ElbBackend.SetHttpClient(httpClient) + self.ElbBackendGroup.SetHttpClient(httpClient) + self.ElbListeners.SetHttpClient(httpClient) + self.ElbCertificates.SetHttpClient(httpClient) + self.ElbHealthCheck.SetHttpClient(httpClient) + self.ElbL7policies.SetHttpClient(httpClient) + self.ElbPolicies.SetHttpClient(httpClient) + self.ElbWhitelist.SetHttpClient(httpClient) + self.Orders.SetHttpClient(httpClient) + self.SecurityGroupRules.SetHttpClient(httpClient) + self.SecurityGroups.SetHttpClient(httpClient) + self.NovaSecurityGroups.SetHttpClient(httpClient) + self.Subnets.SetHttpClient(httpClient) + self.Users.SetHttpClient(httpClient) + self.Interface.SetHttpClient(httpClient) + self.Jobs.SetHttpClient(httpClient) + self.Balances.SetHttpClient(httpClient) + self.Bandwidths.SetHttpClient(httpClient) + self.Port.SetHttpClient(httpClient) + self.Flavors.SetHttpClient(httpClient) + self.VpcRoutes.SetHttpClient(httpClient) + self.SNatRules.SetHttpClient(httpClient) + self.DNatRules.SetHttpClient(httpClient) + self.NatGateways.SetHttpClient(httpClient) + self.DBInstance.SetHttpClient(httpClient) + self.DBInstanceBackup.SetHttpClient(httpClient) + self.DBInstanceFlavor.SetHttpClient(httpClient) + self.DBInstanceJob.SetHttpClient(httpClient) + self.Traces.SetHttpClient(httpClient) + self.CloudEye.SetHttpClient(httpClient) } func (self *Client) InitWithOptions(regionId, projectId string, credential auth.Credential) error { @@ -167,11 +215,6 @@ func (self *Client) initManagers() { self.init = true } -// todo: init from envrioment -func NewClient() (*Client, error) { - return nil, nil -} - func NewClientWithAccessKey(regionId, projectId, accessKey, secretKey string, debug bool) (*Client, error) { c := &Client{debug: debug} err := c.InitWithAccessKey(regionId, projectId, accessKey, secretKey) diff --git a/pkg/multicloud/huawei/client/modules/manager_base.go b/pkg/multicloud/huawei/client/modules/manager_base.go index 999522d7bc..0657473a2d 100644 --- a/pkg/multicloud/huawei/client/modules/manager_base.go +++ b/pkg/multicloud/huawei/client/modules/manager_base.go @@ -62,6 +62,10 @@ func (self *SBaseManager) GetColumns() []string { return self.columns } +func (self *SBaseManager) SetHttpClient(httpClient *http.Client) { + self.httpClient = httpClient +} + func (self *SBaseManager) _list(request requests.IRequest, responseKey string) (*responses.ListResult, error) { _, body, err := self.jsonRequest(request) if err != nil { From 991b3bb5cf4b6cb3606e10f393d5fe4493e92d97 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:11:45 +0800 Subject: [PATCH 16/42] huawei: use cpcfg.ProxyFunc --- pkg/multicloud/huawei/domain.go | 4 +--- pkg/multicloud/huawei/huawei.go | 31 +++++++++++++++++++++++++++++-- pkg/multicloud/huawei/project.go | 3 +-- pkg/multicloud/huawei/region.go | 2 +- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pkg/multicloud/huawei/domain.go b/pkg/multicloud/huawei/domain.go index eb533e4bab..2019b875e7 100644 --- a/pkg/multicloud/huawei/domain.go +++ b/pkg/multicloud/huawei/domain.go @@ -14,8 +14,6 @@ package huawei -import "yunion.io/x/onecloud/pkg/multicloud/huawei/client" - // https://support.huaweicloud.com/api-iam/zh-cn_topic_0057845574.html // 租户列表 type SDomain struct { @@ -29,7 +27,7 @@ type SDomain struct { } func (self *SHuaweiClient) GetDomains() ([]SDomain, error) { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) + huawei, _ := self.newGeneralAPIClient() domains := make([]SDomain, 0) err := doListAll(huawei.Domains.List, nil, &domains) return domains, err diff --git a/pkg/multicloud/huawei/huawei.go b/pkg/multicloud/huawei/huawei.go index 06901b5851..fe22228e2a 100644 --- a/pkg/multicloud/huawei/huawei.go +++ b/pkg/multicloud/huawei/huawei.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth" "yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth/credentials" "yunion.io/x/onecloud/pkg/multicloud/huawei/obs" + "yunion.io/x/onecloud/pkg/util/httputils" ) /* @@ -137,8 +138,34 @@ func (self *SHuaweiClient) initSigner() error { return nil } +func (self *SHuaweiClient) newRegionAPIClient(regionId string) (*client.Client, error) { + cli, err := client.NewClientWithAccessKey(regionId, self.projectId, self.accessKey, self.accessSecret, self.debug) + if err != nil { + return nil, err + } + + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, self.cpcfg.ProxyFunc) + cli.SetHttpClient(httpClient) + + return cli, nil +} + +func (self *SHuaweiClient) newGeneralAPIClient() (*client.Client, error) { + cli, err := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) + if err != nil { + return nil, err + } + + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, self.cpcfg.ProxyFunc) + cli.SetHttpClient(httpClient) + + return cli, nil +} + func (self *SHuaweiClient) fetchRegions() error { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) + huawei, _ := self.newGeneralAPIClient() regions := make([]SRegion, 0) err := doListAll(huawei.Regions.List, nil, ®ions) if err != nil { @@ -402,7 +429,7 @@ func (self *SHuaweiClient) QueryAccountBalance() (*SAccountBalance, error) { // https://support.huaweicloud.com/api-bpconsole/zh-cn_topic_0075213309.html func (self *SHuaweiClient) queryDomainBalance(domainId string) (float64, error) { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) + huawei, _ := self.newGeneralAPIClient() huawei.Balances.SetDomainId(domainId) balances := make([]SBalance, 0) err := doListAll(huawei.Balances.List, nil, &balances) diff --git a/pkg/multicloud/huawei/project.go b/pkg/multicloud/huawei/project.go index bf32d75ac6..7d1e6b9f91 100644 --- a/pkg/multicloud/huawei/project.go +++ b/pkg/multicloud/huawei/project.go @@ -20,7 +20,6 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/multicloud/huawei/client" ) // https://support.huaweicloud.com/api-iam/zh-cn_topic_0057845625.html @@ -49,7 +48,7 @@ func (self *SProject) GetHealthStatus() string { } func (self *SHuaweiClient) fetchProjects() ([]SProject, error) { - huawei, _ := client.NewClientWithAccessKey("", "", self.accessKey, self.accessSecret, self.debug) + huawei, _ := self.newGeneralAPIClient() projects := make([]SProject, 0) err := doListAll(huawei.Projects.List, nil, &projects) return projects, err diff --git a/pkg/multicloud/huawei/region.go b/pkg/multicloud/huawei/region.go index 9aba228e9c..efb2709a31 100644 --- a/pkg/multicloud/huawei/region.go +++ b/pkg/multicloud/huawei/region.go @@ -82,7 +82,7 @@ func (self *SRegion) getECSClient() (*client.Client, error) { } if self.ecsClient == nil { - self.ecsClient, err = client.NewClientWithAccessKey(self.ID, self.client.projectId, self.client.accessKey, self.client.accessSecret, self.client.debug) + self.ecsClient, err = self.client.newRegionAPIClient(self.ID) if err != nil { return nil, err } From 0faa2bfb9c0e5e5e4d28df17d09d7bb6b88526bb Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:28:09 +0800 Subject: [PATCH 17/42] vendor: update github.com/tencentcloud/tencentcloud-sdk-go --- go.mod | 2 +- go.sum | 4 +- .../tencentcloud/common/client.go | 227 +++++++++-- .../tencentcloud/common/credentials.go | 52 ++- .../tencentcloud/common/http/request.go | 18 +- .../tencentcloud/common/http/response.go | 20 +- .../common/profile/client_profile.go | 14 +- .../common/profile/http_profile.go | 6 +- .../tencentcloud/common/sign.go | 31 +- .../golang.org/x/net/http/httpproxy/proxy.go | 370 ++++++++++++++++++ vendor/modules.txt | 3 +- 11 files changed, 680 insertions(+), 67 deletions(-) create mode 100644 vendor/golang.org/x/net/http/httpproxy/proxy.go diff --git a/go.mod b/go.mod index 3f58d41269..e7454c0f5b 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.4.0 - github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e + github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20191108095731-8ca4b370cde4 github.com/tinylib/msgp v1.1.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect diff --git a/go.sum b/go.sum index c76c076e7f..e7881bac7e 100644 --- a/go.sum +++ b/go.sum @@ -693,8 +693,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M= github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e h1:CtKVGXKh2bfmepZ/YogAjvL/CrSy9NGZET500K7arf4= -github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20191108095731-8ca4b370cde4 h1:wVKDVTKxaf0ll8ONv22Q+cazYSSmcWt7JJOeQD0LoKQ= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20191108095731-8ca4b370cde4/go.mod h1:wk2XFUg6egk4tSDNZtXeKfe2G6690UVyt163PuUxBZk= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s= diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/client.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/client.go index 78f3ce7c72..81ce1ac238 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/client.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/client.go @@ -1,21 +1,30 @@ package common import ( + "encoding/hex" + "encoding/json" + "fmt" "log" "net/http" + "net/http/httputil" + "strconv" + "strings" "time" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors" tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" ) type Client struct { - region string - httpClient *http.Client - httpProfile *profile.HttpProfile - credential *Credential - signMethod string - debug bool + region string + httpClient *http.Client + httpProfile *profile.HttpProfile + profile *profile.ClientProfile + credential *Credential + signMethod string + unsignedPayload bool + debug bool } func (c *Client) Send(request tchttp.Request, response tchttp.Response) (err error) { @@ -26,29 +35,184 @@ func (c *Client) Send(request tchttp.Request, response tchttp.Response) (err err } request.SetDomain(domain) } - err = tchttp.ConstructParams(request) - if err != nil { - return + + if request.GetHttpMethod() == "" { + request.SetHttpMethod(c.httpProfile.ReqMethod) } + tchttp.CompleteCommonParams(request, c.GetRegion()) - err = signRequest(request, c.credential, c.signMethod) - if err != nil { - return + + if c.signMethod == "HmacSHA1" || c.signMethod == "HmacSHA256" { + return c.sendWithSignatureV1(request, response) + } else { + return c.sendWithSignatureV3(request, response) } - httpRequest, err := http.NewRequest(request.GetHttpMethod(), request.GetUrl(), request.GetBodyReader()) - if err != nil { - return - } - if request.GetHttpMethod() == "POST" { - httpRequest.Header["Content-Type"] = []string{"application/x-www-form-urlencoded"} - } - //log.Printf("[DEBUG] http request=%v", httpRequest) - httpResponse, err := c.httpClient.Do(httpRequest) +} + +func (c *Client) sendWithSignatureV1(request tchttp.Request, response tchttp.Response) (err error) { + // TODO: not an elegant way, it should be done in common params, but finally it need to refactor + request.GetParams()["Language"] = c.profile.Language + err = tchttp.ConstructParams(request) if err != nil { return err } + err = signRequest(request, c.credential, c.signMethod) + if err != nil { + return err + } + httpRequest, err := http.NewRequest(request.GetHttpMethod(), request.GetUrl(), request.GetBodyReader()) + if err != nil { + return err + } + if request.GetHttpMethod() == "POST" { + httpRequest.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + if c.debug { + outbytes, err := httputil.DumpRequest(httpRequest, true) + if err != nil { + log.Printf("[ERROR] dump request failed because %s", err) + return err + } + log.Printf("[DEBUG] http request = %s", outbytes) + } + httpResponse, err := c.httpClient.Do(httpRequest) + if err != nil { + msg := fmt.Sprintf("Fail to get response because %s", err) + return errors.NewTencentCloudSDKError("ClientError.NetworkError", msg, "") + } err = tchttp.ParseFromHttpResponse(httpResponse, response) - return + return err +} + +func (c *Client) sendWithSignatureV3(request tchttp.Request, response tchttp.Response) (err error) { + headers := map[string]string{ + "Host": request.GetDomain(), + "X-TC-Action": request.GetAction(), + "X-TC-Version": request.GetVersion(), + "X-TC-Timestamp": request.GetParams()["Timestamp"], + "X-TC-RequestClient": request.GetParams()["RequestClient"], + "X-TC-Language": c.profile.Language, + } + if c.region != "" { + headers["X-TC-Region"] = c.region + } + if c.credential.Token != "" { + headers["X-TC-Token"] = c.credential.Token + } + if request.GetHttpMethod() == "GET" { + headers["Content-Type"] = "application/x-www-form-urlencoded" + } else { + headers["Content-Type"] = "application/json" + } + + // start signature v3 process + + // build canonical request string + httpRequestMethod := request.GetHttpMethod() + canonicalURI := "/" + canonicalQueryString := "" + if httpRequestMethod == "GET" { + err = tchttp.ConstructParams(request) + if err != nil { + return err + } + params := make(map[string]string) + for key, value := range request.GetParams() { + params[key] = value + } + delete(params, "Action") + delete(params, "Version") + delete(params, "Nonce") + delete(params, "Region") + delete(params, "RequestClient") + delete(params, "Timestamp") + canonicalQueryString = tchttp.GetUrlQueriesEncoded(params) + } + canonicalHeaders := fmt.Sprintf("content-type:%s\nhost:%s\n", headers["Content-Type"], headers["Host"]) + signedHeaders := "content-type;host" + requestPayload := "" + if httpRequestMethod == "POST" { + b, err := json.Marshal(request) + if err != nil { + return err + } + requestPayload = string(b) + } + hashedRequestPayload := "" + if c.unsignedPayload { + hashedRequestPayload = sha256hex("UNSIGNED-PAYLOAD") + headers["X-TC-Content-SHA256"] = "UNSIGNED-PAYLOAD" + } else { + hashedRequestPayload = sha256hex(requestPayload) + } + canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", + httpRequestMethod, + canonicalURI, + canonicalQueryString, + canonicalHeaders, + signedHeaders, + hashedRequestPayload) + //log.Println("canonicalRequest:", canonicalRequest) + + // build string to sign + algorithm := "TC3-HMAC-SHA256" + requestTimestamp := headers["X-TC-Timestamp"] + timestamp, _ := strconv.ParseInt(requestTimestamp, 10, 64) + t := time.Unix(timestamp, 0).UTC() + // must be the format 2006-01-02, ref to package time for more info + date := t.Format("2006-01-02") + credentialScope := fmt.Sprintf("%s/%s/tc3_request", date, request.GetService()) + hashedCanonicalRequest := sha256hex(canonicalRequest) + string2sign := fmt.Sprintf("%s\n%s\n%s\n%s", + algorithm, + requestTimestamp, + credentialScope, + hashedCanonicalRequest) + //log.Println("string2sign", string2sign) + + // sign string + secretDate := hmacsha256(date, "TC3"+c.credential.SecretKey) + secretService := hmacsha256(request.GetService(), secretDate) + secretKey := hmacsha256("tc3_request", secretService) + signature := hex.EncodeToString([]byte(hmacsha256(string2sign, secretKey))) + //log.Println("signature", signature) + + // build authorization + authorization := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", + algorithm, + c.credential.SecretId, + credentialScope, + signedHeaders, + signature) + //log.Println("authorization", authorization) + + headers["Authorization"] = authorization + url := "https://" + request.GetDomain() + request.GetPath() + if canonicalQueryString != "" { + url = url + "?" + canonicalQueryString + } + httpRequest, err := http.NewRequest(httpRequestMethod, url, strings.NewReader(requestPayload)) + if err != nil { + return err + } + for k, v := range headers { + httpRequest.Header[k] = []string{v} + } + if c.debug { + outbytes, err := httputil.DumpRequest(httpRequest, true) + if err != nil { + log.Printf("[ERROR] dump request failed because %s", err) + return err + } + log.Printf("[DEBUG] http request = %s", outbytes) + } + httpResponse, err := c.httpClient.Do(httpRequest) + if err != nil { + msg := fmt.Sprintf("Fail to get response because %s", err) + return errors.NewTencentCloudSDKError("ClientError.NetworkError", msg, "") + } + err = tchttp.ParseFromHttpResponse(httpResponse, response) + return err } func (c *Client) GetRegion() string { @@ -58,7 +222,7 @@ func (c *Client) GetRegion() string { func (c *Client) Init(region string) *Client { c.httpClient = &http.Client{} c.region = region - c.signMethod = "HmacSHA256" + c.signMethod = "TC3-HMAC-SHA256" c.debug = false log.SetFlags(log.LstdFlags | log.Lshortfile) return c @@ -69,8 +233,15 @@ func (c *Client) WithSecretId(secretId, secretKey string) *Client { return c } +func (c *Client) WithCredential(cred *Credential) *Client { + c.credential = cred + return c +} + func (c *Client) WithProfile(clientProfile *profile.ClientProfile) *Client { + c.profile = clientProfile c.signMethod = clientProfile.SignMethod + c.unsignedPayload = clientProfile.UnsignedPayload c.httpProfile = clientProfile.HttpProfile c.httpClient.Timeout = time.Duration(c.httpProfile.ReqTimeout) * time.Second return c @@ -81,6 +252,16 @@ func (c *Client) WithSignatureMethod(method string) *Client { return c } +func (c *Client) WithHttpTransport(transport http.RoundTripper) *Client { + c.httpClient.Transport = transport + return c +} + +func (c *Client) WithDebug(flag bool) *Client { + c.debug = flag + return c +} + func NewClientWithSecretId(secretId, secretKey, region string) (client *Client, err error) { client = &Client{} client.Init(region).WithSecretId(secretId, secretKey) diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/credentials.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/credentials.go index 19f0722de8..b734c13734 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/credentials.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/credentials.go @@ -3,6 +3,7 @@ package common type Credential struct { SecretId string SecretKey string + Token string } func NewCredential(secretId, secretKey string) *Credential { @@ -12,29 +13,46 @@ func NewCredential(secretId, secretKey string) *Credential { } } -func (c *Credential) GetCredentialParams() map[string]string { - return map[string]string{ - "SecretId": c.SecretId, - } -} - -type TokenCredential struct { - SecretId string - SecretKey string - Token string -} - -func NewTokenCredential(secretId, secretKey, token string) *TokenCredential { - return &TokenCredential{ +func NewTokenCredential(secretId, secretKey, token string) *Credential { + return &Credential{ SecretId: secretId, SecretKey: secretKey, Token: token, } } -func (c *TokenCredential) GetCredentialParams() map[string]string { - return map[string]string{ +func (c *Credential) GetCredentialParams() map[string]string { + p := map[string]string{ "SecretId": c.SecretId, - "Token": c.Token, } + if c.Token != "" { + p["Token"] = c.Token + } + return p } + +// Nowhere use them and we haven't well designed these structures and +// underlying method, which leads to the situation that it is hard to +// refactor it to interfaces. +// Hence they are removed and merged into Credential. + +//type TokenCredential struct { +// SecretId string +// SecretKey string +// Token string +//} + +//func NewTokenCredential(secretId, secretKey, token string) *TokenCredential { +// return &TokenCredential{ +// SecretId: secretId, +// SecretKey: secretKey, +// Token: token, +// } +//} + +//func (c *TokenCredential) GetCredentialParams() map[string]string { +// return map[string]string{ +// "SecretId": c.SecretId, +// "Token": c.Token, +// } +//} diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/request.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/request.go index a42b2c42cd..2f03f2dc2c 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/request.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/request.go @@ -92,7 +92,7 @@ func (r *BaseRequest) GetService() string { func (r *BaseRequest) GetUrl() string { if r.httpMethod == GET { - return "https://" + r.domain + r.path + "?" + getUrlQueriesEncoded(r.params) + return "https://" + r.domain + r.path + "?" + GetUrlQueriesEncoded(r.params) } else if r.httpMethod == POST { return "https://" + r.domain + r.path } else { @@ -104,7 +104,7 @@ func (r *BaseRequest) GetVersion() string { return r.version } -func getUrlQueriesEncoded(params map[string]string) string { +func GetUrlQueriesEncoded(params map[string]string) string { values := url.Values{} for key, value := range params { if value != "" { @@ -116,8 +116,7 @@ func getUrlQueriesEncoded(params map[string]string) string { func (r *BaseRequest) GetBodyReader() io.Reader { if r.httpMethod == POST { - s := getUrlQueriesEncoded(r.params) - //log.Printf("[DEBUG] body: %s", s) + s := GetUrlQueriesEncoded(r.params) return strings.NewReader(s) } else { return strings.NewReader("") @@ -125,7 +124,6 @@ func (r *BaseRequest) GetBodyReader() io.Reader { } func (r *BaseRequest) Init() *BaseRequest { - r.httpMethod = GET r.domain = "" r.path = Path r.params = make(map[string]string) @@ -154,7 +152,7 @@ func CompleteCommonParams(request Request, region string) { params["Action"] = request.GetAction() params["Timestamp"] = strconv.FormatInt(time.Now().Unix(), 10) params["Nonce"] = strconv.Itoa(rand.Int()) - params["RequestClient"] = "SDK_GO_3.0.30" + params["RequestClient"] = "SDK_GO_3.0.135" } func ConstructParams(req Request) (err error) { @@ -220,11 +218,15 @@ func flatStructure(value reflect.Value, request Request, prefix string) (err err } else if kind == reflect.Float64 { request.GetParams()[key] = strconv.FormatFloat(vj.Float(), 'f', -1, 64) } else { - flatStructure(vj, request, key+".") + if err = flatStructure(vj, request, key+"."); err != nil { + return + } } } } else { - flatStructure(reflect.ValueOf(field.Interface()), request, prefix+nameTag+".") + if err = flatStructure(reflect.ValueOf(field.Interface()), request, prefix+nameTag+"."); err != nil { + return + } } } return diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/response.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/response.go index 100fca1102..288f21bdf6 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/response.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http/response.go @@ -2,8 +2,9 @@ package common import ( "encoding/json" + "fmt" "io/ioutil" - // "log" + //"log" "net/http" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors" @@ -36,7 +37,8 @@ func (r *BaseResponse) ParseErrorFromHTTPResponse(body []byte) (err error) { resp := &ErrorResponse{} err = json.Unmarshal(body, resp) if err != nil { - return + msg := fmt.Sprintf("Fail to parse json content: %s, because: %s", body, err) + return errors.NewTencentCloudSDKError("ClientError.ParseJsonError", msg, "") } if resp.Response.Error.Code != "" { return errors.NewTencentCloudSDKError(resp.Response.Error.Code, resp.Response.Error.Message, resp.Response.RequestId) @@ -45,7 +47,8 @@ func (r *BaseResponse) ParseErrorFromHTTPResponse(body []byte) (err error) { deprecated := &DeprecatedAPIErrorResponse{} err = json.Unmarshal(body, deprecated) if err != nil { - return + msg := fmt.Sprintf("Fail to parse json content: %s, because: %s", body, err) + return errors.NewTencentCloudSDKError("ClientError.ParseJsonError", msg, "") } if deprecated.Code != 0 { return errors.NewTencentCloudSDKError(deprecated.CodeDesc, deprecated.Message, "") @@ -57,7 +60,12 @@ func ParseFromHttpResponse(hr *http.Response, response Response) (err error) { defer hr.Body.Close() body, err := ioutil.ReadAll(hr.Body) if err != nil { - return + msg := fmt.Sprintf("Fail to read response body because %s", err) + return errors.NewTencentCloudSDKError("ClientError.IOError", msg, "") + } + if hr.StatusCode != 200 { + msg := fmt.Sprintf("Request fail with http status code: %s, with body: %s", hr.Status, body) + return errors.NewTencentCloudSDKError("ClientError.HttpStatusCodeError", msg, "") } //log.Printf("[DEBUG] Response Body=%s", body) err = response.ParseErrorFromHTTPResponse(body) @@ -65,5 +73,9 @@ func ParseFromHttpResponse(hr *http.Response, response Response) (err error) { return } err = json.Unmarshal(body, &response) + if err != nil { + msg := fmt.Sprintf("Fail to parse json content: %s, because: %s", body, err) + return errors.NewTencentCloudSDKError("ClientError.ParseJsonError", msg, "") + } return } diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/client_profile.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/client_profile.go index 94c63b2f03..21069ff99d 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/client_profile.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/client_profile.go @@ -2,12 +2,20 @@ package profile type ClientProfile struct { HttpProfile *HttpProfile - SignMethod string + // Valid choices: HmacSHA1, HmacSHA256, TC3-HMAC-SHA256. + // Default value is TC3-HMAC-SHA256. + SignMethod string + UnsignedPayload bool + // Valid choices: zh-CN, en-US. + // Default value is zh-CN. + Language string } func NewClientProfile() *ClientProfile { return &ClientProfile{ - HttpProfile: NewHttpProfile(), - SignMethod: "HmacSHA256", + HttpProfile: NewHttpProfile(), + SignMethod: "TC3-HMAC-SHA256", + UnsignedPayload: false, + Language: "zh-CN", } } diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/http_profile.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/http_profile.go index 8d4bf8f57b..cf633a8a04 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/http_profile.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile/http_profile.go @@ -4,7 +4,9 @@ type HttpProfile struct { ReqMethod string ReqTimeout int Endpoint string - Protocol string + // Deprecated, use Scheme instead + Protocol string + Scheme string } func NewHttpProfile() *HttpProfile { @@ -12,6 +14,6 @@ func NewHttpProfile() *HttpProfile { ReqMethod: "POST", ReqTimeout: 60, Endpoint: "", - Protocol: "HTTPS", + Scheme: "HTTPS", } } diff --git a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/sign.go b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/sign.go index 450fe2ccde..0aa7b73557 100644 --- a/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/sign.go +++ b/vendor/github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/sign.go @@ -1,13 +1,13 @@ package common import ( + "bytes" "crypto/hmac" "crypto/sha1" "crypto/sha256" "encoding/base64" - "fmt" + "encoding/hex" "sort" - "strings" tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http" ) @@ -27,6 +27,17 @@ func Sign(s, secretKey, method string) string { return base64.StdEncoding.EncodeToString(hashed.Sum(nil)) } +func sha256hex(s string) string { + b := sha256.Sum256([]byte(s)) + return hex.EncodeToString(b[:]) +} + +func hmacsha256(s, key string) string { + hashed := hmac.New(sha256.New, []byte(key)) + hashed.Write([]byte(s)) + return string(hashed.Sum(nil)) +} + func signRequest(request tchttp.Request, credential *Credential, method string) (err error) { if method != SHA256 { method = SHA1 @@ -53,7 +64,11 @@ func getStringToSign(request tchttp.Request) string { domain := request.GetDomain() path := request.GetPath() - text := method + domain + path + "?" + var buf bytes.Buffer + buf.WriteString(method) + buf.WriteString(domain) + buf.WriteString(path) + buf.WriteString("?") params := request.GetParams() // sort params @@ -65,11 +80,15 @@ func getStringToSign(request tchttp.Request) string { for i := range keys { k := keys[i] + // TODO: check if server side allows empty value in url. if params[k] == "" { continue } - text += fmt.Sprintf("%v=%v&", strings.Replace(k, "_", ".", -1), params[k]) + buf.WriteString(k) + buf.WriteString("=") + buf.WriteString(params[k]) + buf.WriteString("&") } - text = text[:len(text)-1] - return text + buf.Truncate(buf.Len() - 1) + return buf.String() } diff --git a/vendor/golang.org/x/net/http/httpproxy/proxy.go b/vendor/golang.org/x/net/http/httpproxy/proxy.go new file mode 100644 index 0000000000..163645b86f --- /dev/null +++ b/vendor/golang.org/x/net/http/httpproxy/proxy.go @@ -0,0 +1,370 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httpproxy provides support for HTTP proxy determination +// based on environment variables, as provided by net/http's +// ProxyFromEnvironment function. +// +// The API is not subject to the Go 1 compatibility promise and may change at +// any time. +package httpproxy + +import ( + "errors" + "fmt" + "net" + "net/url" + "os" + "strings" + "unicode/utf8" + + "golang.org/x/net/idna" +) + +// Config holds configuration for HTTP proxy settings. See +// FromEnvironment for details. +type Config struct { + // HTTPProxy represents the value of the HTTP_PROXY or + // http_proxy environment variable. It will be used as the proxy + // URL for HTTP requests and HTTPS requests unless overridden by + // HTTPSProxy or NoProxy. + HTTPProxy string + + // HTTPSProxy represents the HTTPS_PROXY or https_proxy + // environment variable. It will be used as the proxy URL for + // HTTPS requests unless overridden by NoProxy. + HTTPSProxy string + + // NoProxy represents the NO_PROXY or no_proxy environment + // variable. It specifies a string that contains comma-separated values + // specifying hosts that should be excluded from proxying. Each value is + // represented by an IP address prefix (1.2.3.4), an IP address prefix in + // CIDR notation (1.2.3.4/8), a domain name, or a special DNS label (*). + // An IP address prefix and domain name can also include a literal port + // number (1.2.3.4:80). + // A domain name matches that name and all subdomains. A domain name with + // a leading "." matches subdomains only. For example "foo.com" matches + // "foo.com" and "bar.foo.com"; ".y.com" matches "x.y.com" but not "y.com". + // A single asterisk (*) indicates that no proxying should be done. + // A best effort is made to parse the string and errors are + // ignored. + NoProxy string + + // CGI holds whether the current process is running + // as a CGI handler (FromEnvironment infers this from the + // presence of a REQUEST_METHOD environment variable). + // When this is set, ProxyForURL will return an error + // when HTTPProxy applies, because a client could be + // setting HTTP_PROXY maliciously. See https://golang.org/s/cgihttpproxy. + CGI bool +} + +// config holds the parsed configuration for HTTP proxy settings. +type config struct { + // Config represents the original configuration as defined above. + Config + + // httpsProxy is the parsed URL of the HTTPSProxy if defined. + httpsProxy *url.URL + + // httpProxy is the parsed URL of the HTTPProxy if defined. + httpProxy *url.URL + + // ipMatchers represent all values in the NoProxy that are IP address + // prefixes or an IP address in CIDR notation. + ipMatchers []matcher + + // domainMatchers represent all values in the NoProxy that are a domain + // name or hostname & domain name + domainMatchers []matcher +} + +// FromEnvironment returns a Config instance populated from the +// environment variables HTTP_PROXY, HTTPS_PROXY and NO_PROXY (or the +// lowercase versions thereof). HTTPS_PROXY takes precedence over +// HTTP_PROXY for https requests. +// +// The environment values may be either a complete URL or a +// "host[:port]", in which case the "http" scheme is assumed. An error +// is returned if the value is a different form. +func FromEnvironment() *Config { + return &Config{ + HTTPProxy: getEnvAny("HTTP_PROXY", "http_proxy"), + HTTPSProxy: getEnvAny("HTTPS_PROXY", "https_proxy"), + NoProxy: getEnvAny("NO_PROXY", "no_proxy"), + CGI: os.Getenv("REQUEST_METHOD") != "", + } +} + +func getEnvAny(names ...string) string { + for _, n := range names { + if val := os.Getenv(n); val != "" { + return val + } + } + return "" +} + +// ProxyFunc returns a function that determines the proxy URL to use for +// a given request URL. Changing the contents of cfg will not affect +// proxy functions created earlier. +// +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request, as +// defined by NO_PROXY. +// +// As a special case, if req.URL.Host is "localhost" (with or without a +// port number), then a nil URL and nil error will be returned. +func (cfg *Config) ProxyFunc() func(reqURL *url.URL) (*url.URL, error) { + // Preprocess the Config settings for more efficient evaluation. + cfg1 := &config{ + Config: *cfg, + } + cfg1.init() + return cfg1.proxyForURL +} + +func (cfg *config) proxyForURL(reqURL *url.URL) (*url.URL, error) { + var proxy *url.URL + if reqURL.Scheme == "https" { + proxy = cfg.httpsProxy + } + if proxy == nil { + proxy = cfg.httpProxy + if proxy != nil && cfg.CGI { + return nil, errors.New("refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } + } + if proxy == nil { + return nil, nil + } + if !cfg.useProxy(canonicalAddr(reqURL)) { + return nil, nil + } + + return proxy, nil +} + +func parseProxy(proxy string) (*url.URL, error) { + if proxy == "" { + return nil, nil + } + + proxyURL, err := url.Parse(proxy) + if err != nil || + (proxyURL.Scheme != "http" && + proxyURL.Scheme != "https" && + proxyURL.Scheme != "socks5") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// useProxy reports whether requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func (cfg *config) useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + ip := net.ParseIP(host) + if ip != nil { + if ip.IsLoopback() { + return false + } + } + + addr = strings.ToLower(strings.TrimSpace(host)) + + if ip != nil { + for _, m := range cfg.ipMatchers { + if m.match(addr, port, ip) { + return false + } + } + } + for _, m := range cfg.domainMatchers { + if m.match(addr, port, ip) { + return false + } + } + return true +} + +func (c *config) init() { + if parsed, err := parseProxy(c.HTTPProxy); err == nil { + c.httpProxy = parsed + } + if parsed, err := parseProxy(c.HTTPSProxy); err == nil { + c.httpsProxy = parsed + } + + for _, p := range strings.Split(c.NoProxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + + if p == "*" { + c.ipMatchers = []matcher{allMatch{}} + c.domainMatchers = []matcher{allMatch{}} + return + } + + // IPv4/CIDR, IPv6/CIDR + if _, pnet, err := net.ParseCIDR(p); err == nil { + c.ipMatchers = append(c.ipMatchers, cidrMatch{cidr: pnet}) + continue + } + + // IPv4:port, [IPv6]:port + phost, pport, err := net.SplitHostPort(p) + if err == nil { + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + if phost[0] == '[' && phost[len(phost)-1] == ']' { + phost = phost[1 : len(phost)-1] + } + } else { + phost = p + } + // IPv4, IPv6 + if pip := net.ParseIP(phost); pip != nil { + c.ipMatchers = append(c.ipMatchers, ipMatch{ip: pip, port: pport}) + continue + } + + if len(phost) == 0 { + // There is no host part, likely the entry is malformed; ignore. + continue + } + + // domain.com or domain.com:80 + // foo.com matches bar.foo.com + // .domain.com or .domain.com:port + // *.domain.com or *.domain.com:port + if strings.HasPrefix(phost, "*.") { + phost = phost[1:] + } + matchHost := false + if phost[0] != '.' { + matchHost = true + phost = "." + phost + } + c.domainMatchers = append(c.domainMatchers, domainMatch{host: phost, port: pport, matchHost: matchHost}) + } +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", + "socks5": "1080", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] + } + return net.JoinHostPort(addr, port) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +func idnaASCII(v string) (string, error) { + // TODO: Consider removing this check after verifying performance is okay. + // Right now punycode verification, length checks, context checks, and the + // permissible character tests are all omitted. It also prevents the ToASCII + // call from salvaging an invalid IDN, when possible. As a result it may be + // possible to have two IDNs that appear identical to the user where the + // ASCII-only version causes an error downstream whereas the non-ASCII + // version does not. + // Note that for correct ASCII IDNs ToASCII will only do considerably more + // work, but it will not cause an allocation. + if isASCII(v) { + return v, nil + } + return idna.Lookup.ToASCII(v) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +// matcher represents the matching rule for a given value in the NO_PROXY list +type matcher interface { + // match returns true if the host and optional port or ip and optional port + // are allowed + match(host, port string, ip net.IP) bool +} + +// allMatch matches on all possible inputs +type allMatch struct{} + +func (a allMatch) match(host, port string, ip net.IP) bool { + return true +} + +type cidrMatch struct { + cidr *net.IPNet +} + +func (m cidrMatch) match(host, port string, ip net.IP) bool { + return m.cidr.Contains(ip) +} + +type ipMatch struct { + ip net.IP + port string +} + +func (m ipMatch) match(host, port string, ip net.IP) bool { + if m.ip.Equal(ip) { + return m.port == "" || m.port == port + } + return false +} + +type domainMatch struct { + host string + port string + + matchHost bool +} + +func (m domainMatch) match(host, port string, ip net.IP) bool { + if strings.HasSuffix(host, m.host) || (m.matchHost && host == m.host[1:]) { + return m.port == "" || m.port == port + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index e3ef90c314..8dc6c633f0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -511,7 +511,7 @@ github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require github.com/stretchr/testify/suite -# github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e +# github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http @@ -628,6 +628,7 @@ golang.org/x/net/bpf golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts +golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/http2/hpack golang.org/x/net/idna From 70b78835d3d98b77e71a865ce192b89aacf04f54 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:19:52 +0800 Subject: [PATCH 18/42] qcloud: use QcloudClientConfig --- cmd/qcloudcli/main.go | 11 +-- pkg/multicloud/qcloud/bucket.go | 6 +- pkg/multicloud/qcloud/host.go | 6 +- pkg/multicloud/qcloud/localstorage.go | 6 +- pkg/multicloud/qcloud/monitor.go | 2 +- pkg/multicloud/qcloud/provider/provider.go | 6 +- pkg/multicloud/qcloud/qcloud.go | 95 +++++++++++++--------- pkg/multicloud/qcloud/storage.go | 6 +- pkg/multicloud/qcloud/storagecache.go | 10 +-- 9 files changed, 87 insertions(+), 61 deletions(-) diff --git a/cmd/qcloudcli/main.go b/cmd/qcloudcli/main.go index cbd7694595..f53db18a48 100644 --- a/cmd/qcloudcli/main.go +++ b/cmd/qcloudcli/main.go @@ -85,11 +85,12 @@ func newClient(options *BaseOptions) (*qcloud.SRegion, error) { return nil, fmt.Errorf("Missing SecretID") } - if cli, err := qcloud.NewQcloudClient("", "", - options.SecretID, - options.SecretKey, - options.AppID, - options.Debug); err != nil { + if cli, err := qcloud.NewQcloudClient( + qcloud.NewQcloudClientConfig( + options.SecretID, + options.SecretKey, + ).AppId(options.AppID).Debug(options.Debug), + ); err != nil { return nil, err } else if region := cli.GetRegion(options.RegionId); region == nil { return nil, fmt.Errorf("No such region %s", options.RegionId) diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index c6e54bba26..158d6dc7d0 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -135,7 +135,7 @@ func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error { } func (b *SBucket) getFullName() string { - return fmt.Sprintf("%s-%s", b.Name, b.region.client.AppID) + return fmt.Sprintf("%s-%s", b.Name, b.region.client.appId) } func (b *SBucket) getBucketUrlHost() string { @@ -393,8 +393,8 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s return "", errors.Wrap(err, "GetCosClient") } url, err := coscli.Object.GetPresignedURL(context.Background(), method, key, - b.region.client.SecretID, - b.region.client.SecretKey, + b.region.client.secretId, + b.region.client.secretKey, expire, nil) if err != nil { return "", errors.Wrap(err, "coscli.Object.GetPresignedURL") diff --git a/pkg/multicloud/qcloud/host.go b/pkg/multicloud/qcloud/host.go index c8c8fe9991..bfa1001a49 100644 --- a/pkg/multicloud/qcloud/host.go +++ b/pkg/multicloud/qcloud/host.go @@ -37,15 +37,15 @@ func (self *SHost) GetMetadata() *jsonutils.JSONDict { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) } func (self *SHost) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetInstanceById(instanceId string) (*SInstance, error) { diff --git a/pkg/multicloud/qcloud/localstorage.go b/pkg/multicloud/qcloud/localstorage.go index 1261719d2f..c52fcb560c 100644 --- a/pkg/multicloud/qcloud/localstorage.go +++ b/pkg/multicloud/qcloud/localstorage.go @@ -35,15 +35,15 @@ func (self *SLocalStorage) GetMetadata() *jsonutils.JSONDict { } func (self *SLocalStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), self.storageType) } func (self *SLocalStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) } func (self *SLocalStorage) GetGlobalId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) } func (self *SLocalStorage) IsEmulated() bool { diff --git a/pkg/multicloud/qcloud/monitor.go b/pkg/multicloud/qcloud/monitor.go index 9661117696..6375b26a45 100644 --- a/pkg/multicloud/qcloud/monitor.go +++ b/pkg/multicloud/qcloud/monitor.go @@ -53,7 +53,7 @@ func (r *SRegion) metricsRequest(action string, params map[string]string) (jsonu if err != nil { return nil, err } - return monitorRequest(cli, action, params, client.Debug) + return monitorRequest(cli, action, params, client.debug) } func (r *SRegion) GetMonitorData(name string, ns string, since time.Time, until time.Time, diff --git a/pkg/multicloud/qcloud/provider/provider.go b/pkg/multicloud/qcloud/provider/provider.go index 564237ddaf..e53f58ea9b 100644 --- a/pkg/multicloud/qcloud/provider/provider.go +++ b/pkg/multicloud/qcloud/provider/provider.go @@ -105,7 +105,11 @@ func (self *SQcloudProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig secretId = tmp[0] appId = tmp[1] } - client, err := qcloud.NewQcloudClient(cfg.Id, cfg.Name, secretId, cfg.Secret, appId, false) + client, err := qcloud.NewQcloudClient( + qcloud.NewQcloudClientConfig( + secretId, cfg.Secret, + ).AppId(appId).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/qcloud/qcloud.go b/pkg/multicloud/qcloud/qcloud.go index b355a7335d..904dfff3a2 100644 --- a/pkg/multicloud/qcloud/qcloud.go +++ b/pkg/multicloud/qcloud/qcloud.go @@ -52,30 +52,51 @@ const ( QCLOUD_AUDIT_API_VERSION = "2019-03-19" ) -type SQcloudClient struct { - providerId string - providerName string - AppID string - SecretID string - SecretKey string +type QcloudClientConfig struct { + cpcfg cloudprovider.ProviderConfig + secretId string + secretKey string + appId string + + debug bool +} + +func NewQcloudClientConfig(secretId, secretKey string) *QcloudClientConfig { + cfg := &QcloudClientConfig{ + secretId: secretId, + secretKey: secretKey, + } + return cfg +} + +func (cfg *QcloudClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *QcloudClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *QcloudClientConfig) AppId(appId string) *QcloudClientConfig { + cfg.appId = appId + return cfg +} + +func (cfg *QcloudClientConfig) Debug(debug bool) *QcloudClientConfig { + cfg.debug = debug + return cfg +} + +type SQcloudClient struct { + *QcloudClientConfig ownerId string ownerName string iregions []cloudprovider.ICloudRegion ibuckets []cloudprovider.ICloudBucket - - Debug bool } -func NewQcloudClient(providerId string, providerName string, secretID string, secretKey string, appID string, isDebug bool) (*SQcloudClient, error) { +func NewQcloudClient(cfg *QcloudClientConfig) (*SQcloudClient, error) { client := SQcloudClient{ - providerId: providerId, - providerName: providerName, - SecretID: secretID, - SecretKey: secretKey, - AppID: appID, - Debug: isDebug, + QcloudClientConfig: cfg, } err := client.fetchRegions() if err != nil { @@ -89,7 +110,7 @@ func NewQcloudClient(providerId string, providerName string, secretID string, se if err != nil { return nil, errors.Wrap(err, "fetchBuckets") } - if isDebug { + if client.debug { log.Debugf("ownerID: %s ownerName: %s", client.ownerId, client.ownerName) } return &client, nil @@ -404,7 +425,7 @@ func (client *SQcloudClient) GetRegions() []SRegion { } func (client *SQcloudClient) getDefaultClient() (*common.Client, error) { - return common.NewClientWithSecretId(client.SecretID, client.SecretKey, QCLOUD_DEFAULT_REGION) + return common.NewClientWithSecretId(client.secretId, client.secretKey, QCLOUD_DEFAULT_REGION) } func (client *SQcloudClient) vpcRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -412,7 +433,7 @@ func (client *SQcloudClient) vpcRequest(apiName string, params map[string]string if err != nil { return nil, err } - return vpcRequest(cli, apiName, params, client.Debug) + return vpcRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) auditRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -420,7 +441,7 @@ func (client *SQcloudClient) auditRequest(apiName string, params map[string]stri if err != nil { return nil, err } - return auditRequest(cli, apiName, params, client.Debug) + return auditRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) cbsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -428,7 +449,7 @@ func (client *SQcloudClient) cbsRequest(apiName string, params map[string]string if err != nil { return nil, err } - return cbsRequest(cli, apiName, params, client.Debug) + return cbsRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) accountRequestRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -436,7 +457,7 @@ func (client *SQcloudClient) accountRequestRequest(apiName string, params map[st if err != nil { return nil, err } - return accountRequest(cli, apiName, params, client.Debug) + return accountRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) clbRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -444,7 +465,7 @@ func (client *SQcloudClient) clbRequest(apiName string, params map[string]string if err != nil { return nil, err } - return clbRequest(cli, apiName, params, client.Debug) + return clbRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) lbRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -452,7 +473,7 @@ func (client *SQcloudClient) lbRequest(apiName string, params map[string]string) if err != nil { return nil, err } - return lbRequest(cli, apiName, params, client.Debug) + return lbRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) wssRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -460,7 +481,7 @@ func (client *SQcloudClient) wssRequest(apiName string, params map[string]string if err != nil { return nil, err } - return wssRequest(cli, apiName, params, client.Debug) + return wssRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) vpc2017Request(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -468,7 +489,7 @@ func (client *SQcloudClient) vpc2017Request(apiName string, params map[string]st if err != nil { return nil, err } - return vpc2017Request(cli, apiName, params, client.Debug) + return vpc2017Request(cli, apiName, params, client.debug) } func (client *SQcloudClient) billingRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -476,7 +497,7 @@ func (client *SQcloudClient) billingRequest(apiName string, params map[string]st if err != nil { return nil, err } - return billingRequest(cli, apiName, params, client.Debug) + return billingRequest(cli, apiName, params, client.debug) } func (client *SQcloudClient) jsonRequest(apiName string, params map[string]string, retry bool) (jsonutils.JSONObject, error) { @@ -484,7 +505,7 @@ func (client *SQcloudClient) jsonRequest(apiName string, params map[string]strin if err != nil { return nil, err } - return jsonRequest(cli, apiName, params, client.Debug, retry) + return jsonRequest(cli, apiName, params, client.debug, retry) } func (client *SQcloudClient) fetchRegions() error { @@ -520,13 +541,13 @@ func (client *SQcloudClient) getCosClient(bucket *SBucket) (*cos.Client, error) baseUrl, &http.Client{ Transport: &cos.AuthorizationTransport{ - SecretID: client.SecretID, - SecretKey: client.SecretKey, + SecretID: client.secretId, + SecretKey: client.secretKey, Transport: &debug.DebugRequestTransport{ - RequestHeader: client.Debug, - RequestBody: client.Debug, - ResponseHeader: client.Debug, - ResponseBody: client.Debug, + RequestHeader: client.debug, + RequestBody: client.debug, + ResponseHeader: client.debug, + ResponseBody: client.debug, }, }, }, @@ -614,11 +635,11 @@ func (client *SQcloudClient) GetSubAccounts() ([]cloudprovider.SSubAccount, erro return nil, err } subAccount := cloudprovider.SSubAccount{} - subAccount.Name = client.providerName - subAccount.Account = client.SecretID + subAccount.Name = client.cpcfg.Name + subAccount.Account = client.secretId subAccount.HealthStatus = api.CLOUD_PROVIDER_HEALTH_NORMAL - if len(client.AppID) > 0 { - subAccount.Account = fmt.Sprintf("%s/%s", client.SecretID, client.AppID) + if len(client.appId) > 0 { + subAccount.Account = fmt.Sprintf("%s/%s", client.secretId, client.appId) } return []cloudprovider.SSubAccount{subAccount}, nil } diff --git a/pkg/multicloud/qcloud/storage.go b/pkg/multicloud/qcloud/storage.go index d997e8e992..2cd13ae3c5 100644 --- a/pkg/multicloud/qcloud/storage.go +++ b/pkg/multicloud/qcloud/storage.go @@ -37,15 +37,15 @@ func (self *SStorage) GetMetadata() *jsonutils.JSONDict { } func (self *SStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), strings.ToLower(self.storageType)) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), strings.ToLower(self.storageType)) } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), strings.ToLower(self.storageType)) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), strings.ToLower(self.storageType)) } func (self *SStorage) GetGlobalId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), strings.ToLower(self.storageType)) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), strings.ToLower(self.storageType)) } func (self *SStorage) IsEmulated() bool { diff --git a/pkg/multicloud/qcloud/storagecache.go b/pkg/multicloud/qcloud/storagecache.go index e99e8ce2ed..c71d1dda18 100644 --- a/pkg/multicloud/qcloud/storagecache.go +++ b/pkg/multicloud/qcloud/storagecache.go @@ -44,11 +44,11 @@ func (self *SStoragecache) GetMetadata() *jsonutils.JSONDict { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetStatus() string { @@ -60,7 +60,7 @@ func (self *SStoragecache) Refresh() error { } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) IsEmulated() bool { @@ -153,8 +153,8 @@ func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.To } func (self *SRegion) getCosUrl(bucket, object string) string { - //signature := cosauth.NewSignature(self.client.AppID, bucket, self.client.SecretID, time.Now().Add(time.Minute*30).String(), time.Now().String(), "yunion", object).SignOnce(self.client.SecretKey) - return fmt.Sprintf("http://%s-%s.cos.%s.myqcloud.com/%s", bucket, self.client.AppID, self.Region, object) + //signature := cosauth.NewSignature(self.client.AppID, bucket, self.client.secretId, time.Now().Add(time.Minute*30).String(), time.Now().String(), "yunion", object).SignOnce(self.client.secretKey) + return fmt.Sprintf("http://%s-%s.cos.%s.myqcloud.com/%s", bucket, self.client.appId, self.Region, object) } func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { From 11ee270911fa103721255a5a7c2e9823a5a0683a Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:31:12 +0800 Subject: [PATCH 19/42] qcloud: use cpcfg.ProxyFunc --- pkg/multicloud/qcloud/qcloud.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/multicloud/qcloud/qcloud.go b/pkg/multicloud/qcloud/qcloud.go index 904dfff3a2..0a53a2f7f3 100644 --- a/pkg/multicloud/qcloud/qcloud.go +++ b/pkg/multicloud/qcloud/qcloud.go @@ -425,7 +425,15 @@ func (client *SQcloudClient) GetRegions() []SRegion { } func (client *SQcloudClient) getDefaultClient() (*common.Client, error) { - return common.NewClientWithSecretId(client.secretId, client.secretKey, QCLOUD_DEFAULT_REGION) + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, client.cpcfg.ProxyFunc) + + cli, err := common.NewClientWithSecretId(client.secretId, client.secretKey, QCLOUD_DEFAULT_REGION) + if err != nil { + return nil, err + } + cli.WithHttpTransport(httpClient.Transport) + return cli, nil } func (client *SQcloudClient) vpcRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { From b4438122f9b81ec885c6675e387316b853f83509 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:35:48 +0800 Subject: [PATCH 20/42] ucloud: use UcloudClientConfig --- cmd/ucloudcli/main.go | 11 ++--- pkg/multicloud/ucloud/client.go | 2 +- pkg/multicloud/ucloud/host.go | 4 +- pkg/multicloud/ucloud/provider/provider.go | 6 ++- pkg/multicloud/ucloud/storage.go | 6 +-- pkg/multicloud/ucloud/storagecache.go | 6 +-- pkg/multicloud/ucloud/ucloud.go | 47 ++++++++++++++++------ 7 files changed, 55 insertions(+), 27 deletions(-) diff --git a/cmd/ucloudcli/main.go b/cmd/ucloudcli/main.go index 00e11b7c94..fd912e7e04 100644 --- a/cmd/ucloudcli/main.go +++ b/cmd/ucloudcli/main.go @@ -86,11 +86,12 @@ func newClient(options *BaseOptions) (*ucloud.SRegion, error) { return nil, fmt.Errorf("Missing secret") } - cli, err := ucloud.NewUcloudClient("", "", - options.AccessKey, - options.Secret, - options.ProjectId, - options.Debug) + cli, err := ucloud.NewUcloudClient( + ucloud.NewUcloudClientConfig( + options.AccessKey, + options.Secret, + ).ProjectId(options.ProjectId).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/ucloud/client.go b/pkg/multicloud/ucloud/client.go index 6e7de88b38..9308527854 100644 --- a/pkg/multicloud/ucloud/client.go +++ b/pkg/multicloud/ucloud/client.go @@ -174,7 +174,7 @@ func jsonRequest(client *SUcloudClient, params SParams) (jsonutils.JSONObject, e UCLOUD_API_HOST, nil, BuildParams(params, client.accessKeySecret), - client.Debug) + client.debug) if err == nil { return parseUcloudResponse(params, resp) diff --git a/pkg/multicloud/ucloud/host.go b/pkg/multicloud/ucloud/host.go index 5ccce9e792..12c9a61ec1 100644 --- a/pkg/multicloud/ucloud/host.go +++ b/pkg/multicloud/ucloud/host.go @@ -37,11 +37,11 @@ type SHost struct { } func (self *SHost) GetId() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerId, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId()) } func (self *SHost) GetName() string { - return fmt.Sprintf("%s-%s", self.zone.region.client.providerName, self.zone.GetId()) + return fmt.Sprintf("%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId()) } func (self *SHost) GetGlobalId() string { diff --git a/pkg/multicloud/ucloud/provider/provider.go b/pkg/multicloud/ucloud/provider/provider.go index d8433b35de..7561a3c24d 100644 --- a/pkg/multicloud/ucloud/provider/provider.go +++ b/pkg/multicloud/ucloud/provider/provider.go @@ -84,7 +84,11 @@ func parseAccount(account string) (accessKey string, projectId string) { func (self *SUcloudProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { accessKey, projectId := parseAccount(cfg.Account) - client, err := ucloud.NewUcloudClient(cfg.Id, cfg.Name, accessKey, cfg.Secret, projectId, false) + client, err := ucloud.NewUcloudClient( + ucloud.NewUcloudClientConfig( + accessKey, cfg.Secret, + ).ProjectId(projectId).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/ucloud/storage.go b/pkg/multicloud/ucloud/storage.go index 6b7c6df914..385c131beb 100644 --- a/pkg/multicloud/ucloud/storage.go +++ b/pkg/multicloud/ucloud/storage.go @@ -30,15 +30,15 @@ type SStorage struct { } func (self *SStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetId(), self.storageType) } func (self *SStorage) GetName() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerName, self.zone.GetId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Name, self.zone.GetId(), self.storageType) } func (self *SStorage) GetGlobalId() string { - return fmt.Sprintf("%s-%s-%s", self.zone.region.client.providerId, self.zone.GetGlobalId(), self.storageType) + return fmt.Sprintf("%s-%s-%s", self.zone.region.client.cpcfg.Id, self.zone.GetGlobalId(), self.storageType) } func (self *SStorage) GetStatus() string { diff --git a/pkg/multicloud/ucloud/storagecache.go b/pkg/multicloud/ucloud/storagecache.go index 99b6ae70d8..cdcaef867b 100644 --- a/pkg/multicloud/ucloud/storagecache.go +++ b/pkg/multicloud/ucloud/storagecache.go @@ -61,15 +61,15 @@ func GetBucketName(regionId string, imageId string) string { } func (self *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetId()) } func (self *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", self.region.client.providerName, self.region.GetId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Name, self.region.GetId()) } func (self *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", self.region.client.providerId, self.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", self.region.client.cpcfg.Id, self.region.GetGlobalId()) } func (self *SStoragecache) GetStatus() string { diff --git a/pkg/multicloud/ucloud/ucloud.go b/pkg/multicloud/ucloud/ucloud.go index 07425d851c..0c3d9ff8dd 100644 --- a/pkg/multicloud/ucloud/ucloud.go +++ b/pkg/multicloud/ucloud/ucloud.go @@ -56,30 +56,53 @@ const ( UCLOUD_API_VERSION = "2019-02-28" ) -type SUcloudClient struct { - providerId string - providerName string +type UcloudClientConfig struct { + cpcfg cloudprovider.ProviderConfig + accessKeyId string accessKeySecret string projectId string + debug bool +} + +func NewUcloudClientConfig(accessKeyId, accessKeySecret string) *UcloudClientConfig { + cfg := &UcloudClientConfig{ + accessKeyId: accessKeyId, + accessKeySecret: accessKeySecret, + } + return cfg +} + +func (cfg *UcloudClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *UcloudClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *UcloudClientConfig) ProjectId(projectId string) *UcloudClientConfig { + cfg.projectId = projectId + return cfg +} + +func (cfg *UcloudClientConfig) Debug(debug bool) *UcloudClientConfig { + cfg.debug = debug + return cfg +} + +type SUcloudClient struct { + *UcloudClientConfig + iregions []cloudprovider.ICloudRegion iBuckets []cloudprovider.ICloudBucket httpClient *http.Client - Debug bool } // 进行资源操作时参数account 对应数据库cloudprovider表中的account字段,由accessKey和projectID两部分组成，通过"/"分割。 // 初次导入Subaccount时，参数account对应cloudaccounts表中的account字段，即accesskey。此时projectID为空，只能进行同步子账号（项目）、查询region列表等projectId无关的操作。 -func NewUcloudClient(providerId string, providerName string, accessKey string, secret string, projectId string, isDebug bool) (*SUcloudClient, error) { +func NewUcloudClient(cfg *UcloudClientConfig) (*SUcloudClient, error) { client := SUcloudClient{ - providerId: providerId, - providerName: providerName, - accessKeyId: accessKey, - accessKeySecret: secret, - projectId: projectId, - Debug: isDebug, + UcloudClientConfig: cfg, } err := client.fetchRegions() @@ -243,7 +266,7 @@ func (self *SUcloudClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) subAccounts := make([]cloudprovider.SSubAccount, 0) for _, project := range projects { subAccount := cloudprovider.SSubAccount{} - subAccount.Name = fmt.Sprintf("%s-%s", self.providerName, project.ProjectName) + subAccount.Name = fmt.Sprintf("%s-%s", self.cpcfg.Name, project.ProjectName) // ucloud账号ID中可能包含/。因此使用::作为分割符号 subAccount.Account = fmt.Sprintf("%s::%s", self.accessKeyId, project.ProjectID) subAccount.HealthStatus = api.CLOUD_PROVIDER_HEALTH_NORMAL From 64387f83d8c35a0dba795783aca88d248d115958 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:41:14 +0800 Subject: [PATCH 21/42] ucloud: use cpcfg.ProxyFunc --- pkg/multicloud/ucloud/ucloud.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/multicloud/ucloud/ucloud.go b/pkg/multicloud/ucloud/ucloud.go index 0c3d9ff8dd..98f26e6d5a 100644 --- a/pkg/multicloud/ucloud/ucloud.go +++ b/pkg/multicloud/ucloud/ucloud.go @@ -24,6 +24,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/httputils" ) /* @@ -101,8 +102,12 @@ type SUcloudClient struct { // 进行资源操作时参数account 对应数据库cloudprovider表中的account字段,由accessKey和projectID两部分组成，通过"/"分割。 // 初次导入Subaccount时，参数account对应cloudaccounts表中的account字段，即accesskey。此时projectID为空，只能进行同步子账号（项目）、查询region列表等projectId无关的操作。 func NewUcloudClient(cfg *UcloudClientConfig) (*SUcloudClient, error) { + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, cfg.cpcfg.ProxyFunc) + client := SUcloudClient{ UcloudClientConfig: cfg, + httpClient: httpClient, } err := client.fetchRegions() From 22bc479420aeb6063a2db1ba299775e61446c5b6 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:43:54 +0800 Subject: [PATCH 22/42] zstack: remove sessionID field --- pkg/multicloud/zstack/zstack.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/multicloud/zstack/zstack.go b/pkg/multicloud/zstack/zstack.go index 884a8a7455..1a2f74a909 100644 --- a/pkg/multicloud/zstack/zstack.go +++ b/pkg/multicloud/zstack/zstack.go @@ -53,8 +53,6 @@ type SZStackClient struct { password string authURL string - sessionID string - iregions []cloudprovider.ICloudRegion debug bool @@ -177,10 +175,6 @@ func (cli *SZStackClient) listAll(resource string, params url.Values, retVal int } func (cli *SZStackClient) sign(uri, method string, header http.Header) error { - if len(cli.sessionID) > 0 { - header.Set("Authorization", "OAuth "+cli.sessionID) - return nil - } url, err := getSignUrl(uri) if err != nil { return errors.Wrap(err, "sign.getSignUrl") From c3c5b0b1563c3323ddc93b140dbb7575e057c136 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 19:47:51 +0800 Subject: [PATCH 23/42] zstack: use ZstackClientConfig --- cmd/zstackcli/main.go | 8 +++- pkg/multicloud/zstack/provider/provider.go | 6 ++- pkg/multicloud/zstack/region.go | 4 +- pkg/multicloud/zstack/storagecache.go | 4 +- pkg/multicloud/zstack/vpc.go | 2 +- pkg/multicloud/zstack/zstack.go | 50 +++++++++++++++------- 6 files changed, 51 insertions(+), 23 deletions(-) diff --git a/cmd/zstackcli/main.go b/cmd/zstackcli/main.go index b4f8f540da..4694b18b3a 100644 --- a/cmd/zstackcli/main.go +++ b/cmd/zstackcli/main.go @@ -89,7 +89,13 @@ func newClient(options *BaseOptions) (*zstack.SRegion, error) { return nil, fmt.Errorf("Missing Password") } - cli, err := zstack.NewZStackClient("", "", options.AuthURL, options.Username, options.Password, options.Debug) + cli, err := zstack.NewZStackClient( + zstack.NewZstackClientConfig( + options.AuthURL, + options.Username, + options.Password, + ).Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/zstack/provider/provider.go b/pkg/multicloud/zstack/provider/provider.go index c82ba70615..38b0a363a3 100644 --- a/pkg/multicloud/zstack/provider/provider.go +++ b/pkg/multicloud/zstack/provider/provider.go @@ -77,7 +77,11 @@ func (self *SZStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx con } func (self *SZStackProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { - client, err := zstack.NewZStackClient(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) + client, err := zstack.NewZStackClient( + zstack.NewZstackClientConfig( + cfg.URL, cfg.Account, cfg.Secret, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/zstack/region.go b/pkg/multicloud/zstack/region.go index 14547e3fec..346198956b 100644 --- a/pkg/multicloud/zstack/region.go +++ b/pkg/multicloud/zstack/region.go @@ -58,11 +58,11 @@ func (region *SRegion) GetId() string { } func (region *SRegion) GetName() string { - return region.client.providerName + return region.client.cpcfg.Name } func (region *SRegion) GetGlobalId() string { - return fmt.Sprintf("%s/%s", CLOUD_PROVIDER_ZSTACK, region.client.providerID) + return fmt.Sprintf("%s/%s", CLOUD_PROVIDER_ZSTACK, region.client.cpcfg.Id) } func (region *SRegion) IsEmulated() bool { diff --git a/pkg/multicloud/zstack/storagecache.go b/pkg/multicloud/zstack/storagecache.go index 82ca01b53b..fb9829ee70 100644 --- a/pkg/multicloud/zstack/storagecache.go +++ b/pkg/multicloud/zstack/storagecache.go @@ -41,11 +41,11 @@ func (scache *SStoragecache) GetMetadata() *jsonutils.JSONDict { } func (scache *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s/%s", scache.region.client.providerID, scache.region.GetId(), scache.ZoneId) + return fmt.Sprintf("%s-%s/%s", scache.region.client.cpcfg.Id, scache.region.GetId(), scache.ZoneId) } func (scache *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s/%s", scache.region.client.providerName, scache.region.GetId(), scache.ZoneId) + return fmt.Sprintf("%s-%s/%s", scache.region.client.cpcfg.Name, scache.region.GetId(), scache.ZoneId) } func (scache *SStoragecache) GetStatus() string { diff --git a/pkg/multicloud/zstack/vpc.go b/pkg/multicloud/zstack/vpc.go index 6a450829da..631c89c51e 100644 --- a/pkg/multicloud/zstack/vpc.go +++ b/pkg/multicloud/zstack/vpc.go @@ -41,7 +41,7 @@ func (vpc *SVpc) GetId() string { } func (vpc *SVpc) GetName() string { - return fmt.Sprintf("%s-VPC", vpc.region.client.providerName) + return fmt.Sprintf("%s-VPC", vpc.region.client.cpcfg.Name) } func (vpc *SVpc) GetGlobalId() string { diff --git a/pkg/multicloud/zstack/zstack.go b/pkg/multicloud/zstack/zstack.go index 1a2f74a909..61c770799e 100644 --- a/pkg/multicloud/zstack/zstack.go +++ b/pkg/multicloud/zstack/zstack.go @@ -46,18 +46,41 @@ var ( SkipEsxi bool = true ) -type SZStackClient struct { - providerID string - providerName string - username string - password string - authURL string +type ZstackClientConfig struct { + cpcfg cloudprovider.ProviderConfig - iregions []cloudprovider.ICloudRegion + authURL string + username string + password string debug bool } +func NewZstackClientConfig(authURL, username, password string) *ZstackClientConfig { + cfg := &ZstackClientConfig{ + authURL: authURL, + username: username, + password: password, + } + return cfg +} + +func (cfg *ZstackClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *ZstackClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *ZstackClientConfig) Debug(debug bool) *ZstackClientConfig { + cfg.debug = debug + return cfg +} + +type SZStackClient struct { + *ZstackClientConfig + + iregions []cloudprovider.ICloudRegion +} + func getTime() string { return time.Now().Format("Mon, 02 Jan 2006 15:04:05 MST") } @@ -76,14 +99,9 @@ func getSignUrl(uri string) (string, error) { return strings.TrimPrefix(u.Path, "/zstack"), nil } -func NewZStackClient(providerID string, providerName string, authURL string, username string, password string, isDebug bool) (*SZStackClient, error) { +func NewZStackClient(cfg *ZstackClientConfig) (*SZStackClient, error) { cli := &SZStackClient{ - providerID: providerID, - providerName: providerName, - authURL: authURL, - username: username, - password: password, - debug: isDebug, + ZstackClientConfig: cfg, } if err := cli.connect(); err != nil { return nil, err @@ -93,13 +111,13 @@ func NewZStackClient(providerID string, providerName string, authURL string, use } func (cli *SZStackClient) GetCloudRegionExternalIdPrefix() string { - return fmt.Sprintf("%s/%s", CLOUD_PROVIDER_ZSTACK, cli.providerID) + return fmt.Sprintf("%s/%s", CLOUD_PROVIDER_ZSTACK, cli.cpcfg.Id) } func (cli *SZStackClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { subAccount := cloudprovider.SSubAccount{ Account: cli.username, - Name: cli.providerName, + Name: cli.cpcfg.Name, HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL, } return []cloudprovider.SSubAccount{subAccount}, nil From 874699a1d71c430f0ca6bd2e6765a6b987273043 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 18 Mar 2020 20:02:37 +0800 Subject: [PATCH 24/42] zstack: use cpcfg.ProxyFunc --- pkg/multicloud/zstack/zstack.go | 52 ++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/pkg/multicloud/zstack/zstack.go b/pkg/multicloud/zstack/zstack.go index 61c770799e..c2967aa880 100644 --- a/pkg/multicloud/zstack/zstack.go +++ b/pkg/multicloud/zstack/zstack.go @@ -21,6 +21,7 @@ import ( "crypto/sha512" "encoding/base64" "fmt" + "io" "net/http" "net/url" "strings" @@ -78,6 +79,8 @@ func (cfg *ZstackClientConfig) Debug(debug bool) *ZstackClientConfig { type SZStackClient struct { *ZstackClientConfig + httpClient *http.Client + iregions []cloudprovider.ICloudRegion } @@ -100,8 +103,12 @@ func getSignUrl(uri string) (string, error) { } func NewZStackClient(cfg *ZstackClientConfig) (*SZStackClient, error) { + httpClient := httputils.GetDefaultClient() + httputils.SetClientProxyFunc(httpClient, cfg.cpcfg.ProxyFunc) + cli := &SZStackClient{ ZstackClientConfig: cfg, + httpClient: httpClient, } if err := cli.connect(); err != nil { return nil, err @@ -150,7 +157,6 @@ func (cli *SZStackClient) testAccessKey() error { } func (cli *SZStackClient) connect() error { - client := httputils.GetDefaultClient() header := http.Header{} header.Add("Content-Type", "application/json") authURL := cli.authURL + "/zstack/v1/accounts/login" @@ -160,7 +166,7 @@ func (cli *SZStackClient) connect() error { "password": fmt.Sprintf("%x", sha512.Sum512([]byte(cli.password))), }, }) - _, _, err := httputils.JSONRequest(client, context.Background(), "PUT", authURL, header, body, cli.debug) + _, _, err := httputils.JSONRequest(cli.httpClient, context.Background(), "PUT", authURL, header, body, cli.debug) if err != nil { err = cli.testAccessKey() if err == nil { @@ -206,7 +212,6 @@ func (cli *SZStackClient) sign(uri, method string, header http.Header) error { } func (cli *SZStackClient) _list(resource string, start int, limit int, params url.Values) (jsonutils.JSONObject, error) { - client := httputils.GetDefaultClient() header := http.Header{} if params == nil { params = url.Values{} @@ -222,7 +227,7 @@ func (cli *SZStackClient) _list(resource string, start int, limit int, params ur if err != nil { return nil, err } - _, resp, err := httputils.JSONRequest(client, context.Background(), "GET", requestURL, header, nil, cli.debug) + _, resp, err := httputils.JSONRequest(cli.httpClient, context.Background(), "GET", requestURL, header, nil, cli.debug) if err != nil { return nil, errors.Wrapf(err, fmt.Sprintf("GET %s params: %s", resource, params)) } @@ -246,20 +251,19 @@ func (cli *SZStackClient) delete(resource, resourceId, deleteMode string) error } func (cli *SZStackClient) _delete(resource, resourceId, deleteMode string) (jsonutils.JSONObject, error) { - client := httputils.GetDefaultClient() header := http.Header{} requestURL := cli.getDeleteURL(resource, resourceId, deleteMode) err := cli.sign(requestURL, "DELETE", header) if err != nil { return nil, err } - _, resp, err := httputils.JSONRequest(client, context.Background(), "DELETE", requestURL, header, nil, cli.debug) + _, resp, err := httputils.JSONRequest(cli.httpClient, context.Background(), "DELETE", requestURL, header, nil, cli.debug) if err != nil { return nil, errors.Wrapf(err, fmt.Sprintf("DELETE %s %s %s", resource, resourceId, deleteMode)) } if resp.Contains("location") { location, _ := resp.GetString("location") - return cli.wait(client, header, "delete", requestURL, jsonutils.NewDict(), location) + return cli.wait(header, "delete", requestURL, jsonutils.NewDict(), location) } return resp, nil } @@ -283,20 +287,19 @@ func (cli *SZStackClient) put(resource, resourceId string, params jsonutils.JSON } func (cli *SZStackClient) _put(resource, resourceId string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - client := httputils.GetDefaultClient() header := http.Header{} requestURL := cli.getURL(resource, resourceId, "actions") err := cli.sign(requestURL, "PUT", header) if err != nil { return nil, err } - _, resp, err := httputils.JSONRequest(client, context.Background(), "PUT", requestURL, header, params, cli.debug) + _, resp, err := httputils.JSONRequest(cli.httpClient, context.Background(), "PUT", requestURL, header, params, cli.debug) if err != nil { return nil, err } if resp.Contains("location") { location, _ := resp.GetString("location") - return cli.wait(client, header, "update", requestURL, params, location) + return cli.wait(header, "update", requestURL, params, location) } return resp, nil } @@ -328,7 +331,6 @@ func (cli *SZStackClient) getMonitor(resource string, params jsonutils.JSONObjec } func (cli *SZStackClient) _getMonitor(resource string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - client := httputils.GetDefaultClient() header := http.Header{} requestURL := cli.getPostURL(resource) paramDict := params.(*jsonutils.JSONDict) @@ -347,7 +349,7 @@ func (cli *SZStackClient) _getMonitor(resource string, params jsonutils.JSONObje if err != nil { return nil, err } - _, resp, err = httputils.JSONRequest(client, context.Background(), "GET", requestURL, header, nil, cli.debug) + _, resp, err = cli.jsonRequest(context.TODO(), "GET", requestURL, header, nil) if err != nil { if strings.Contains(err.Error(), "exceeded while awaiting headers") { time.Sleep(time.Second * 5) @@ -360,7 +362,7 @@ func (cli *SZStackClient) _getMonitor(resource string, params jsonutils.JSONObje if resp.Contains("location") { location, _ := resp.GetString("location") - return cli.wait(client, header, "get", requestURL, jsonutils.NewDict(), location) + return cli.wait(header, "get", requestURL, jsonutils.NewDict(), location) } return resp, nil } @@ -370,7 +372,6 @@ func (cli *SZStackClient) get(resource, resourceId string, spec string) (jsonuti } func (cli *SZStackClient) _get(resource, resourceId string, spec string) (jsonutils.JSONObject, error) { - client := httputils.GetDefaultClient() header := http.Header{} requestURL := cli.getURL(resource, resourceId, spec) var resp jsonutils.JSONObject @@ -380,7 +381,7 @@ func (cli *SZStackClient) _get(resource, resourceId string, spec string) (jsonut if err != nil { return nil, err } - _, resp, err = httputils.JSONRequest(client, context.Background(), "GET", requestURL, header, nil, cli.debug) + _, resp, err = cli.jsonRequest(context.TODO(), "GET", requestURL, header, nil) if err != nil { if strings.Contains(err.Error(), "exceeded while awaiting headers") { time.Sleep(time.Second * 5) @@ -393,7 +394,7 @@ func (cli *SZStackClient) _get(resource, resourceId string, spec string) (jsonut if resp.Contains("location") { location, _ := resp.GetString("location") - return cli.wait(client, header, "get", requestURL, jsonutils.NewDict(), location) + return cli.wait(header, "get", requestURL, jsonutils.NewDict(), location) } return resp, nil } @@ -413,11 +414,21 @@ func (cli *SZStackClient) post(resource string, params jsonutils.JSONObject) (js return cli._post(resource, params) } -func (cli *SZStackClient) wait(client *http.Client, header http.Header, action string, requestURL string, params jsonutils.JSONObject, location string) (jsonutils.JSONObject, error) { +func (cli *SZStackClient) request(ctx context.Context, method httputils.THttpMethod, urlStr string, header http.Header, body io.Reader) (*http.Response, error) { + resp, err := httputils.Request(cli.httpClient, ctx, method, urlStr, header, body, cli.debug) + return resp, err +} + +func (cli *SZStackClient) jsonRequest(ctx context.Context, method httputils.THttpMethod, urlStr string, header http.Header, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { + hdr, data, err := httputils.JSONRequest(cli.httpClient, ctx, method, urlStr, header, body, cli.debug) + return hdr, data, err +} + +func (cli *SZStackClient) wait(header http.Header, action string, requestURL string, params jsonutils.JSONObject, location string) (jsonutils.JSONObject, error) { startTime := time.Now() timeout := time.Minute * 30 for { - resp, err := httputils.Request(client, context.Background(), "GET", location, header, nil, cli.debug) + resp, err := cli.request(context.TODO(), "GET", location, header, nil) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("wait location %s", location)) } @@ -441,20 +452,19 @@ func (cli *SZStackClient) wait(client *http.Client, header http.Header, action s } func (cli *SZStackClient) _post(resource string, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { - client := httputils.GetDefaultClient() header := http.Header{} requestURL := cli.getPostURL(resource) err := cli.sign(requestURL, "POST", header) if err != nil { return nil, err } - _, resp, err := httputils.JSONRequest(client, context.Background(), "POST", requestURL, header, params, cli.debug) + _, resp, err := cli.jsonRequest(context.TODO(), "POST", requestURL, header, params) if err != nil { return nil, errors.Wrapf(err, fmt.Sprintf("POST %s %s", resource, params.String())) } if resp.Contains("location") { location, _ := resp.GetString("location") - return cli.wait(client, header, "create", requestURL, params, location) + return cli.wait(header, "create", requestURL, params, location) } return resp, nil } From a7cff31e3ca62ec1f35711e7bfec1dc4e1dc6d25 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 10:18:21 +0800 Subject: [PATCH 25/42] openstack: use OpenstackClientConfig --- cmd/openstackcli/main.go | 13 ++- pkg/multicloud/openstack/novastorage.go | 2 +- pkg/multicloud/openstack/openstack.go | 80 +++++++++++++------ pkg/multicloud/openstack/provider/provider.go | 13 ++- pkg/multicloud/openstack/region.go | 4 +- pkg/multicloud/openstack/storagecache.go | 6 +- 6 files changed, 84 insertions(+), 34 deletions(-) diff --git a/cmd/openstackcli/main.go b/cmd/openstackcli/main.go index 6975a98923..0b87dffc00 100644 --- a/cmd/openstackcli/main.go +++ b/cmd/openstackcli/main.go @@ -93,7 +93,18 @@ func newClient(options *BaseOptions) (*openstack.SRegion, error) { return nil, fmt.Errorf("Missing Password") } - cli, err := openstack.NewOpenStackClient("", "", options.AuthURL, options.Username, options.Password, options.Project, options.EndpointType, options.DomainName, options.ProjectDomain, options.Debug) + cli, err := openstack.NewOpenStackClient( + openstack.NewOpenstackClientConfig( + options.AuthURL, + options.Username, + options.Password, + options.Project, + options.ProjectDomain, + ). + EndpointType(options.EndpointType). + DomainName(options.DomainName). + Debug(options.Debug), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/openstack/novastorage.go b/pkg/multicloud/openstack/novastorage.go index 956858d43e..8430758003 100644 --- a/pkg/multicloud/openstack/novastorage.go +++ b/pkg/multicloud/openstack/novastorage.go @@ -32,7 +32,7 @@ func (storage *SNovaStorage) GetMetadata() *jsonutils.JSONDict { } func (storage *SNovaStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.providerID, storage.zone.GetGlobalId(), storage.GetName()) + return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Id, storage.zone.GetGlobalId(), storage.GetName()) } func (storage *SNovaStorage) GetName() string { diff --git a/pkg/multicloud/openstack/openstack.go b/pkg/multicloud/openstack/openstack.go index 7af8110f8a..4011b483e8 100644 --- a/pkg/multicloud/openstack/openstack.go +++ b/pkg/multicloud/openstack/openstack.go @@ -37,47 +37,75 @@ const ( OPENSTACK_DEFAULT_REGION = "RegionOne" ) -type SOpenStackClient struct { - providerID string - providerName string - authURL string - username string - password string - project string - projectDomain string - endpointType string - domainName string - client *mcclient.Client - tokenCredential mcclient.TokenCredential - iregions []cloudprovider.ICloudRegion +type OpenstackClientConfig struct { + cpcfg cloudprovider.ProviderConfig - Debug bool + authURL string + username string + password string + project string + projectDomain string + + domainName string + endpointType string + + debug bool } -func NewOpenStackClient(providerID string, providerName string, authURL string, username string, password string, project string, endpointType string, domainName string, projectDomainName string, isDebug bool) (*SOpenStackClient, error) { - cli := &SOpenStackClient{ - providerID: providerID, - providerName: providerName, - authURL: strings.TrimRight(authURL, "/"), +func NewOpenstackClientConfig(authURL, username, password, project, projectDomain string) *OpenstackClientConfig { + cfg := &OpenstackClientConfig{ + authURL: authURL, username: username, password: password, project: project, - projectDomain: projectDomainName, - endpointType: endpointType, - domainName: domainName, - Debug: isDebug, + projectDomain: projectDomain, + } + return cfg +} + +func (cfg *OpenstackClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *OpenstackClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *OpenstackClientConfig) DomainName(domainName string) *OpenstackClientConfig { + cfg.domainName = domainName + return cfg +} + +func (cfg *OpenstackClientConfig) EndpointType(endpointType string) *OpenstackClientConfig { + cfg.endpointType = endpointType + return cfg +} + +func (cfg *OpenstackClientConfig) Debug(debug bool) *OpenstackClientConfig { + cfg.debug = debug + return cfg +} + +type SOpenStackClient struct { + *OpenstackClientConfig + + client *mcclient.Client + tokenCredential mcclient.TokenCredential + iregions []cloudprovider.ICloudRegion +} + +func NewOpenStackClient(cfg *OpenstackClientConfig) (*SOpenStackClient, error) { + cli := &SOpenStackClient{ + OpenstackClientConfig: cfg, } return cli, cli.fetchRegions() } func (cli *SOpenStackClient) GetCloudRegionExternalIdPrefix() string { - return fmt.Sprintf("%s/%s/", CLOUD_PROVIDER_OPENSTACK, cli.providerID) + return fmt.Sprintf("%s/%s/", CLOUD_PROVIDER_OPENSTACK, cli.cpcfg.Id) } func (cli *SOpenStackClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { subAccount := cloudprovider.SSubAccount{ Account: fmt.Sprintf("%s/%s", cli.project, cli.username), - Name: cli.providerName, + Name: cli.cpcfg.Name, } if len(cli.domainName) > 0 { subAccount.Account = fmt.Sprintf("%s/%s", subAccount.Account, cli.domainName) @@ -194,7 +222,7 @@ func (cli *SOpenStackClient) getVersion(region string, service string) (string, } func (cli *SOpenStackClient) connect() error { - cli.client = mcclient.NewClient(cli.authURL, 5, cli.Debug, false, "", "") + cli.client = mcclient.NewClient(cli.authURL, 5, cli.debug, false, "", "") tokenCredential, err := cli.client.Authenticate(cli.username, cli.password, cli.domainName, cli.project, cli.projectDomain) if err != nil { return err diff --git a/pkg/multicloud/openstack/provider/provider.go b/pkg/multicloud/openstack/provider/provider.go index fd7c335d3a..68e46cffde 100644 --- a/pkg/multicloud/openstack/provider/provider.go +++ b/pkg/multicloud/openstack/provider/provider.go @@ -111,7 +111,18 @@ func (self *SOpenStackProviderFactory) GetProvider(cfg cloudprovider.ProviderCon if len(accountInfo) == 3 { domainName, projectDomainName = accountInfo[2], accountInfo[2] } - client, err := openstack.NewOpenStackClient(cfg.Id, cfg.Name, cfg.URL, username, cfg.Secret, project, endpointType, domainName, projectDomainName, false) + client, err := openstack.NewOpenStackClient( + openstack.NewOpenstackClientConfig( + cfg.URL, + username, + cfg.Secret, + project, + projectDomainName, + ). + DomainName(domainName). + EndpointType(endpointType). + CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/openstack/region.go b/pkg/multicloud/openstack/region.go index 6e2db336be..a0f018fe4e 100644 --- a/pkg/multicloud/openstack/region.go +++ b/pkg/multicloud/openstack/region.go @@ -59,11 +59,11 @@ func (region *SRegion) GetId() string { } func (region *SRegion) GetName() string { - return fmt.Sprintf("%s-%s", region.client.providerName, region.Name) + return fmt.Sprintf("%s-%s", region.client.cpcfg.Name, region.Name) } func (region *SRegion) GetGlobalId() string { - return fmt.Sprintf("%s/%s/%s", CLOUD_PROVIDER_OPENSTACK, region.client.providerID, region.Name) + return fmt.Sprintf("%s/%s/%s", CLOUD_PROVIDER_OPENSTACK, region.client.cpcfg.Id, region.Name) } func (region *SRegion) IsEmulated() bool { diff --git a/pkg/multicloud/openstack/storagecache.go b/pkg/multicloud/openstack/storagecache.go index 4579603474..9cf3ec1d9c 100644 --- a/pkg/multicloud/openstack/storagecache.go +++ b/pkg/multicloud/openstack/storagecache.go @@ -42,11 +42,11 @@ func (cache *SStoragecache) GetMetadata() *jsonutils.JSONDict { } func (cache *SStoragecache) GetId() string { - return fmt.Sprintf("%s-%s", cache.region.client.providerID, cache.region.GetId()) + return fmt.Sprintf("%s-%s", cache.region.client.cpcfg.Id, cache.region.GetId()) } func (cache *SStoragecache) GetName() string { - return fmt.Sprintf("%s-%s", cache.region.client.providerName, cache.region.GetId()) + return fmt.Sprintf("%s-%s", cache.region.client.cpcfg.Name, cache.region.GetId()) } func (cache *SStoragecache) GetStatus() string { @@ -58,7 +58,7 @@ func (cache *SStoragecache) Refresh() error { } func (cache *SStoragecache) GetGlobalId() string { - return fmt.Sprintf("%s-%s", cache.region.client.providerID, cache.region.GetGlobalId()) + return fmt.Sprintf("%s-%s", cache.region.client.cpcfg.Id, cache.region.GetGlobalId()) } func (cache *SStoragecache) IsEmulated() bool { From 6a7382f5be522f5a2719809d7740c70746b23844 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 10:31:15 +0800 Subject: [PATCH 26/42] mcclient: add SetHttpTransportProxyFunc() --- pkg/mcclient/mcclient.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/mcclient/mcclient.go b/pkg/mcclient/mcclient.go index 9966f6c49b..deccd06974 100644 --- a/pkg/mcclient/mcclient.go +++ b/pkg/mcclient/mcclient.go @@ -79,6 +79,10 @@ func (this *Client) HttpClient() *http.Client { return this.httpconn } +func (this *Client) SetHttpTransportProxyFunc(proxyFunc httputils.TransportProxyFunc) { + httputils.SetClientProxyFunc(this.httpconn, proxyFunc) +} + func (this *Client) SetDebug(debug bool) { this.debug = debug } From 4f270a3ff00e4d1cff05d9d5e4d5f51f1f72f6d6 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 10:31:48 +0800 Subject: [PATCH 27/42] openstack: use cpcfg.ProxyFunc --- pkg/multicloud/openstack/openstack.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/multicloud/openstack/openstack.go b/pkg/multicloud/openstack/openstack.go index 4011b483e8..99a91784ee 100644 --- a/pkg/multicloud/openstack/openstack.go +++ b/pkg/multicloud/openstack/openstack.go @@ -223,6 +223,7 @@ func (cli *SOpenStackClient) getVersion(region string, service string) (string, func (cli *SOpenStackClient) connect() error { cli.client = mcclient.NewClient(cli.authURL, 5, cli.debug, false, "", "") + cli.client.SetHttpTransportProxyFunc(cli.cpcfg.ProxyFunc) tokenCredential, err := cli.client.Authenticate(cli.username, cli.password, cli.domainName, cli.project, cli.projectDomain) if err != nil { return err From 4c40e3bf40929346baa11cd594e793c0007e9764 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 15:00:06 +0800 Subject: [PATCH 28/42] esxi: use ESXiClientConfig --- cmd/esxicli/main.go | 9 ++- pkg/multicloud/esxi/manager.go | 73 ++++++++++++++++++------ pkg/multicloud/esxi/provider/provider.go | 6 +- 3 files changed, 68 insertions(+), 20 deletions(-) diff --git a/cmd/esxicli/main.go b/cmd/esxicli/main.go index c7ad77b712..7114b230a5 100644 --- a/cmd/esxicli/main.go +++ b/cmd/esxicli/main.go @@ -88,7 +88,14 @@ func newClient(options *BaseOptions) (*esxi.SESXiClient, error) { return nil, fmt.Errorf("Missing password") } - return esxi.NewESXiClient2("", "", options.Host, options.Port, options.Account, options.Password, false) + return esxi.NewESXiClient2( + esxi.NewESXiClientConfig( + options.Host, + options.Port, + options.Account, + options.Password, + ), + ) } func main() { diff --git a/pkg/multicloud/esxi/manager.go b/pkg/multicloud/esxi/manager.go index 823ead0212..d9b7569871 100644 --- a/pkg/multicloud/esxi/manager.go +++ b/pkg/multicloud/esxi/manager.go @@ -57,30 +57,60 @@ func init() { defaultDc.ManagedEntity.ExtensibleManagedObject.Self.Value = defaultDcId } +type ESXiClientConfig struct { + cpcfg cloudprovider.ProviderConfig + + host string + port int + account string + password string + + managed bool +} + +func NewESXiClientConfig(host string, port int, account, password string) *ESXiClientConfig { + cfg := &ESXiClientConfig{ + host: host, + port: port, + account: account, + password: password, + } + return cfg +} + +func (cfg *ESXiClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *ESXiClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *ESXiClientConfig) Managed(managed bool) *ESXiClientConfig { + cfg.managed = managed + return cfg +} + type SESXiClient struct { + *ESXiClientConfig + cloudprovider.SFakeOnPremiseRegion multicloud.SRegion multicloud.SNoObjectStorageRegion - providerId string - providerName string - host string - port int - account string - password string - client *govmomi.Client - context context.Context + client *govmomi.Client + context context.Context datacenters []*SDatacenter } -func NewESXiClient(providerId string, providerName string, host string, port int, account string, passwd string) (*SESXiClient, error) { - return NewESXiClient2(providerId, providerName, host, port, account, passwd, true) +func NewESXiClient(cfg *ESXiClientConfig) (*SESXiClient, error) { + cfg.Managed(true) + return NewESXiClient2(cfg) } -func NewESXiClient2(providerId string, providerName string, host string, port int, account string, passwd string, managed bool) (*SESXiClient, error) { - cli := &SESXiClient{providerId: providerId, providerName: providerName, - host: host, port: port, account: account, password: passwd, context: context.Background()} +func NewESXiClient2(cfg *ESXiClientConfig) (*SESXiClient, error) { + cli := &SESXiClient{ + ESXiClientConfig: cfg, + context: context.Background(), + } err := cli.connect() if err != nil { @@ -90,7 +120,7 @@ func NewESXiClient2(providerId string, providerName string, host string, port in if !cli.IsVCenter() { err := cli.checkHostManagedByVCenter() if err != nil { - if managed { + if cfg.managed { cli.disconnect() return nil, err } else { @@ -118,7 +148,14 @@ func NewESXiClientFromAccessInfo(ctx context.Context, accessInfo *models.SVCente accessInfo.Password = tmp } } - client, err := NewESXiClient("", "", accessInfo.Host, accessInfo.Port, accessInfo.Account, accessInfo.Password) + client, err := NewESXiClient( + NewESXiClientConfig( + accessInfo.Host, + accessInfo.Port, + accessInfo.Account, + accessInfo.Password, + ).Managed(true), + ) if err != nil { return nil, err } @@ -180,7 +217,7 @@ func (cli *SESXiClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { } subAccount := cloudprovider.SSubAccount{ Account: cli.account, - Name: cli.providerName, + Name: cli.cpcfg.Name, HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL, } return []cloudprovider.SSubAccount{subAccount}, nil @@ -382,8 +419,8 @@ func (cli *SESXiClient) FindHostByMoId(moId string) (cloudprovider.ICloudHost, e } func (cli *SESXiClient) getPrivateId(idStr string) string { - if len(cli.providerId) > 0 && strings.HasPrefix(idStr, cli.providerId) { - idStr = idStr[len(cli.providerId)+1:] + if len(cli.cpcfg.Id) > 0 && strings.HasPrefix(idStr, cli.cpcfg.Id) { + idStr = idStr[len(cli.cpcfg.Id)+1:] } return idStr } diff --git a/pkg/multicloud/esxi/provider/provider.go b/pkg/multicloud/esxi/provider/provider.go index 5b0175f13f..31a55308af 100644 --- a/pkg/multicloud/esxi/provider/provider.go +++ b/pkg/multicloud/esxi/provider/provider.go @@ -111,7 +111,11 @@ func (self *SESXiProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) return nil, err } - client, err := esxi.NewESXiClient(cfg.Id, cfg.Name, host, port, cfg.Account, cfg.Secret) + client, err := esxi.NewESXiClient( + esxi.NewESXiClientConfig( + host, port, cfg.Account, cfg.Secret, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } From 1695947221efad0b54f1e0f50a66d30e57ad714e Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 15:16:41 +0800 Subject: [PATCH 29/42] esxi: use cpcfg.ProxyFunc --- pkg/multicloud/esxi/manager.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/pkg/multicloud/esxi/manager.go b/pkg/multicloud/esxi/manager.go index d9b7569871..cff7b4dcd8 100644 --- a/pkg/multicloud/esxi/manager.go +++ b/pkg/multicloud/esxi/manager.go @@ -26,8 +26,10 @@ import ( "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/session" "github.com/vmware/govmomi/view" + "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/methods" "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" "yunion.io/x/jsonutils" @@ -39,6 +41,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/httputils" ) const ( @@ -184,9 +187,20 @@ func (cli *SESXiClient) connect() error { return fmt.Errorf("Illegal url %s: %s", cli.url(), err) } - govmcli, err := govmomi.NewClient(cli.context, u, true) - if err != nil { - return err + var govmcli *govmomi.Client + { + insecure := true + soapCli := soap.NewClient(u, insecure) + httpClient := &soapCli.Client + httputils.SetClientProxyFunc(httpClient, cli.cpcfg.ProxyFunc) + vimCli, err := vim25.NewClient(cli.context, soapCli) + if err != nil { + return err + } + govmcli = &govmomi.Client{ + Client: vimCli, + SessionManager: session.NewManager(vimCli), + } } userinfo := url.UserPassword(cli.account, cli.password) From 06fad05278dc4f2df5362ef16dc9e724b2bd74a3 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 22:04:13 +0800 Subject: [PATCH 30/42] objectstore: use ObjectStoreClientConfig --- cmd/s3cli/main.go | 18 +++- pkg/multicloud/objectstore/ceph/ceph.go | 14 ++- .../objectstore/ceph/provider/provider.go | 7 +- pkg/multicloud/objectstore/objectstore.go | 88 ++++++++++++++----- .../objectstore/provider/provider.go | 5 +- .../objectstore/xsky/provider/provider.go | 13 ++- pkg/multicloud/objectstore/xsky/xsky.go | 20 +++-- 7 files changed, 126 insertions(+), 39 deletions(-) diff --git a/cmd/s3cli/main.go b/cmd/s3cli/main.go index 474740df64..2c6fee1903 100644 --- a/cmd/s3cli/main.go +++ b/cmd/s3cli/main.go @@ -94,11 +94,23 @@ func newClient(options *BaseOptions) (cloudprovider.ICloudRegion, error) { } if options.Backend == api.CLOUD_PROVIDER_CEPH { - return ceph.NewCephRados("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug) + return ceph.NewCephRados( + objectstore.NewObjectStoreClientConfig( + options.AccessUrl, options.AccessKey, options.Secret, + ).Debug(options.Debug), + ) } else if options.Backend == api.CLOUD_PROVIDER_XSKY { - return xsky.NewXskyClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug) + return xsky.NewXskyClient( + objectstore.NewObjectStoreClientConfig( + options.AccessUrl, options.AccessKey, options.Secret, + ).Debug(options.Debug), + ) } - return objectstore.NewObjectStoreClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug) + return objectstore.NewObjectStoreClient( + objectstore.NewObjectStoreClientConfig( + options.AccessUrl, options.AccessKey, options.Secret, + ).Debug(options.Debug), + ) } func main() { diff --git a/pkg/multicloud/objectstore/ceph/ceph.go b/pkg/multicloud/objectstore/ceph/ceph.go index eba3013382..694db2cecb 100644 --- a/pkg/multicloud/objectstore/ceph/ceph.go +++ b/pkg/multicloud/objectstore/ceph/ceph.go @@ -38,12 +38,18 @@ type SCephRadosClient struct { userInfo *SUserInfo } -func NewCephRados(providerId string, providerName string, endpoint string, accessKey string, secret string, isDebug bool) (*SCephRadosClient, error) { - s3store, err := objectstore.NewObjectStoreClientAndFetch(providerId, providerName, endpoint, accessKey, secret, isDebug, false) +func NewCephRados(cfg *objectstore.ObjectStoreClientConfig) (*SCephRadosClient, error) { + s3store, err := objectstore.NewObjectStoreClientAndFetch(cfg, false) if err != nil { return nil, errors.Wrap(err, "NewObjectStoreClient") } - adminApi := newCephAdminApi(accessKey, secret, endpoint, isDebug, "") + adminApi := newCephAdminApi( + cfg.GetAccessKey(), + cfg.GetAccessSecret(), + cfg.GetEndpoint(), + cfg.GetDebug(), + "", + ) client := SCephRadosClient{ SObjectStoreClient: s3store, @@ -75,7 +81,7 @@ func NewCephRados(providerId string, providerName string, endpoint string, acces log.Errorf("adminApi.GetUserInfo fail: %s", err) } } - if isDebug { + if cfg.GetDebug() { log.Debugf("%#v %#v %#v", userQuota, bucketQuota, userInfo) } client.userQuota = userQuota diff --git a/pkg/multicloud/objectstore/ceph/provider/provider.go b/pkg/multicloud/objectstore/ceph/provider/provider.go index c4c60439ce..fc8935f66f 100644 --- a/pkg/multicloud/objectstore/ceph/provider/provider.go +++ b/pkg/multicloud/objectstore/ceph/provider/provider.go @@ -17,6 +17,7 @@ package provider import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/objectstore" "yunion.io/x/onecloud/pkg/multicloud/objectstore/ceph" s3provider "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider" ) @@ -34,7 +35,11 @@ func (self *SCephRadosProviderFactory) GetName() string { } func (self *SCephRadosProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { - client, err := ceph.NewCephRados(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) + client, err := ceph.NewCephRados( + objectstore.NewObjectStoreClientConfig( + cfg.URL, cfg.Account, cfg.Secret, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/objectstore/objectstore.go b/pkg/multicloud/objectstore/objectstore.go index 05773c6e0d..4c9e552a7f 100644 --- a/pkg/multicloud/objectstore/objectstore.go +++ b/pkg/multicloud/objectstore/objectstore.go @@ -31,42 +31,80 @@ import ( "yunion.io/x/onecloud/pkg/util/httputils" ) +type ObjectStoreClientConfig struct { + cpcfg cloudprovider.ProviderConfig + + endpoint string + accessKey string + accessSecret string + + debug bool +} + +func NewObjectStoreClientConfig(endpoint, accessKey, accessSecret string) *ObjectStoreClientConfig { + cfg := &ObjectStoreClientConfig{ + endpoint: endpoint, + accessKey: accessKey, + accessSecret: accessSecret, + } + return cfg +} + +func (cfg *ObjectStoreClientConfig) CloudproviderConfig(cpcfg cloudprovider.ProviderConfig) *ObjectStoreClientConfig { + cfg.cpcfg = cpcfg + return cfg +} + +func (cfg *ObjectStoreClientConfig) Debug(debug bool) *ObjectStoreClientConfig { + cfg.debug = debug + return cfg +} + +func (cfg *ObjectStoreClientConfig) GetCloudproviderConfig() cloudprovider.ProviderConfig { + return cfg.cpcfg +} + +func (cfg *ObjectStoreClientConfig) GetEndpoint() string { + return cfg.endpoint +} + +func (cfg *ObjectStoreClientConfig) GetAccessKey() string { + return cfg.accessKey +} + +func (cfg *ObjectStoreClientConfig) GetAccessSecret() string { + return cfg.accessSecret +} + +func (cfg *ObjectStoreClientConfig) GetDebug() bool { + return cfg.debug +} + type SObjectStoreClient struct { object.SObject + *ObjectStoreClientConfig + cloudprovider.SFakeOnPremiseRegion multicloud.SRegion - providerId string - providerName string - endpoint string - accessKey string - secret string - ownerId string ownerName string iBuckets []cloudprovider.ICloudBucket client *s3cli.Client - - Debug bool } -func NewObjectStoreClient(providerId string, providerName string, endpoint string, accessKey string, secret string, isDebug bool) (*SObjectStoreClient, error) { - return NewObjectStoreClientAndFetch(providerId, providerName, endpoint, accessKey, secret, isDebug, true) +func NewObjectStoreClient(cfg *ObjectStoreClientConfig) (*SObjectStoreClient, error) { + return NewObjectStoreClientAndFetch(cfg, true) } -func NewObjectStoreClientAndFetch(providerId string, providerName string, endpoint string, accessKey string, secret string, isDebug bool, doFetch bool) (*SObjectStoreClient, error) { +func NewObjectStoreClientAndFetch(cfg *ObjectStoreClientConfig, doFetch bool) (*SObjectStoreClient, error) { client := SObjectStoreClient{ - providerId: providerId, - providerName: providerName, - endpoint: endpoint, - accessKey: accessKey, - secret: secret, - Debug: isDebug, + ObjectStoreClientConfig: cfg, } - parts, err := url.Parse(endpoint) + parts, err := url.Parse(cfg.endpoint) if err != nil { return nil, errors.Wrap(err, "url.Parse endpoint") } @@ -74,7 +112,13 @@ func NewObjectStoreClientAndFetch(providerId string, providerName string, endpoi if parts.Scheme == "https" { useSsl = true } - cli, err := s3cli.New(parts.Host, accessKey, secret, useSsl, client.Debug) + cli, err := s3cli.New( + parts.Host, + client.accessKey, + client.accessSecret, + useSsl, + client.debug, + ) if err != nil { return nil, errors.Wrap(err, "minio.New") } @@ -85,7 +129,7 @@ func NewObjectStoreClientAndFetch(providerId string, providerName string, endpoi client.client = cli client.SetVirtualObject(&client) - if isDebug { + if client.debug { cli.TraceOn(os.Stderr) } @@ -102,7 +146,7 @@ func NewObjectStoreClientAndFetch(providerId string, providerName string, endpoi func (cli *SObjectStoreClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { subAccount := cloudprovider.SSubAccount{ Account: cli.accessKey, - Name: cli.providerName, + Name: cli.cpcfg.Name, HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL, } return []cloudprovider.SSubAccount{subAccount}, nil @@ -158,7 +202,7 @@ func (cli *SObjectStoreClient) S3Client() *s3cli.Client { func (cli *SObjectStoreClient) GetClientRC() map[string]string { return map[string]string{ "S3_ACCESS_KEY": cli.accessKey, - "S3_SECRET": cli.secret, + "S3_SECRET": cli.accessSecret, "S3_ACCESS_URL": cli.endpoint, "S3_BACKEND": api.CLOUD_PROVIDER_GENERICS3, } diff --git a/pkg/multicloud/objectstore/provider/provider.go b/pkg/multicloud/objectstore/provider/provider.go index cf3bb7eaf0..8d5060b0b0 100644 --- a/pkg/multicloud/objectstore/provider/provider.go +++ b/pkg/multicloud/objectstore/provider/provider.go @@ -72,9 +72,10 @@ func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ct } func (self *SObjectStoreProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { - dbg := false client, err := objectstore.NewObjectStoreClient( - cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, dbg, + objectstore.NewObjectStoreClientConfig( + cfg.URL, cfg.Account, cfg.Secret, + ).CloudproviderConfig(cfg), ) if err != nil { return nil, err diff --git a/pkg/multicloud/objectstore/xsky/provider/provider.go b/pkg/multicloud/objectstore/xsky/provider/provider.go index dfa7e4e3c7..66fd021c2a 100644 --- a/pkg/multicloud/objectstore/xsky/provider/provider.go +++ b/pkg/multicloud/objectstore/xsky/provider/provider.go @@ -17,6 +17,7 @@ package provider import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/objectstore" s3provider "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider" "yunion.io/x/onecloud/pkg/multicloud/objectstore/xsky" ) @@ -34,7 +35,11 @@ func (self *SXskyProviderFactory) GetName() string { } func (self *SXskyProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { - client, err := xsky.NewXskyClient(cfg.Id, cfg.Name, cfg.URL, cfg.Account, cfg.Secret, false) + client, err := xsky.NewXskyClient( + objectstore.NewObjectStoreClientConfig( + cfg.URL, cfg.Account, cfg.Secret, + ).CloudproviderConfig(cfg), + ) if err != nil { return nil, err } @@ -42,7 +47,11 @@ func (self *SXskyProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) } func (self *SXskyProviderFactory) GetClientRC(url, account, secret string) (map[string]string, error) { - client, err := xsky.NewXskyClient("", "", url, account, secret, false) + client, err := xsky.NewXskyClient( + objectstore.NewObjectStoreClientConfig( + url, account, secret, + ), + ) if err != nil { return nil, err } diff --git a/pkg/multicloud/objectstore/xsky/xsky.go b/pkg/multicloud/objectstore/xsky/xsky.go index 4cd98a53d5..888312dbbf 100644 --- a/pkg/multicloud/objectstore/xsky/xsky.go +++ b/pkg/multicloud/objectstore/xsky/xsky.go @@ -46,9 +46,14 @@ func parseAccount(account string) (user string, accessKey string) { return } -func NewXskyClient(providerId string, providerName string, endpoint string, account string, password string, isDebug bool) (*SXskyClient, error) { - usrname, accessKey := parseAccount(account) - adminApi := newXskyAdminApi(usrname, password, endpoint, isDebug) +func NewXskyClient(cfg *objectstore.ObjectStoreClientConfig) (*SXskyClient, error) { + usrname, accessKey := parseAccount(cfg.GetEndpoint()) + adminApi := newXskyAdminApi( + usrname, + cfg.GetAccessSecret(), + cfg.GetEndpoint(), + cfg.GetDebug(), + ) gwEp, err := adminApi.getS3GatewayEndpoint(context.Background()) if err != nil { return nil, errors.Wrap(err, "adminApi.getS3GatewayIP") @@ -68,7 +73,12 @@ func NewXskyClient(providerId string, providerName string, endpoint string, acco } } - s3store, err := objectstore.NewObjectStoreClientAndFetch(providerId, providerName, gwEp, accessKey, key.SecretKey, isDebug, false) + s3store, err := objectstore.NewObjectStoreClientAndFetch( + objectstore.NewObjectStoreClientConfig( + gwEp, accessKey, key.SecretKey, + ).Debug(cfg.GetDebug()).CloudproviderConfig(cfg.GetCloudproviderConfig()), + false, + ) if err != nil { return nil, errors.Wrap(err, "NewObjectStoreClient") } @@ -80,7 +90,7 @@ func NewXskyClient(providerId string, providerName string, endpoint string, acco } if len(accessKey) > 0 { - client.initAccount = account + client.initAccount = cfg.GetAccessKey() } client.SetVirtualObject(&client) From 462b832e1a3cd6c4eb14f4b4b0f6e8f0ae01cc22 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 22:08:40 +0800 Subject: [PATCH 31/42] objectstore: use cpcfg.ProxyFunc --- pkg/multicloud/objectstore/ceph/adminapi.go | 4 ++++ pkg/multicloud/objectstore/ceph/ceph.go | 2 ++ pkg/multicloud/objectstore/objectstore.go | 1 + pkg/multicloud/objectstore/xsky/adminapi.go | 4 ++++ pkg/multicloud/objectstore/xsky/xsky.go | 3 +++ 5 files changed, 14 insertions(+) diff --git a/pkg/multicloud/objectstore/ceph/adminapi.go b/pkg/multicloud/objectstore/ceph/adminapi.go index f32cd786eb..1cb76f0d20 100644 --- a/pkg/multicloud/objectstore/ceph/adminapi.go +++ b/pkg/multicloud/objectstore/ceph/adminapi.go @@ -63,6 +63,10 @@ func getJsonBodyReader(body jsonutils.JSONObject) io.Reader { return reqBody } +func (api *SCephAdminApi) httpClient() *http.Client { + return api.client +} + func (api *SCephAdminApi) jsonRequest(ctx context.Context, method httputils.THttpMethod, path string, hdr http.Header, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { urlStr := strings.TrimRight(api.endpoint, "/") + "/" + strings.TrimLeft(path, "/") req, err := http.NewRequest(string(method), urlStr, getJsonBodyReader(body)) diff --git a/pkg/multicloud/objectstore/ceph/ceph.go b/pkg/multicloud/objectstore/ceph/ceph.go index 694db2cecb..ff4a5ee20d 100644 --- a/pkg/multicloud/objectstore/ceph/ceph.go +++ b/pkg/multicloud/objectstore/ceph/ceph.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/multicloud/objectstore" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SCephRadosClient struct { @@ -50,6 +51,7 @@ func NewCephRados(cfg *objectstore.ObjectStoreClientConfig) (*SCephRadosClient, cfg.GetDebug(), "", ) + httputils.SetClientProxyFunc(adminApi.httpClient(), cfg.GetCloudproviderConfig().ProxyFunc) client := SCephRadosClient{ SObjectStoreClient: s3store, diff --git a/pkg/multicloud/objectstore/objectstore.go b/pkg/multicloud/objectstore/objectstore.go index 4c9e552a7f..39accaf8a7 100644 --- a/pkg/multicloud/objectstore/objectstore.go +++ b/pkg/multicloud/objectstore/objectstore.go @@ -124,6 +124,7 @@ func NewObjectStoreClientAndFetch(cfg *ObjectStoreClientConfig, doFetch bool) (* } tr := httputils.GetTransport(true) + tr.Proxy = cfg.cpcfg.ProxyFunc cli.SetCustomTransport(tr) client.client = cli diff --git a/pkg/multicloud/objectstore/xsky/adminapi.go b/pkg/multicloud/objectstore/xsky/adminapi.go index 02d46404aa..319115aeb5 100644 --- a/pkg/multicloud/objectstore/xsky/adminapi.go +++ b/pkg/multicloud/objectstore/xsky/adminapi.go @@ -60,6 +60,10 @@ func getJsonBodyReader(body jsonutils.JSONObject) io.Reader { return reqBody } +func (api *SXskyAdminApi) httpClient() *http.Client { + return api.client +} + func (api *SXskyAdminApi) jsonRequest(ctx context.Context, method httputils.THttpMethod, path string, hdr http.Header, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { urlStr := strings.TrimRight(api.endpoint, "/") + "/" + strings.TrimLeft(path, "/") req, err := http.NewRequest(string(method), urlStr, getJsonBodyReader(body)) diff --git a/pkg/multicloud/objectstore/xsky/xsky.go b/pkg/multicloud/objectstore/xsky/xsky.go index 888312dbbf..2eb9712850 100644 --- a/pkg/multicloud/objectstore/xsky/xsky.go +++ b/pkg/multicloud/objectstore/xsky/xsky.go @@ -26,6 +26,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud/objectstore" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SXskyClient struct { @@ -54,6 +55,8 @@ func NewXskyClient(cfg *objectstore.ObjectStoreClientConfig) (*SXskyClient, erro cfg.GetEndpoint(), cfg.GetDebug(), ) + httputils.SetClientProxyFunc(adminApi.httpClient(), cfg.GetCloudproviderConfig().ProxyFunc) + gwEp, err := adminApi.getS3GatewayEndpoint(context.Background()) if err != nil { return nil, errors.Wrap(err, "adminApi.getS3GatewayIP") From 1d78192a36e83c3ed5edecc64054464705ac54e6 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 11:44:54 +0800 Subject: [PATCH 32/42] cloudprovider: add TestProxyFunc --- pkg/multicloud/loader/proxyfunc_test.go | 114 ++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 pkg/multicloud/loader/proxyfunc_test.go diff --git a/pkg/multicloud/loader/proxyfunc_test.go b/pkg/multicloud/loader/proxyfunc_test.go new file mode 100644 index 0000000000..50cae39319 --- /dev/null +++ b/pkg/multicloud/loader/proxyfunc_test.go @@ -0,0 +1,114 @@ +package loader + +import ( + "net/http" + "net/url" + "testing" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +func TestProxyFunc(t *testing.T) { + runT := func(t *testing.T, cpcfg *cloudprovider.ProviderConfig) { + vendor := cpcfg.Vendor + proxied := false + proxyFunc := func(req *http.Request) (*url.URL, error) { + proxied = true + return url.Parse("http://TestProxyFunc" + "." + vendor + "/") + } + cpcfg.ProxyFunc = proxyFunc + _, err := cloudprovider.GetProvider(*cpcfg) + if !proxied { + if err != nil { + t.Logf("vendor %s: err: %v", vendor, err) + } + t.Errorf("vendor %s: proxyFunc not working", vendor) + } + } + + t.Parallel() + t.Run("cloud", func(t *testing.T) { + cpcfgs := map[string]*cloudprovider.ProviderConfig{ + compute.CLOUD_PROVIDER_VMWARE: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_QCLOUD: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_AZURE: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_HUAWEI: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_OPENSTACK: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_ZSTACK: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_UCLOUD: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_ALIYUN: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_AWS: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_CTYUN: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_GOOGLE: &cloudprovider.ProviderConfig{}, + } + for vendor := range cpcfgs { + cpcfgs[vendor].Vendor = vendor + cpcfgs[vendor].Id = vendor + "-Id" + cpcfgs[vendor].Name = vendor + "-Name" + cpcfgs[vendor].URL = "http://" + vendor + ".AccessUrl/TestProxyFunc" + cpcfgs[vendor].Account = vendor + "-Account" + cpcfgs[vendor].Secret = vendor + "-Secret" + } + + if true { + cpcfgs[compute.CLOUD_PROVIDER_OPENSTACK].Account = "projectId/username/domainName" + } + if true { + cpcfgs[compute.CLOUD_PROVIDER_AZURE].URL = "AzureChinaCloud" + cpcfgs[compute.CLOUD_PROVIDER_AZURE].Account = "tenantId/subscriptionId" + cpcfgs[compute.CLOUD_PROVIDER_AZURE].Secret = "appId/appKey" + } + if true { + cpcfgs[compute.CLOUD_PROVIDER_UCLOUD].Account = "accessKey::projectId" + } + if true { + const pkey = `-----BEGIN RSA PRIVATE KEY-----` + "\n" + + `MIICXgIBAAKBgQDIUZ819CKNzPf3dL1aTC9tx6Y+Z/wQ74hWgyxM4DM5kxMZDoWd` + "\n" + + `2zzj7B8vUU64tYaeCaPFahxcMNs705wNACkFnmqop08zUSWTfbPg/uNdhGvuF0vT` + "\n" + + `3o5oNbUfVLrusvVJwP6mAnfJsECZJsPMqVIyM5z6uMBpAkjsaqEP7cOISwIDAQAB` + "\n" + + `AoGBAIzBXZ/ptcXxumM0skCau9DlspizNFkqFqDDdwNlnljcUVUP8S9kd7qnlZoI` + "\n" + + `BcqgMsElgOAcmWTmJ4Y4QLSZ4jgDthDUqt+dv384G4tUHv5LDU4SMnKiPFzqzsOn` + "\n" + + `PR72ZcoZZZW9PvNMmJDWSIwuEXgWovXhK5pO3wuuHUDjPHlxAkEA8DdcBRAV9yuq` + "\n" + + `AbVFSWxBIpNkXkIoOCQiBOP8LBTvua5Dkuxe5qZkDXuTupYmLRmIoDJuZVKo0jWw` + "\n" + + `6xBg1Io4BQJBANV7J3u5HV7IxlG35g8iCsU/AnLYQYWIWqF9+DKM8fqkDHGx6KGl` + "\n" + + `THG0gKGjhkWe/qzTsQbe4gHWZp6z1apVQA8CQQC9aDkzeMKJGTG8lQzj3urg82zx` + "\n" + + `caK62arFRmrA62U2eHSptZ0CqvI7O1R5jAjgCMEU547qb6nTzigIwOpIzA9tAkB6` + "\n" + + `EyaM1Vo6LU1axXJFDQ5jpJhd29P76/EHj+Ux4u8siEqlaTrB1KhWDQXRaJULktPH` + "\n" + + `OHZum11Z96RO9D8mXAF5AkEAq1xtCrJp8rWTGH5PGShyCOk2ZNbV0AL+may4FAgc` + "\n" + + `hbZ+WzxylVxMKJmWqEBAYF3/7oouyteF8Vq3TVOv442NSg==` + "\n" + + `-----END RSA PRIVATE KEY-----` + cpcfgs[compute.CLOUD_PROVIDER_GOOGLE].Account = "projectId/email" + cpcfgs[compute.CLOUD_PROVIDER_GOOGLE].Secret = "keyid/" + pkey + } + for _, cpcfg := range cpcfgs { + cpcfg := cpcfg + t.Run(cpcfg.Vendor, func(t *testing.T) { + //t.Parallel() + runT(t, cpcfg) + }) + } + }) + t.Run("objectstore", func(t *testing.T) { + cpcfgs := map[string]*cloudprovider.ProviderConfig{ + compute.CLOUD_PROVIDER_CEPH: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_XSKY: &cloudprovider.ProviderConfig{}, + compute.CLOUD_PROVIDER_GENERICS3: &cloudprovider.ProviderConfig{}, + } + for vendor := range cpcfgs { + cpcfgs[vendor].Vendor = vendor + cpcfgs[vendor].Id = vendor + "-Id" + cpcfgs[vendor].Name = vendor + "-Name" + cpcfgs[vendor].URL = "http://" + vendor + ".AccessUrl/TestProxyFunc" + cpcfgs[vendor].Account = vendor + "-Account" + cpcfgs[vendor].Secret = vendor + "-Secret" + } + for _, cpcfg := range cpcfgs { + cpcfg := cpcfg + t.Run(cpcfg.Vendor, func(t *testing.T) { + //t.Parallel() + runT(t, cpcfg) + }) + } + }) +} From cf7c63be3fbb86ff41cb13c4652a29252fbbb144 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 25 Mar 2020 17:26:18 +0800 Subject: [PATCH 33/42] region: add cloudaccount.ProxySettingId --- pkg/apis/compute/cloudaccount.go | 3 ++ pkg/apis/compute/cloudprovider.go | 3 ++ pkg/apis/compute/zz_generated.model.go | 3 ++ pkg/compute/models/cloudaccounts.go | 69 ++++++++++++++++++++++---- pkg/compute/models/cloudproviders.go | 25 +++++++--- pkg/compute/models/initdb.go | 3 ++ pkg/compute/service/handlers.go | 4 ++ 7 files changed, 95 insertions(+), 15 deletions(-) diff --git a/pkg/apis/compute/cloudaccount.go b/pkg/apis/compute/cloudaccount.go index f66cd08cad..343746ceac 100644 --- a/pkg/apis/compute/cloudaccount.go +++ b/pkg/apis/compute/cloudaccount.go @@ -79,6 +79,9 @@ type CloudaccountCreateInput struct { // 额外信息,例如账单的access key Options *jsonutils.JSONObject `json:"options"` + // 代理配置 + ProxySettingId string `json:"proxy_setting_id"` + cloudprovider.SCloudaccount cloudprovider.SCloudaccountCredential } diff --git a/pkg/apis/compute/cloudprovider.go b/pkg/apis/compute/cloudprovider.go index cfbe6f9ea1..4fded746dd 100644 --- a/pkg/apis/compute/cloudprovider.go +++ b/pkg/apis/compute/cloudprovider.go @@ -18,6 +18,7 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis" + proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -96,6 +97,8 @@ type CloudproviderDetails struct { apis.StandaloneResourceDetails SCloudprovider + ProxySetting proxyapi.SProxySetting `json:"proxy_setting"` + // 云账号名称 // example: google-account Cloudaccount string `json:"cloudaccount"` diff --git a/pkg/apis/compute/zz_generated.model.go b/pkg/apis/compute/zz_generated.model.go index d0e858f3d3..a8c3d22a42 100644 --- a/pkg/apis/compute/zz_generated.model.go +++ b/pkg/apis/compute/zz_generated.model.go @@ -195,6 +195,8 @@ type SCloudaccount struct { IsPublic bool `json:"is_public"` // add share_mode field to indicate the share range of this account ShareMode string `json:"share_mode"` + // 默认值proxyapi.ProxySettingId_DIRECT + ProxySettingId string `json:"proxy_setting_id"` } // SCloudprovider is an autogenerated struct via yunion.io/x/onecloud/pkg/compute/models.SCloudprovider. @@ -215,6 +217,7 @@ type SCloudprovider struct { type SCloudproviderCapability struct { apis.SResourceBase CloudproviderId string `json:"cloudprovider_id"` + CloudregionId string `json:"cloudregion_id"` Capability string `json:"capability"` } diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 622b793b55..5d5b67a523 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -32,10 +32,12 @@ import ( "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" + proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" + "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudcommon/policy" "yunion.io/x/onecloud/pkg/cloudcommon/validators" @@ -46,6 +48,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/choices" + "yunion.io/x/onecloud/pkg/util/httputils" "yunion.io/x/onecloud/pkg/util/logclient" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/stringutils2" @@ -149,7 +152,8 @@ type SCloudaccount struct { // add share_mode field to indicate the share range of this account ShareMode string `width:"32" charset:"ascii" nullable:"true" list:"domain"` - ProxySettingId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"optional" default:"default"` + // 默认值proxyapi.ProxySettingId_DIRECT + ProxySettingId string `width:"36" charset:"ascii" nullable:"false" list:"domain" create:"optional" update:"domain" default:"DIRECT"` } func (self *SCloudaccountManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { @@ -307,6 +311,16 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl } data.Set("options", optionsJson) } + + v := validators.NewModelIdOrNameValidator( + "proxy_setting", + proxy.ProxySettingManager.KeywordPlural(), + userCred, + ) + if err := v.Validate(data); err != nil { + return nil, err + } + return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) } @@ -364,11 +378,26 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use return input, httperrors.NewConflictError("The account has been registered") } + var proxyFunc httputils.TransportProxyFunc + { + if input.ProxySettingId == "" { + input.ProxySettingId = proxyapi.ProxySettingId_DIRECT + } + m, err := proxy.ProxySettingManager.FetchByIdOrName(userCred, input.ProxySettingId) + if err != nil { + return input, httperrors.NewInputParameterError("fetch proxysetting %s: %s", + input.ProxySettingId, err) + } + proxySetting := m.(*proxy.SProxySetting) + input.ProxySettingId = proxySetting.Id + proxyFunc = proxySetting.HttpTransportProxyFunc() + } accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{ - Vendor: input.Provider, - URL: input.AccessUrl, - Account: input.Account, - Secret: input.Secret, + Vendor: input.Provider, + URL: input.AccessUrl, + Account: input.Account, + Secret: input.Secret, + ProxyFunc: proxyFunc, }) if err != nil { if err == cloudprovider.ErrNoSuchProvder { @@ -524,10 +553,11 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred originSecret, _ := self.getPassword() accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{ - Vendor: self.Provider, - URL: self.AccessUrl, - Account: account.Account, - Secret: account.Secret, + Vendor: self.Provider, + URL: self.AccessUrl, + Account: account.Account, + Secret: account.Secret, + ProxyFunc: self.proxyFunc(), }) if err != nil { return nil, httperrors.NewInputParameterError("invalid cloud account info error: %s", err.Error()) @@ -680,6 +710,25 @@ func (self *SCloudaccount) GetProvider() (cloudprovider.ICloudProvider, error) { return self.getProviderInternal() } +func (self *SCloudaccount) proxySetting() *proxy.SProxySetting { + m, err := proxy.ProxySettingManager.FetchById(self.ProxySettingId) + if err != nil { + log.Errorf("cloudaccount %s(%s): get proxysetting %s: %v", + self.Name, self.Id, self.ProxySettingId, err) + return nil + } + ps := m.(*proxy.SProxySetting) + return ps +} + +func (self *SCloudaccount) proxyFunc() httputils.TransportProxyFunc { + ps := self.proxySetting() + if ps != nil { + return ps.HttpTransportProxyFunc() + } + return nil +} + func (self *SCloudaccount) getProviderInternal() (cloudprovider.ICloudProvider, error) { secret, err := self.getPassword() if err != nil { @@ -692,6 +741,8 @@ func (self *SCloudaccount) getProviderInternal() (cloudprovider.ICloudProvider, URL: self.AccessUrl, Account: self.Account, Secret: secret, + + ProxyFunc: self.proxyFunc(), }) } diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index 4b642a619c..c44ea5da58 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -737,13 +737,17 @@ func (self *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, error) if err != nil { return nil, err } + + account := self.GetCloudaccount() + return cloudprovider.GetProvider(cloudprovider.ProviderConfig{ - Id: self.Id, - Name: self.Name, - Vendor: self.Provider, - URL: accessUrl, - Account: self.Account, - Secret: passwd, + Id: self.Id, + Name: self.Name, + Vendor: self.Provider, + URL: accessUrl, + Account: self.Account, + Secret: passwd, + ProxyFunc: account.proxyFunc(), }) } @@ -878,7 +882,16 @@ func (self *SCloudprovider) getMoreDetails(ctx context.Context, out api.Cloudpro // 此字段不能删除，公有云日志同步需要这个字段 out.Brand = account.Brand out.Cloudaccount = account.GetName() + ps := account.proxySetting() + if ps != nil { + out.ProxySetting.Id = ps.Id + out.ProxySetting.Name = ps.Name + out.ProxySetting.HTTPProxy = ps.HTTPProxy + out.ProxySetting.HTTPSProxy = ps.HTTPSProxy + out.ProxySetting.NoProxy = ps.NoProxy + } } + out.SyncStatus2 = self.getSyncStatus2() capabilities, _ := CloudproviderCapabilityManager.getCapabilities(self.Id) if len(capabilities) > 0 { diff --git a/pkg/compute/models/initdb.go b/pkg/compute/models/initdb.go index 2fa7b8b185..7deee65868 100644 --- a/pkg/compute/models/initdb.go +++ b/pkg/compute/models/initdb.go @@ -18,6 +18,7 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy" ) func InitDB() error { @@ -28,6 +29,8 @@ func InitDB() error { */ db.TenantCacheManager, + proxy.ProxySettingManager, + QuotaManager, CloudproviderManager, diff --git a/pkg/compute/service/handlers.go b/pkg/compute/service/handlers.go index d8dbc461d9..d0c1253484 100644 --- a/pkg/compute/service/handlers.go +++ b/pkg/compute/service/handlers.go @@ -18,6 +18,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/appsrv/dispatcher" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/capabilities" @@ -81,6 +82,9 @@ func InitHandlers(app *appsrv.Application) { for _, manager := range []db.IModelManager{ db.OpsLog, db.Metadata, + + proxy.ProxySettingManager, + models.BucketManager, models.CloudaccountManager, models.CloudproviderManager, From adfad567f615cac985e87aea559d4f4812fc771a Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 19 Mar 2020 21:32:56 +0800 Subject: [PATCH 34/42] s3gateway: use cpcfg.ProxyFunc --- pkg/s3gateway/models/cloudproviders.go | 33 +++++++++++++++++++++----- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/pkg/s3gateway/models/cloudproviders.go b/pkg/s3gateway/models/cloudproviders.go index 866fe331a0..16d346392a 100644 --- a/pkg/s3gateway/models/cloudproviders.go +++ b/pkg/s3gateway/models/cloudproviders.go @@ -16,17 +16,23 @@ package models import ( "context" + "net/http" + "net/url" "time" + "golang.org/x/net/http/httpproxy" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/utils" + proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/s3gateway/session" "yunion.io/x/onecloud/pkg/util/hashcache" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SCloudproviderManagerDelegate struct { @@ -93,6 +99,8 @@ type SCloudproviderDelegate struct { Provider string Brand string + + ProxySetting proxyapi.SProxySetting } func (manager *SCloudproviderManagerDelegate) GetById(ctx context.Context, userCred mcclient.TokenCredential, id string) (*SCloudproviderDelegate, error) { @@ -136,12 +144,25 @@ func (provider *SCloudproviderDelegate) GetProvider() (cloudprovider.ICloudProvi if err != nil { return nil, err } + var proxyFunc httputils.TransportProxyFunc + { + cfg := &httpproxy.Config{ + HTTPProxy: provider.ProxySetting.HTTPProxy, + HTTPSProxy: provider.ProxySetting.HTTPSProxy, + NoProxy: provider.ProxySetting.NoProxy, + } + cfgProxyFunc := cfg.ProxyFunc() + proxyFunc = func(req *http.Request) (*url.URL, error) { + return cfgProxyFunc(req.URL) + } + } return cloudprovider.GetProvider(cloudprovider.ProviderConfig{ - Id: provider.Id, - Name: provider.Name, - Vendor: provider.Provider, - URL: accessUrl, - Account: provider.Account, - Secret: passwd, + Id: provider.Id, + Name: provider.Name, + Vendor: provider.Provider, + URL: accessUrl, + Account: provider.Account, + Secret: passwd, + ProxyFunc: proxyFunc, }) } From 644482533143ee226653f985aa837d203d999992 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 20 Mar 2020 00:07:09 +0800 Subject: [PATCH 35/42] cloudevent: add proxy.ProxySettingManager --- pkg/cloudevent/models/initdb.go | 3 +++ pkg/cloudevent/service/handlers.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/pkg/cloudevent/models/initdb.go b/pkg/cloudevent/models/initdb.go index ddc91143fd..c8fa7f8655 100644 --- a/pkg/cloudevent/models/initdb.go +++ b/pkg/cloudevent/models/initdb.go @@ -18,6 +18,7 @@ import ( "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy" ) func InitDB() error { @@ -28,6 +29,8 @@ func InitDB() error { */ db.TenantCacheManager, + proxy.ProxySettingManager, + CloudproviderManager, CloudeventManager, } { diff --git a/pkg/cloudevent/service/handlers.go b/pkg/cloudevent/service/handlers.go index 526abecc0a..644bf4022e 100644 --- a/pkg/cloudevent/service/handlers.go +++ b/pkg/cloudevent/service/handlers.go @@ -32,6 +32,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/appsrv/dispatcher" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/proxy" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/cloudevent/models" ) @@ -54,6 +55,7 @@ func InitHandlers(app *appsrv.Application) { for _, manager := range []db.IModelManager{ db.OpsLog, + proxy.ProxySettingManager, models.CloudeventManager, } { db.RegisterModelManager(manager) From 5813d64a7daa493b877fca53fbecf9f9ceb1cfd9 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 20 Mar 2020 22:43:17 +0800 Subject: [PATCH 36/42] cloudcommon: proxysetting: referrerMen --- pkg/cloudcommon/db/proxy/proxysetting.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pkg/cloudcommon/db/proxy/proxysetting.go b/pkg/cloudcommon/db/proxy/proxysetting.go index e466c4389b..912966e922 100644 --- a/pkg/cloudcommon/db/proxy/proxysetting.go +++ b/pkg/cloudcommon/db/proxy/proxysetting.go @@ -66,6 +66,23 @@ func (ps *SProxySetting) HttpTransportProxyFunc() httputils.TransportProxyFunc { } } +func (ps *SProxySetting) ValidateDeleteCondition(ctx context.Context) error { + for _, man := range referrersMen { + t := man.TableSpec().Instance() + n, err := t.Query(). + Equals("proxy_setting_id", ps.Id). + CountWithError() + if err != nil { + return httperrors.NewInternalServerError("get proxysetting refcount fail %s", err) + } + if n > 0 { + return httperrors.NewResourceBusyError("proxysetting %s is still referred to by %d %s", + ps.Id, n, man.KeywordPlural()) + } + } + return nil +} + func (man *SProxySettingManager) InitializeData() error { _, err := man.FetchById(proxyapi.ProxySettingId_DIRECT) if err == nil { @@ -88,3 +105,9 @@ func (man *SProxySettingManager) InitializeData() error { } return nil } + +var referrersMen []db.IModelManager + +func RegisterReferrer(man db.IModelManager) { + referrersMen = append(referrersMen, man) +} From 0e5f5221f0d96da0bdb57ecbbf9b413cd5b27214 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Fri, 20 Mar 2020 22:43:52 +0800 Subject: [PATCH 37/42] region: refer to proxySetting --- pkg/compute/models/cloudaccounts.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 5d5b67a523..35a0ec020f 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -71,6 +71,8 @@ func init() { ), } CloudaccountManager.SetVirtualObject(CloudaccountManager) + + proxy.RegisterReferrer(CloudaccountManager) } type SCloudaccount struct { From 140ac46e678d5abb3546ea2ef646825cb69158cf Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Sun, 22 Mar 2020 18:02:01 +0800 Subject: [PATCH 38/42] cloudevent: use cloudprovider proxysetting --- pkg/cloudevent/models/cloudproviders.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/cloudevent/models/cloudproviders.go b/pkg/cloudevent/models/cloudproviders.go index 34817f340e..08a4116136 100644 --- a/pkg/cloudevent/models/cloudproviders.go +++ b/pkg/cloudevent/models/cloudproviders.go @@ -16,8 +16,12 @@ package models import ( "context" + "net/http" + "net/url" "time" + "golang.org/x/net/http/httpproxy" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -25,6 +29,7 @@ import ( "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" + proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -68,6 +73,8 @@ type SCloudprovider struct { Provider string `width:"64" charset:"ascii" list:"domain"` Brand string `width:"64" charset:"ascii" list:"domain"` + + ProxySetting *proxyapi.SProxySetting } func (manager *SCloudproviderManager) GetRegionCloudproviders(ctx context.Context, userCred mcclient.TokenCredential) ([]SCloudprovider, error) { @@ -299,6 +306,17 @@ func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, err if err != nil { return nil, err } + + ps := provider.ProxySetting + cfg := &httpproxy.Config{ + HTTPProxy: ps.HTTPProxy, + HTTPSProxy: ps.HTTPSProxy, + NoProxy: ps.NoProxy, + } + cfgProxyFunc := cfg.ProxyFunc() + proxyFunc := func(req *http.Request) (*url.URL, error) { + return cfgProxyFunc(req.URL) + } return cloudprovider.GetProvider( cloudprovider.ProviderConfig{ Id: provider.Id, @@ -307,6 +325,8 @@ func (provider *SCloudprovider) GetProvider() (cloudprovider.ICloudProvider, err URL: accessUrl, Account: provider.Account, Secret: passwd, + + ProxyFunc: proxyFunc, }, ) } From 291230694dad28719edf0ee482495731dcdc6b89 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Sun, 22 Mar 2020 18:06:45 +0800 Subject: [PATCH 39/42] mcclient: add proxysettings manager --- cmd/climc/shell/proxysettings.go | 73 +++++++++++++++++++++++ pkg/mcclient/modules/mod_proxysettings.go | 40 +++++++++++++ pkg/mcclient/options/proxysettings.go | 42 +++++++++++++ 3 files changed, 155 insertions(+) create mode 100644 cmd/climc/shell/proxysettings.go create mode 100644 pkg/mcclient/modules/mod_proxysettings.go create mode 100644 pkg/mcclient/options/proxysettings.go diff --git a/cmd/climc/shell/proxysettings.go b/cmd/climc/shell/proxysettings.go new file mode 100644 index 0000000000..f3f0d7bf81 --- /dev/null +++ b/cmd/climc/shell/proxysettings.go @@ -0,0 +1,73 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +func init() { + R(&options.ProxySettingCreateOptions{}, "proxysetting-create", "Create proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingCreateOptions) error { + params, err := options.StructToParams(opts) + if err != nil { + return err + } + proxysetting, err := modules.ProxySettings.Create(s, params) + if err != nil { + return err + } + printObject(proxysetting) + return nil + }) + R(&options.ProxySettingGetOptions{}, "proxysetting-show", "Show proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingGetOptions) error { + proxysetting, err := modules.ProxySettings.Get(s, opts.ID, nil) + if err != nil { + return err + } + printObject(proxysetting) + return nil + }) + R(&options.ProxySettingListOptions{}, "proxysetting-list", "List proxysettings", func(s *mcclient.ClientSession, opts *options.ProxySettingListOptions) error { + params, err := options.ListStructToParams(opts) + if err != nil { + return err + } + result, err := modules.ProxySettings.List(s, params) + if err != nil { + return err + } + printList(result, modules.ProxySettings.GetColumns(s)) + return nil + }) + R(&options.ProxySettingUpdateOptions{}, "proxysetting-update", "Update proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingUpdateOptions) error { + params, err := options.StructToParams(opts) + proxysetting, err := modules.ProxySettings.Update(s, opts.ID, params) + if err != nil { + return err + } + printObject(proxysetting) + return nil + }) + R(&options.ProxySettingDeleteOptions{}, "proxysetting-delete", "Delete proxysetting", func(s *mcclient.ClientSession, opts *options.ProxySettingDeleteOptions) error { + proxysetting, err := modules.ProxySettings.Delete(s, opts.ID, nil) + if err != nil { + return err + } + printObject(proxysetting) + return nil + }) +} diff --git a/pkg/mcclient/modules/mod_proxysettings.go b/pkg/mcclient/modules/mod_proxysettings.go new file mode 100644 index 0000000000..1d10347897 --- /dev/null +++ b/pkg/mcclient/modules/mod_proxysettings.go @@ -0,0 +1,40 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package modules + +import ( + "yunion.io/x/onecloud/pkg/mcclient/modulebase" +) + +type ProxySettingManager struct { + modulebase.ResourceManager +} + +var ( + ProxySettings ProxySettingManager +) + +func init() { + ProxySettings = ProxySettingManager{NewComputeManager("proxysetting", "proxysettings", + []string{ + "ID", + "Name", + "http_proxy", + "https_proxy", + "no_proxy", + }, + []string{})} + registerCompute(&ProxySettings) +} diff --git a/pkg/mcclient/options/proxysettings.go b/pkg/mcclient/options/proxysettings.go new file mode 100644 index 0000000000..f8838014ff --- /dev/null +++ b/pkg/mcclient/options/proxysettings.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +type ProxySettingCreateOptions struct { + NAME string + + HTTPProxy string + HTTPSProxy string + NoProxy string +} + +type ProxySettingGetOptions struct { + ID string `json:"-"` +} + +type ProxySettingUpdateOptions struct { + ID string `json:"-"` + Name string + + BackendGroup string +} + +type ProxySettingDeleteOptions struct { + ID string `json:"-"` +} + +type ProxySettingListOptions struct { + BaseListOptions +} From 1bcd3dcab671e842b5f66696819a0012b68d2f93 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 25 Mar 2020 11:58:52 +0800 Subject: [PATCH 40/42] mcclient: cloudaccounts: allow updating proxysetting --- pkg/mcclient/options/cloudaccounts.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index acf3039d92..76c5515157 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -227,8 +227,9 @@ type SCloudAccountUpdateBaseOptions struct { ID string `help:"ID or Name of cloud account" json:"-"` Name string `help:"New name to update"` - SyncIntervalSeconds int `help:"auto synchornize interval in seconds"` - AutoCreateProject *bool `help:"automatically create local project for new remote project"` + SyncIntervalSeconds int `help:"auto synchornize interval in seconds"` + AutoCreateProject *bool `help:"automatically create local project for new remote project"` + ProxySetting string `help:"proxy setting name or id"` Desc string `help:"Description" json:"description" token:"desc"` } From 0fb6de4d80222cdeb40c47778ee947fea52b2279 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 25 Mar 2020 12:00:37 +0800 Subject: [PATCH 41/42] cloudaccounts: fix checking update data --- pkg/compute/models/cloudaccounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 35a0ec020f..4b70ef3777 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -316,7 +316,7 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl v := validators.NewModelIdOrNameValidator( "proxy_setting", - proxy.ProxySettingManager.KeywordPlural(), + proxy.ProxySettingManager.Keyword(), userCred, ) if err := v.Validate(data); err != nil { From 45a4848808b6bd21eebfec9a38820143a0ab6284 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 25 Mar 2020 12:30:11 +0800 Subject: [PATCH 42/42] climc: add cloud-account-update-google --- cmd/climc/shell/cloudaccounts.go | 13 +++++++++++++ pkg/mcclient/options/cloudaccounts.go | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/cmd/climc/shell/cloudaccounts.go b/cmd/climc/shell/cloudaccounts.go index 1aa62ea1f9..74ed40b35a 100644 --- a/cmd/climc/shell/cloudaccounts.go +++ b/cmd/climc/shell/cloudaccounts.go @@ -381,6 +381,19 @@ func init() { return nil }) + R(&options.SGoogleCloudAccountUpdateOptions{}, "cloud-account-update-google", "update a google cloud account", func(s *mcclient.ClientSession, args *options.SGoogleCloudAccountUpdateOptions) error { + params := jsonutils.Marshal(args).(*jsonutils.JSONDict) + if params.Size() == 0 { + return InvalidUpdateError() + } + result, err := modules.Cloudaccounts.Update(s, args.ID, params) + if err != nil { + return err + } + printObject(result) + return nil + }) + R(&options.SAWSCloudAccountUpdateOptions{}, "cloud-account-update-aws", "update an AWS cloud account", func(s *mcclient.ClientSession, args *options.SAWSCloudAccountUpdateOptions) error { params := jsonutils.Marshal(args).(*jsonutils.JSONDict) diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index 76c5515157..ed2c7ba57d 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -260,6 +260,10 @@ type SQcloudCloudAccountUpdateOptions struct { SCloudAccountUpdateBaseOptions } +type SGoogleCloudAccountUpdateOptions struct { + SCloudAccountUpdateBaseOptions +} + type SAWSCloudAccountUpdateOptions struct { SCloudAccountUpdateBaseOptions