From 29c8d86870850bb92c1973650f56d6d64e07c41d Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 5 Dec 2018 15:27:38 +0800 Subject: [PATCH 1/4] skus sync with account --- pkg/compute/skus/skus.go | 33 +++++++++++++++++--- pkg/compute/tasks/cloud_account_sync_task.go | 6 ++++ pkg/util/aws/image.go | 13 ++++++-- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/pkg/compute/skus/skus.go b/pkg/compute/skus/skus.go index 23cd3ce508..823d1fd3bc 100644 --- a/pkg/compute/skus/skus.go +++ b/pkg/compute/skus/skus.go @@ -208,6 +208,9 @@ func (self *SkusZone) getExternalZone() (string, string, string) { if len(parts) == 3 { // provider, region, zone return parts[0], parts[1], parts[2] + } else if len(parts) == 2 && parts[0] == models.CLOUD_PROVIDER_AZURE { + // azure 没有zone的概念 + return parts[0], parts[1], parts[1] } log.Debugf("SkusZone invalid external zone id %s", self.ExternalZoneId) @@ -235,9 +238,17 @@ func (self *SkusZoneList) initData(provider string, region models.SCloudregion, } } -func (self *SkusZoneList) Refresh() error { - provideIds := cloudprovider.GetRegistedProviderIds() - for _, p := range provideIds { +func (self *SkusZoneList) Refresh(providerIds *[]string) error { + self.Data = []*SkusZone{} + + var pIds []string + if providerIds == nil { + pIds = cloudprovider.GetRegistedProviderIds() + } else { + pIds = *providerIds + } + + for _, p := range pIds { regions, e := models.CloudregionManager.GetRegionByProvider(p) if e != nil { return e @@ -288,7 +299,7 @@ func (self *SkusZoneList) SyncToLocalDB() error { func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential) { skus := SkusZoneList{} - if e := skus.Refresh(); e != nil { + if e := skus.Refresh(nil); e != nil { log.Errorf("SyncSkus refresh failed, %s", e.Error()) } @@ -296,3 +307,17 @@ func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential) { log.Errorf("SyncSkus sync to local db failed, %s", e.Error()) } } + + +func SyncSkusByProviderIds(providerIds []string) error { + skus := SkusZoneList{} + if e := skus.Refresh(&providerIds); e != nil { + return fmt.Errorf("SyncSkus refresh failed, %s", e.Error()) + } + + if e := skus.SyncToLocalDB(); e != nil { + return fmt.Errorf("SyncSkus sync to local db failed, %s", e.Error()) + } + + return nil +} \ No newline at end of file diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go index 6806cf5508..ec5c79e24c 100644 --- a/pkg/compute/tasks/cloud_account_sync_task.go +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -7,6 +7,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/compute/skus" ) type CloudAccountSyncInfoTask struct { @@ -82,6 +83,11 @@ func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Con if account != nil { account.SetStatus(self.UserCred, models.CLOUD_PROVIDER_CONNECTED, "") } + + // sync skus + if err := skus.SyncSkusByProviderIds([]string{cloudprovider.GetId()}); err != nil { + self.SetStageFailed(ctx, err.Error()) + } } self.SetStageComplete(ctx, nil) } diff --git a/pkg/util/aws/image.go b/pkg/util/aws/image.go index 62fe86210a..2df0ae1f4b 100644 --- a/pkg/util/aws/image.go +++ b/pkg/util/aws/image.go @@ -87,7 +87,6 @@ func (self *SImage) GetStatus() string { } func (self *SImage) Refresh() error { - // todo: GetImage new, err := self.storageCache.region.GetImage(self.ImageId) if err != nil { return err @@ -173,7 +172,11 @@ func (self *SRegion) ExportImage(instanceId string, imageId string) (*ImageExpor } func (self *SRegion) GetImage(imageId string) (*SImage, error) { - images, _, err := self.GetImages("", ImageOwnerSelf, []string{imageId}, "", 0, 1) + if len(imageId) == 0 { + return nil, fmt.Errorf("image id should not be empty") + } + + images, _, err := self.GetImages("", ImageOwnerType(""), []string{imageId}, "", 0, 1) if err != nil { return nil, err } @@ -184,7 +187,11 @@ func (self *SRegion) GetImage(imageId string) (*SImage, error) { } func (self *SRegion) GetImageByName(name string) (*SImage, error) { - images, _, err := self.GetImages("", ImageOwnerSelf, nil, name, 0, 1) + if len(name) == 0 { + return nil, fmt.Errorf("image name should not be empty") + } + + images, _, err := self.GetImages("", ImageOwnerType(""), nil, name, 0, 1) if err != nil { return nil, err } From a0ea5fb76b213595ab0fd9510264210e6d10300d Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 5 Dec 2018 16:09:25 +0800 Subject: [PATCH 2/4] sku sync add limit & offset --- pkg/compute/skus/skus.go | 39 ++++++++++++++++++++------------ pkg/mcclient/modules/mod_skus.go | 4 ++-- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/pkg/compute/skus/skus.go b/pkg/compute/skus/skus.go index 823d1fd3bc..6276e0a813 100644 --- a/pkg/compute/skus/skus.go +++ b/pkg/compute/skus/skus.go @@ -103,32 +103,41 @@ func processSkuData(ndata jsonutils.JSONObject) jsonutils.JSONObject { func (self *SkusZone) Init() error { s := auth.GetAdminSession(options.Options.Region, "") p, r, z := self.getExternalZone() - - ret, e := modules.CloudmetaSkus.GetSkus(s, p, r, z) - if e != nil { - log.Debugf("SkusZone %s init failed, %s", z, e.Error()) - return e - } + limit := 1024 + offset := 0 + total := 1024 records := map[string]jsonutils.JSONObject{} - for _, sku := range ret.Data { - name, err := sku.GetString("name") - if err != nil { - log.Debugf("SkusZone sku name empty : %s", sku) - return err + for offset < total { + ret, e := modules.CloudmetaSkus.GetSkus(s, p, r, z, limit, offset) + if e != nil { + log.Debugf("SkusZone %s init failed, %s", z, e.Error()) + return e } - if odata, exists := records[name]; exists { - records[name] = mergeSkuData(odata, sku) - } else { - records[name] = processSkuData(sku) + for _, sku := range ret.Data { + name, err := sku.GetString("name") + if err != nil { + log.Debugf("SkusZone sku name empty : %s", sku) + return err + } + + if odata, exists := records[name]; exists { + records[name] = mergeSkuData(odata, sku) + } else { + records[name] = processSkuData(sku) + } } + + offset += limit + total = ret.Total } filtedData := []jsonutils.JSONObject{} for _, item := range records { filtedData = append(filtedData, item) } + self.total = len(records) self.skus = filtedData return nil diff --git a/pkg/mcclient/modules/mod_skus.go b/pkg/mcclient/modules/mod_skus.go index 7f3064e0ce..eac5cb8ebd 100644 --- a/pkg/mcclient/modules/mod_skus.go +++ b/pkg/mcclient/modules/mod_skus.go @@ -37,11 +37,11 @@ func init() { registerCompute(&ServerSkus) } -func (self *SkusManager) GetSkus(s *mcclient.ClientSession, providerId, regionId, zoneId string) (*ListResult, error) { +func (self *SkusManager) GetSkus(s *mcclient.ClientSession, providerId, regionId, zoneId string, limit, offset int) (*ListResult, error) { p := strings.ToLower(providerId) r := strings.ToLower(regionId) z := strings.ToLower(zoneId) - url := fmt.Sprintf("/providers/%s/regions/%s/zones/%s/skus", p, r, z) + url := fmt.Sprintf("/providers/%s/regions/%s/zones/%s/skus?limit=%d&offset=%d", p, r, z, limit, offset) ret, err := self._list(s, url, self.KeywordPlural) if err != nil { return &ListResult{}, err From 568f54c35e68f166dff5b1f3106a5197eee715c5 Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 5 Dec 2018 16:29:26 +0800 Subject: [PATCH 3/4] sync skus bugfix --- pkg/compute/skus/skus.go | 1 + pkg/compute/tasks/cloud_account_sync_task.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/compute/skus/skus.go b/pkg/compute/skus/skus.go index 6276e0a813..5505c3e8a4 100644 --- a/pkg/compute/skus/skus.go +++ b/pkg/compute/skus/skus.go @@ -320,6 +320,7 @@ func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential) { func SyncSkusByProviderIds(providerIds []string) error { skus := SkusZoneList{} + log.Debugf("SyncSkusByProviderIds %s", providerIds) if e := skus.Refresh(&providerIds); e != nil { return fmt.Errorf("SyncSkus refresh failed, %s", e.Error()) } diff --git a/pkg/compute/tasks/cloud_account_sync_task.go b/pkg/compute/tasks/cloud_account_sync_task.go index ec5c79e24c..7cd5349743 100644 --- a/pkg/compute/tasks/cloud_account_sync_task.go +++ b/pkg/compute/tasks/cloud_account_sync_task.go @@ -85,7 +85,7 @@ func (self *CloudAccountSyncInfoTask) OnCloudaccountSyncComplete(ctx context.Con } // sync skus - if err := skus.SyncSkusByProviderIds([]string{cloudprovider.GetId()}); err != nil { + if err := skus.SyncSkusByProviderIds([]string{cloudprovider.Provider}); err != nil { self.SetStageFailed(ctx, err.Error()) } } From ea3df1d1901df9f4869669e894c1894657fb2a33 Mon Sep 17 00:00:00 2001 From: TangBin Date: Wed, 5 Dec 2018 16:57:05 +0800 Subject: [PATCH 4/4] make fmt --- pkg/compute/skus/skus.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/compute/skus/skus.go b/pkg/compute/skus/skus.go index 5505c3e8a4..de3efe4acf 100644 --- a/pkg/compute/skus/skus.go +++ b/pkg/compute/skus/skus.go @@ -317,7 +317,6 @@ func SyncSkus(ctx context.Context, userCred mcclient.TokenCredential) { } } - func SyncSkusByProviderIds(providerIds []string) error { skus := SkusZoneList{} log.Debugf("SyncSkusByProviderIds %s", providerIds) @@ -330,4 +329,4 @@ func SyncSkusByProviderIds(providerIds []string) error { } return nil -} \ No newline at end of file +}