opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3486 from ioito/hotfix/qx-dbinstance-sku-sync

Browse Source 
fix: 从aliyun对象存储上面同步dbinstance sku信息
This commit is contained in:
yunion-ci-robot
2019-11-08 16:13:15 +08:00
committed by GitHub
parent f3a4db5a4b 92bb83025e
commit e9414a8ee6
3 changed files with 88 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/compute/models/dbinstance_skus.go
+9 -41
View File
@@ -17,7 +17,6 @@ package models
import (
"context"
"fmt"
"net/url"
"strings"
"yunion.io/x/jsonutils"
@@ -30,11 +29,8 @@ import (
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/cloudcommon/validators"
"yunion.io/x/onecloud/pkg/compute/options"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
"yunion.io/x/onecloud/pkg/util/httputils"
)
type SDBInstanceSkuManager struct {
@@ -337,51 +333,17 @@ func (manager *SDBInstanceSkuManager) GetDBInstanceSkus(provider, cloudregionId,
return skus, nil
}
func (manager *SDBInstanceSkuManager) syncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) compare.SyncResult {
func (manager *SDBInstanceSkuManager) syncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, meta *SSkuResourcesMeta) compare.SyncResult {
lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred))
defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred))
syncResult := compare.SyncResult{}
regionInfo := strings.Split(region.ExternalId, "/")
if len(regionInfo) == 0 {
syncResult.Error(fmt.Errorf("region %s is not belong public cloud???", region.Name))
return syncResult
}
regionId := regionInfo[len(regionInfo)-1]
s := auth.GetAdminSession(ctx, options.Options.Region, "")
uri, err := s.GetServiceURL("yunionmeta", "")
iskus, err := meta.GetDBInstanceSkusByRegion(region.ExternalId)
if err != nil {
syncResult.Error(err)
return syncResult
}
iskus := []SDBInstanceSku{}
params := url.Values{}
params.Add("region_id", regionId)
params.Add("provider", region.Provider)
params.Set("limit", "2048")
for {
params.Set("offset", fmt.Sprintf("%d", len(iskus)))
_, resp, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, httputils.THttpMethod("GET"), fmt.Sprintf("%s/dbinstance_skus?%s", uri, params.Encode()), nil, nil, false)
if err != nil {
syncResult.Error(errors.Wrap(err, "request yunionmeta dbinstance_skus"))
return syncResult
}
parts := []SDBInstanceSku{}
err = resp.Unmarshal(&parts, "dbinstance_skus")
if err != nil {
syncResult.Error(errors.Wrapf(err, "skus.Unmarshal"))
return syncResult
}
iskus = append(iskus, parts...)
total, _ := resp.Int("total")
if len(iskus) >= int(total) {
break
}
}
dbSkus, err := manager.fetchDBInstanceSkus(region.Provider, region)
if err != nil {
@@ -501,8 +463,14 @@ func SyncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential,
return
}
meta, err := fetchSkuResourcesMeta()
if err != nil {
log.Errorf("failed to fetch sku resource meta: %v", err)
return
}
for _, region := range cloudregions {
result := DBInstanceSkuManager.syncDBInstanceSkus(ctx, userCred, &region)
result := DBInstanceSkuManager.syncDBInstanceSkus(ctx, userCred, &region, meta)
msg := result.Result()
notes := fmt.Sprintf("SyncDBInstanceSkus for region %s result: %s", region.Name, msg)
log.Infof(notes)
pkg/compute/models/skus_tools.go
+79 -38
View File
@@ -25,63 +25,93 @@ server: 虚拟机
elasticcache: 弹性缓存(redis&memcached)
*/
type SSkuResourcesMeta struct {
region *SCloudregion
caches map[string][]jsonutils.JSONObject
zoneCaches map[string]*SZone
regionCaches map[string]*SCloudregion
region *SCloudregion
caches map[string][]jsonutils.JSONObject
regionalSkuCaches map[string]map[string][]jsonutils.JSONObject
zoneCaches map[string]*SZone
regionCaches map[string]*SCloudregion
Server string
ElasticCache string
DBInstance string `json:"dbinstance"`
}
// todo: 待测试
func (self *SSkuResourcesMeta) GetServerSkus() ([]SServerSku, error) {
result := []SServerSku{}
err := self.getSkus(self.Server, &result)
objs, err := self.get(self.Server)
if err != nil {
return nil, errors.Wrap(err, "SkuResourcesMeta.GetServerSkus")
return nil, errors.Wrap(err, "self.get")
}
for _, obj := range objs {
sku := SServerSku{}
err = obj.Unmarshal(&sku)
if err != nil {
return nil, errors.Wrap(err, "obj.Unmarshal")
}
result = append(result, sku)
}
return result, nil
}
// todo: process data here
func (self *SSkuResourcesMeta) GetDBInstanceSkusByRegion(regionId string) ([]SDBInstanceSku, error) {
result := []SDBInstanceSku{}
objs, err := self.getSkusByRegion(self.DBInstance, regionId)
if err != nil {
return nil, errors.Wrapf(err, "getSkusByRegion")
}
for _, obj := range objs {
sku := SDBInstanceSku{}
err = obj.Unmarshal(&sku)
if err != nil {
return nil, errors.Wrapf(err, "obj.Unmarshal")
}
result = append(result, sku)
}
return result, nil
}
func (self *SSkuResourcesMeta) GetElasticCacheSkus() ([]SElasticcacheSku, error) {
result := []SElasticcacheSku{}
err := self.getSkus(self.ElasticCache, &result)
objs, err := self.get(self.ElasticCache)
if err != nil {
return nil, errors.Wrap(err, "SkuResourcesMeta.GetElasticCacheSkus")
return nil, errors.Wrap(err, "self.get(self.ElasticCache)")
}
for _, obj := range objs {
sku := SElasticcacheSku{}
err = obj.Unmarshal(&sku)
if err != nil {
return nil, errors.Wrap(err, "obj.Unmarshal")
}
// 处理数据
provider := sku.Provider
region := sku.CloudregionId
// 处理数据
for i := range result {
provider := result[i].Provider
region := result[i].CloudregionId
result[i].Id = ""
sku.Id = ""
r, err := self.fetchRegion(provider, region)
if err != nil {
return nil, errors.Wrap(err, "SkuResourcesMeta.GetElasticCacheSkus.fetchRegion")
}
result[i].CloudregionId = r.GetId()
sku.CloudregionId = r.GetId()
if len(result[i].ZoneId) > 0 {
zone, err := self.fetchZone(provider, region, result[i].ZoneId)
if len(sku.ZoneId) > 0 {
zone, err := self.fetchZone(provider, region, sku.ZoneId)
if err != nil {
return nil, errors.Wrap(err, "SkuResourcesMeta.GetElasticCacheSkus.MasterZone")
}
result[i].ZoneId = zone.GetId()
sku.ZoneId = zone.GetId()
}
if len(result[i].SlaveZoneId) > 0 {
zone, err := self.fetchZone(provider, region, result[i].SlaveZoneId)
if len(sku.SlaveZoneId) > 0 {
zone, err := self.fetchZone(provider, region, sku.SlaveZoneId)
if err != nil {
return nil, errors.Wrap(err, "SkuResourcesMeta.GetElasticCacheSkus.SlaveZone")
}
result[i].SlaveZoneId = zone.GetId()
sku.SlaveZoneId = zone.GetId()
}
result = append(result, sku)
}
return result, nil
@@ -127,21 +157,6 @@ func (self *SSkuResourcesMeta) fetchRegion(provider, region string) (*SCloudregi
return r, nil
}
func (self *SSkuResourcesMeta) getSkus(url string, result interface{}) error {
objs, err := self.get(self.ElasticCache)
if err != nil {
return errors.Wrap(err, "SkuResourcesMeta.getSkus")
}
objArray := jsonutils.Marshal(objs)
err = objArray.Unmarshal(result)
if err != nil {
return errors.Wrap(err, "SkuResourcesMeta.Unmarshal")
}
return nil
}
func (self *SSkuResourcesMeta) SetRegionFilter(region *SCloudregion) {
self.region = region
}
@@ -184,6 +199,32 @@ func (self *SSkuResourcesMeta) get(url string) ([]jsonutils.JSONObject, error) {
return self.filterByRegion(items), nil
}
func (self *SSkuResourcesMeta) getSkusByRegion(url string, region string) ([]jsonutils.JSONObject, error) {
items, err := self.get(url)
if err != nil {
return nil, err
}
if self.regionalSkuCaches == nil {
self.regionalSkuCaches = map[string]map[string][]jsonutils.JSONObject{}
self.regionalSkuCaches[url] = map[string][]jsonutils.JSONObject{}
for i := range items {
cloudregion, _ := items[i].GetString("cloudregion_id")
if _, ok := self.regionalSkuCaches[url][cloudregion]; !ok && len(cloudregion) > 0 {
self.regionalSkuCaches[url][cloudregion] = []jsonutils.JSONObject{}
}
self.regionalSkuCaches[url][cloudregion] = append(self.regionalSkuCaches[url][cloudregion], items[i])
}
}
for regionId, skus := range self.regionalSkuCaches[url] {
if strings.HasSuffix(region, regionId) {
return skus, nil
}
}
return []jsonutils.JSONObject{}, nil
}
func (self *SSkuResourcesMeta) _get(url string) ([]jsonutils.JSONObject, error) {
if !strings.HasPrefix(url, "http") {
return nil, fmt.Errorf("SkuResourcesMeta.get invalid url %s.expected has prefix 'http'", url)
@@ -208,7 +249,7 @@ func (self *SSkuResourcesMeta) _get(url string) ([]jsonutils.JSONObject, error)
ret := []jsonutils.JSONObject{}
err = contentJson.Unmarshal(&ret)
if err != nil {
return nil, fmt.Errorf("SkuResourcesMeta.get.Unmarshal %s", err)
return nil, fmt.Errorf("SkuResourcesMeta.get.Unmarshal(%s) %s", contentJson.String(), err)
}
return ret, nil
pkg/mcclient/modules/managers.go
-6
View File
@@ -152,12 +152,6 @@ func NewCloudmetaManager(keyword, keywordPlural string, columns, adminColumns []
Keyword: keyword, KeywordPlural: keywordPlural}
}
func NewYunionmetaManager(keyword, keywordPlural string, columns, adminColumns []string) modulebase.ResourceManager {
return modulebase.ResourceManager{
BaseManager: *modulebase.NewBaseManager("yunionmeta", "", "", columns, adminColumns),
Keyword: keyword, KeywordPlural: keywordPlural}
}
func NewOfflineCloudmetaManager(keyword, keywordPlural string, columns, adminColumns []string) modulebase.ResourceManager {
return modulebase.ResourceManager{
BaseManager: *modulebase.NewBaseManager("offlinecloudmeta", "", "", columns, adminColumns),