Merge pull request #503 from tb365/bugfix/tb-instance-spec-cache-fix

fix instance spec list query cache error

cmd/climc/shell/skus.go

@@ -148,4 +148,25 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type ServerSkuSpecsListOptions struct {
+		Provider       string  `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Qcloud|Azure|Aws|Huawei|Openstack|Ucloud" json:"provider"`
+		PublicCloud    *bool   `help:"List objects belonging to public cloud" json:"public_cloud"`
+		Zone           string  `help:"zone Id or name"`
+		PostpaidStatus *string `help:"skus available status for postpaid instance" choices:"available|soldout"`
+		PrepaidStatus  *string `help:"skus available status for prepaid instance"  choices:"available|soldout"`
+		IngoreCache    bool    `help:"query without cache"`
+	}
+	R(&ServerSkuSpecsListOptions{}, "server-sku-specs-list", "List all avaiable Server SKU specifications", func(s *mcclient.ClientSession, args *ServerSkuSpecsListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.ServerSkus.Get(s, "instance-specs", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

pkg/compute/models/cloudproviderregions.go

@@ -19,16 +19,15 @@ import (
 	"database/sql"
 	"math/rand"
 	"time"
-
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
-	"yunion.io/x/pkg/util/compare"
-	"yunion.io/x/pkg/util/timeutils"
-
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/sqlchemy"
 )
 
 type SCloudproviderregionManager struct {
@@ -155,6 +154,25 @@ func (self *SCloudproviderregion) Detach(ctx context.Context, userCred mcclient.
 	return db.DetachJoint(ctx, userCred, self)
 }
 
+/*
+过滤出指定cloudAccountId || providerIds || cloudAccountId+providerIds关联的region id
+*/
+func (manager *SCloudproviderregionManager) QueryRelatedRegionIds(cloudAccountId string, providerIds ...string) *sqlchemy.SSubQuery {
+	q := manager.Query("cloudregion_id")
+
+	if len(providerIds) > 0  {
+		q = q.Filter(sqlchemy.In(q.Field("cloudprovider_id"), providerIds))
+	}
+
+	if len(cloudAccountId) > 0 {
+		providers := CloudproviderManager.Query().SubQuery()
+		q = q.Join(providers, sqlchemy.Equals(providers.Field("id"), q.Field("cloudprovider_id")))
+		q.Filter(sqlchemy.Equals(providers.Field("cloudaccount_id"), cloudAccountId))
+	}
+
+	return q.Distinct().SubQuery()
+}
+
 func (manager *SCloudproviderregionManager) FetchByIds(providerId string, regionId string) *SCloudproviderregion {
 	q := manager.Query().Equals("cloudprovider_id", providerId).Equals("cloudregion_id", regionId)
 	obj, err := db.NewModelObject(manager)

pkg/compute/models/cloudregions.go

@@ -17,9 +17,7 @@ package models
 import (
 	"context"
 	"database/sql"
-	"strings"
 	"time"
-
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/compare"
@@ -445,34 +443,13 @@ func (manager *SCloudregionManager) ListItemFilter(ctx context.Context, q *sqlch
 
 	managerStr, _ := query.GetString("manager")
 	if len(managerStr) > 0 {
-		managerObj, err := CloudproviderManager.FetchByIdOrName(userCred, managerStr)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		manager := managerObj.(*SCloudprovider)
-		q = q.Equals("provider", manager.Provider)
-		if manager.Provider == api.CLOUD_PROVIDER_HUAWEI {
-			region := strings.Split(manager.Name, "_")[0]
-			prefix := api.CLOUD_PROVIDER_HUAWEI + "/" + region
-			q = q.Startswith("external_id", prefix)
-		}
+		subq := CloudproviderRegionManager.QueryRelatedRegionIds("", managerStr)
+		q = q.In("id", subq)
 	}
 	accountStr, _ := query.GetString("account")
 	if len(accountStr) > 0 {
-		accountObj, err := CloudaccountManager.FetchByIdOrName(userCred, accountStr)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		account := accountObj.(*SCloudaccount)
-		q = q.In("provider", account.Provider)
+		subq := CloudproviderRegionManager.QueryRelatedRegionIds(accountStr)
+		q = q.In("id", subq)
 	}
 
 	if jsonutils.QueryBoolean(query, "usable", false) || jsonutils.QueryBoolean(query, "usable_vpc", false) {

pkg/compute/models/skus.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"crypto/md5"
 	"database/sql"
 	"fmt"
 	"math"
@@ -24,6 +25,8 @@ import (
 	"strings"
 	"time"
 
+	"yunion.io/x/pkg/utils"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/compare"
@@ -102,6 +105,44 @@ type SServerSku struct {
 	Provider      string `width:"64" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"`
 }
 
+type SInstanceSpecQueryParams struct {
+	Provider       string
+	PublicCloud    bool
+	ZoneId         string
+	PostpaidStatus string
+	PrepaidStatus  string
+	IngoreCache    bool
+}
+
+func (self *SInstanceSpecQueryParams) GetCacheKey() string {
+	hashStr := fmt.Sprintf("%s:%t:%s:%s:%s", self.Provider, self.PublicCloud, self.ZoneId, self.PostpaidStatus, self.PrepaidStatus)
+	_md5 := md5.Sum([]byte(hashStr))
+	return "InstanceSpecs_" + fmt.Sprintf("%x", _md5)
+}
+
+func NewInstanceSpecQueryParams(query jsonutils.JSONObject) *SInstanceSpecQueryParams {
+	zone := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
+	postpaid, _ := query.GetString("postpaid_status")
+	prepaid, _ := query.GetString("prepaid_status")
+	ingore_cache, _ := query.Bool("ingore_cache")
+	provider := normalizeProvider(jsonutils.GetAnyString(query, []string{"provider"}))
+	public_cloud, _ := query.Bool("public_cloud")
+	if utils.IsInStringArray(provider, cloudprovider.GetPublicProviders()) {
+		public_cloud = true
+	}
+
+	params := &SInstanceSpecQueryParams{
+		Provider:       provider,
+		PublicCloud:    public_cloud,
+		ZoneId:         zone,
+		PostpaidStatus: postpaid,
+		PrepaidStatus:  prepaid,
+		IngoreCache:    ingore_cache,
+	}
+
+	return params
+}
+
 func sliceToJsonObject(items []int) jsonutils.JSONObject {
 	sort.Slice(items, func(i, j int) bool {
 		if items[i] < items[j] {
@@ -473,7 +514,7 @@ func providerFilter(q *sqlchemy.SQuery, provider string, public_cloud bool) *sql
 	if provider == "all" {
 		// provider 参数为all时。表示查询所有instance type.
 		return q
-	} else if len(provider) > 0 {
+	} else if len(provider) > 0 && !utils.IsInStringArray(provider, []string{api.CLOUD_PROVIDER_ONECLOUD, api.CLOUD_PROVIDER_VMWARE, "kvm"}) {
 		q = q.Equals("provider", provider)
 	} else if public_cloud {
 		q = q.IsNotEmpty("provider")
@@ -489,26 +530,27 @@ func providerFilter(q *sqlchemy.SQuery, provider string, public_cloud bool) *sql
 }
 
 func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	specsKey := "InstanceSpecs"
-	v := Cache.Get(specsKey)
-	if v != nil {
-		return v.(*jsonutils.JSONDict), nil
+	params := NewInstanceSpecQueryParams(query)
+	if !params.IngoreCache {
+		v := Cache.Get(params.GetCacheKey())
+		if v != nil {
+			if cacheRet, ok := v.(*jsonutils.JSONDict); ok {
+				return cacheRet, nil
+			}
+		}
 	}
 
 	q := self.Query()
 	// 未明确指定provider或者public_cloud时，默认查询私有云
-	provider, _ := query.GetString("provider")
-	public_cloud, _ := query.Bool("public_cloud")
-	q = providerFilter(q, provider, public_cloud)
+	q = providerFilter(q, params.Provider, params.PublicCloud)
 	q = excludeSkus(q)
 
 	// 如果是查询私有云需要忽略zone参数
-	zone := jsonutils.GetAnyString(query, []string{"zone", "zone_id"})
-	if public_cloud && len(zone) > 0 {
-		zoneObj, err := ZoneManager.FetchByIdOrName(userCred, zone)
+	if params.PublicCloud && len(params.ZoneId) > 0 {
+		zoneObj, err := ZoneManager.FetchByIdOrName(userCred, params.ZoneId)
 		if err != nil {
 			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), zone)
+				return nil, httperrors.NewResourceNotFoundError2(ZoneManager.Keyword(), params.ZoneId)
 			}
 			return nil, httperrors.NewGeneralError(err)
 		}
@@ -517,14 +559,12 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use
 	}
 
 	skus := make([]SServerSku, 0)
-	postpaid, _ := query.GetString("postpaid_status")
-	if len(postpaid) > 0 {
-		q.Equals("postpaid_status", postpaid)
+	if len(params.PostpaidStatus) > 0 {
+		q.Equals("postpaid_status", params.PostpaidStatus)
 	}
 
-	prepaid, _ := query.GetString("prepaid_status")
-	if len(prepaid) > 0 {
-		q.Equals("prepaid_status", prepaid)
+	if len(params.PrepaidStatus) > 0 {
+		q.Equals("prepaid_status", params.PrepaidStatus)
 	}
 	q = q.GroupBy(q.Field("cpu_core_count"), q.Field("memory_size_mb"))
 	q = q.Asc(q.Field("cpu_core_count"), q.Field("memory_size_mb"))
@@ -571,8 +611,8 @@ func (self *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, use
 
 	r_obj := jsonutils.Marshal(&cpu_mems_mb)
 	ret.Add(r_obj, "cpu_mems_mb")
-	// cache
-	Cache.Set(specsKey, ret)
+	// cache 1min
+	Cache.Set(params.GetCacheKey(), ret, time.Now().Add(60*time.Second))
 	return ret, nil
 }
 
@@ -709,6 +749,9 @@ func (self *SServerSku) GetZoneExternalId() (string, error) {
 func (manager *SServerSkuManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	provider := normalizeProvider(jsonutils.GetAnyString(query, []string{"provider"}))
 	public_cloud, _ := query.Bool("public_cloud")
+	if utils.IsInStringArray(provider, cloudprovider.GetPublicProviders()) {
+		public_cloud = true
+	}
 	queryDict := query.(*jsonutils.JSONDict)
 	// 手动处理provider查询
 	queryDict.Remove("provider")

pkg/compute/models/zones.go

@@ -604,30 +604,12 @@ func (manager *SZoneManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 
 	managerStr, _ := query.GetString("manager")
 	if len(managerStr) > 0 {
-		providerObj, err := CloudproviderManager.FetchByIdOrName(userCred, managerStr)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		provider := providerObj.(*SCloudprovider)
-		subq := CloudregionManager.Query("id").Equals("provider", provider.Provider).SubQuery()
+		subq := CloudproviderRegionManager.QueryRelatedRegionIds("", managerStr)
 		q = q.In("cloudregion_id", subq)
 	}
 	accountStr, _ := query.GetString("account")
 	if len(accountStr) > 0 {
-		accountObj, err := CloudaccountManager.FetchByIdOrName(userCred, accountStr)
-		if err != nil {
-			if err == sql.ErrNoRows {
-				return nil, httperrors.NewResourceNotFoundError2(CloudaccountManager.Keyword(), accountStr)
-			} else {
-				return nil, httperrors.NewGeneralError(err)
-			}
-		}
-		account := accountObj.(*SCloudaccount)
-		subq := CloudregionManager.Query("id").Equals("provider", account.Provider).SubQuery()
+		subq := CloudproviderRegionManager.QueryRelatedRegionIds(accountStr)
 		q = q.In("cloudregion_id", subq)
 	}
 	providerStr, _ := query.GetString("provider")