fix(cloudcommon): disable quota check by default

1. add enable_quota_check common option
2. disable quota check by default
3. specify scope of quota limit in OutOfQuota error message

cmd/climc/shell/compute/quotas.go

@@ -317,7 +317,7 @@ func init() {
 		QuotaSetBaseOptions
 		IdentityQuotaOptions
 	}
-	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *InfrasQuotaSetOptions) error {
+	R(&IdentityQuotaSetOptions{}, "identity-quota-set", "Set identity quota for domain", func(s *mcclient.ClientSession, args *IdentityQuotaSetOptions) error {
 		params := jsonutils.Marshal(args)
 		quotas, e := modules.IdentityQuotas.DoQuotaSet(s, params)
 		if e != nil {

pkg/apis/identity/consts.go

@@ -99,6 +99,7 @@ var (
 
 	CommonWhitelistOptionMap = map[string][]string{
 		"default": []string{
+			"enable_quota_check",
 			"default_quota_value",
 			"enable_rbac",
 			"non_default_domain_projects",

pkg/cloudcommon/consts/consts.go

@@ -37,6 +37,8 @@ var (
 	domainizedNamespace = true
 
 	historicalUniqueName = false
+
+	enableQuotaCheck = false
 )
 
 func SetRegion(region string) {
@@ -99,3 +101,11 @@ func DisableHistoricalUniqueName() {
 func IsHistoricalUniqueName() bool {
 	return historicalUniqueName
 }
+
+func SetEnableQuotaCheck(val bool) {
+	enableQuotaCheck = val
+}
+
+func EnableQuotaCheck() bool {
+	return enableQuotaCheck
+}

pkg/cloudcommon/db/quotas/errors.go

@@ -19,9 +19,11 @@ import (
 	"strings"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SOutOfQuotaError struct {
+	scope   rbacutils.TRbacScope
 	name    string
 	limit   int
 	used    int
@@ -37,7 +39,7 @@ func (e *SOutOfQuotaError) Cause() error {
 }
 
 func (e *SOutOfQuotaError) Error() string {
-	return fmt.Sprintf("%s limit %d used %d request %d", e.name, e.limit, e.used, e.request)
+	return fmt.Sprintf("[%s.%s] limit %d used %d request %d", e.scope, e.name, e.limit, e.used, e.request)
 }
 
 func (es *SOutOfQuotaErrors) Error() string {
@@ -63,8 +65,10 @@ func NewOutOfQuotaError() *SOutOfQuotaErrors {
 	}
 }
 
-func (es *SOutOfQuotaErrors) Add(name string, limit int, used int, request int) {
+func (es *SOutOfQuotaErrors) Add(quota IQuota, name string, limit int, used int, request int) {
+	scope := quota.GetKeys().Scope()
 	e := SOutOfQuotaError{
+		scope:   scope,
 		name:    name,
 		limit:   limit,
 		used:    used,

pkg/cloudcommon/db/quotas/handler.go

@@ -132,6 +132,12 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 	ret := jsonutils.NewDict()
 
 	keys := quota.GetKeys()
+	ret.Update(jsonutils.Marshal(keys))
+	ret.Update(quota.ToJSON(""))
+
+	if !consts.EnableQuotaCheck() {
+		return ret, nil
+	}
 
 	usage := manager.newQuota()
 	err := manager.usageStore.GetQuota(ctx, keys, usage)
@@ -150,8 +156,6 @@ func (manager *SQuotaBaseManager) queryQuota(ctx context.Context, quota IQuota,
 		return nil, errors.Wrap(err, "manager.GetPendingUsages")
 	}
 
-	ret.Update(jsonutils.Marshal(keys))
-	ret.Update(quota.ToJSON(""))
 	if usage != nil {
 		ret.Update(usage.ToJSON("usage"))
 	}

pkg/cloudcommon/db/quotas/interface.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type IQuotaKeys interface {
@@ -30,6 +31,8 @@ type IQuotaKeys interface {
 	Compare(IQuotaKeys) int
 
 	OwnerId() mcclient.IIdentityProvider
+
+	Scope() rbacutils.TRbacScope
 }
 
 type IQuota interface {

pkg/cloudcommon/db/quotas/register.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -57,6 +58,10 @@ func getQuotaManager(quota IQuota) IQuotaManager {
 }
 
 func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential, localUsage IQuota, cancelUsage IQuota, save bool) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	if localUsage == nil {
 		return nil
 	}
@@ -65,6 +70,10 @@ func CancelPendingUsage(ctx context.Context, userCred mcclient.TokenCredential,
 }
 
 func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential, quota IQuota) error {
+	if !consts.EnableQuotaCheck() {
+		return nil
+	}
+
 	manager := getQuotaManager(quota)
 	err := manager.checkSetPendingQuota(ctx, userCred, quota)
 	if err != nil {
@@ -75,6 +84,9 @@ func CheckSetPendingQuota(ctx context.Context, userCred mcclient.TokenCredential
 }
 
 func CancelUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		cancelUsage(ctx, userCred, usage.(IQuota))
 	}
@@ -89,6 +101,9 @@ func cancelUsage(ctx context.Context, userCred mcclient.TokenCredential, usage I
 }
 
 func AddUsages(ctx context.Context, userCred mcclient.TokenCredential, usages []db.IUsage) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
 	for _, usage := range usages {
 		addUsage(ctx, userCred, usage.(IQuota))
 	}

pkg/cloudcommon/db/quotas/usageworker.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -61,6 +62,12 @@ func isDirty(key string) bool {
 }
 
 func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan IQuota, realTime bool) {
+	if !consts.EnableQuotaCheck() {
+		go func() {
+			usageChan <- nil
+		}()
+		return
+	}
 	key := QuotaKeyString(keys)
 	setDirty(key)
 
@@ -105,6 +112,10 @@ func (manager *SQuotaBaseManager) PostUsageJob(keys IQuotaKeys, usageChan chan I
 }
 
 func (manager *SQuotaBaseManager) CalculateQuotaUsages(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if !consts.EnableQuotaCheck() {
+		return
+	}
+
 	log.Infof("CalculateQuotaUsages")
 	quota := manager.newQuota()
 	keys := quota.GetKeys()

pkg/cloudcommon/options/changes.go

@@ -50,6 +50,9 @@ func OnBaseOptionsChange(oOpts, nOpts interface{}) bool {
 		netutils.SetPrivatePrefixes(newOpts.CustomizedPrivatePrefixes)
 		log.Debugf("Customized private prefixes: %s", netutils.GetPrivateIPRanges())
 	}
+	if oldOpts.EnableQuotaCheck != newOpts.EnableQuotaCheck {
+		consts.SetEnableQuotaCheck(newOpts.EnableQuotaCheck)
+	}
 	return changed
 }
 

pkg/cloudcommon/options/options.go

@@ -78,6 +78,7 @@ type BaseOptions struct {
 	IsSlaveNode        bool `help:"Slave mode"`
 	CronJobWorkerCount int  `help:"Cron job worker count" default:"4"`
 
+	EnableQuotaCheck  bool   `help:"enable quota check" default:"false"`
 	DefaultQuotaValue string `help:"default quota value" choices:"unlimit|zero|default" default:"default"`
 
 	CalculateQuotaUsageIntervalSeconds int `help:"interval to calculate quota usages, default 30 minutes" default:"900"`

pkg/compute/models/domainquota.go

@@ -191,13 +191,13 @@ func (used *SDomainQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) err
 	sreq := request.(*SDomainQuota)
 	squota := quota.(*SDomainQuota)
 	if quotas.Exceed(used.Globalvpc, sreq.Globalvpc, squota.Globalvpc) {
-		err.Add("globalvpc", squota.Globalvpc, used.Globalvpc, sreq.Globalvpc)
+		err.Add(used, "globalvpc", squota.Globalvpc, used.Globalvpc, sreq.Globalvpc)
 	}
 	if quotas.Exceed(used.Cloudaccount, sreq.Cloudaccount, squota.Cloudaccount) {
-		err.Add("cloudaccount", squota.Cloudaccount, used.Cloudaccount, sreq.Cloudaccount)
+		err.Add(used, "cloudaccount", squota.Cloudaccount, used.Cloudaccount, sreq.Cloudaccount)
 	}
 	if quotas.Exceed(used.DnsZone, sreq.DnsZone, squota.DnsZone) {
-		err.Add("dns_zone", squota.DnsZone, used.DnsZone, sreq.DnsZone)
+		err.Add(used, "dns_zone", squota.DnsZone, used.DnsZone, sreq.DnsZone)
 	}
 	if err.IsError() {
 		return err

pkg/compute/models/infrasquota.go

@@ -207,10 +207,10 @@ func (used *SInfrasQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) err
 	sreq := request.(*SInfrasQuota)
 	squota := quota.(*SInfrasQuota)
 	if quotas.Exceed(used.Host, sreq.Host, squota.Host) {
-		err.Add("host", squota.Host, used.Host, sreq.Host)
+		err.Add(used, "host", squota.Host, used.Host, sreq.Host)
 	}
 	if quotas.Exceed(used.Vpc, sreq.Vpc, squota.Vpc) {
-		err.Add("vpc", squota.Vpc, used.Vpc, sreq.Vpc)
+		err.Add(used, "vpc", squota.Vpc, used.Vpc, sreq.Vpc)
 	}
 	if err.IsError() {
 		return err

pkg/compute/models/projectquota.go

@@ -165,7 +165,7 @@ func (used *SProjectQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) er
 	sreq := request.(*SProjectQuota)
 	squota := quota.(*SProjectQuota)
 	if quotas.Exceed(used.Secgroup, sreq.Secgroup, squota.Secgroup) {
-		err.Add("secgroup", squota.Secgroup, used.Secgroup, sreq.Secgroup)
+		err.Add(used, "secgroup", squota.Secgroup, used.Secgroup, sreq.Secgroup)
 	}
 	if err.IsError() {
 		return err

pkg/compute/models/quotas.go

@@ -317,22 +317,22 @@ func (used *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error {
 	sreq := request.(*SQuota)
 	squota := quota.(*SQuota)
 	if quotas.Exceed(used.Count, sreq.Count, squota.Count) {
-		err.Add("count", squota.Count, used.Count, sreq.Count)
+		err.Add(used, "count", squota.Count, used.Count, sreq.Count)
 	}
 	if quotas.Exceed(used.Cpu, sreq.Cpu, squota.Cpu) {
-		err.Add("cpu", squota.Cpu, used.Cpu, sreq.Cpu)
+		err.Add(used, "cpu", squota.Cpu, used.Cpu, sreq.Cpu)
 	}
 	if quotas.Exceed(used.Memory, sreq.Memory, squota.Memory) {
-		err.Add("memory", squota.Memory, used.Memory, sreq.Memory)
+		err.Add(used, "memory", squota.Memory, used.Memory, sreq.Memory)
 	}
 	if quotas.Exceed(used.Storage, sreq.Storage, squota.Storage) {
-		err.Add("storage", squota.Storage, used.Storage, sreq.Storage)
+		err.Add(used, "storage", squota.Storage, used.Storage, sreq.Storage)
 	}
 	if quotas.Exceed(used.Group, sreq.Group, squota.Group) {
-		err.Add("group", squota.Group, used.Group, sreq.Group)
+		err.Add(used, "group", squota.Group, used.Group, sreq.Group)
 	}
 	if quotas.Exceed(used.IsolatedDevice, sreq.IsolatedDevice, squota.IsolatedDevice) {
-		err.Add("isolated_device", squota.IsolatedDevice, used.IsolatedDevice, sreq.IsolatedDevice)
+		err.Add(used, "isolated_device", squota.IsolatedDevice, used.IsolatedDevice, sreq.IsolatedDevice)
 	}
 	if err.IsError() {
 		return err

pkg/compute/models/regionquota.go

@@ -422,43 +422,43 @@ func (used *SRegionQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) err
 	sreq := request.(*SRegionQuota)
 	squota := quota.(*SRegionQuota)
 	if quotas.Exceed(used.Port, sreq.Port, squota.Port) {
-		err.Add("port", squota.Port, used.Port, sreq.Port)
+		err.Add(used, "port", squota.Port, used.Port, sreq.Port)
 	}
 	if quotas.Exceed(used.Eip, sreq.Eip, squota.Eip) {
-		err.Add("eip", squota.Eip, used.Eip, sreq.Eip)
+		err.Add(used, "eip", squota.Eip, used.Eip, sreq.Eip)
 	}
 	if quotas.Exceed(used.Eport, sreq.Eport, squota.Eport) {
-		err.Add("eport", squota.Eport, used.Eport, sreq.Eport)
+		err.Add(used, "eport", squota.Eport, used.Eport, sreq.Eport)
 	}
 	//if quotas.Exceed(used.Bw, sreq.Bw, squota.Bw) {
-	//	err.Add("bw", squota.Bw, used.Bw, sreq.Bw)
+	//	err.Add(used, "bw", squota.Bw, used.Bw, sreq.Bw)
 	//}
 	//if quotas.Exceed(used.Bw, sreq.Ebw, squota.Ebw) {
-	//	err.Add("ebw", squota.Ebw, used.Ebw, sreq.Ebw)
+	//	err.Add(used, "ebw", squota.Ebw, used.Ebw, sreq.Ebw)
 	//}
 	if quotas.Exceed(used.Snapshot, sreq.Snapshot, squota.Snapshot) {
-		err.Add("snapshot", squota.Snapshot, used.Snapshot, sreq.Snapshot)
+		err.Add(used, "snapshot", squota.Snapshot, used.Snapshot, sreq.Snapshot)
 	}
 	if quotas.Exceed(used.InstanceSnapshot, sreq.InstanceSnapshot, squota.InstanceSnapshot) {
-		err.Add("instance_snapshot", squota.InstanceSnapshot, used.InstanceSnapshot, sreq.InstanceSnapshot)
+		err.Add(used, "instance_snapshot", squota.InstanceSnapshot, used.InstanceSnapshot, sreq.InstanceSnapshot)
 	}
 	if quotas.Exceed(used.Bucket, sreq.Bucket, squota.Bucket) {
-		err.Add("bucket", squota.Bucket, used.Bucket, sreq.Bucket)
+		err.Add(used, "bucket", squota.Bucket, used.Bucket, sreq.Bucket)
 	}
 	if quotas.Exceed(used.ObjectGB, sreq.ObjectGB, squota.ObjectGB) {
-		err.Add("object_gb", squota.ObjectGB, used.ObjectGB, sreq.ObjectGB)
+		err.Add(used, "object_gb", squota.ObjectGB, used.ObjectGB, sreq.ObjectGB)
 	}
 	if quotas.Exceed(used.ObjectCnt, sreq.ObjectCnt, squota.ObjectCnt) {
-		err.Add("object_cnt", squota.ObjectCnt, used.ObjectCnt, sreq.ObjectCnt)
+		err.Add(used, "object_cnt", squota.ObjectCnt, used.ObjectCnt, sreq.ObjectCnt)
 	}
 	if quotas.Exceed(used.Rds, sreq.Rds, squota.Rds) {
-		err.Add("rds", squota.Rds, used.Rds, sreq.Rds)
+		err.Add(used, "rds", squota.Rds, used.Rds, sreq.Rds)
 	}
 	if quotas.Exceed(used.Cache, sreq.Cache, squota.Cache) {
-		err.Add("cache", squota.Cache, used.Cache, sreq.Cache)
+		err.Add(used, "cache", squota.Cache, used.Cache, sreq.Cache)
 	}
 	if quotas.Exceed(used.Loadbalancer, sreq.Loadbalancer, squota.Loadbalancer) {
-		err.Add("loadbalancer", squota.Loadbalancer, used.Loadbalancer, sreq.Loadbalancer)
+		err.Add(used, "loadbalancer", squota.Loadbalancer, used.Loadbalancer, sreq.Loadbalancer)
 	}
 	if err.IsError() {
 		return err

pkg/image/models/quotas.go

@@ -184,7 +184,7 @@ func (used *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error {
 	sreq := request.(*SQuota)
 	squota := quota.(*SQuota)
 	if quotas.Exceed(used.Image, sreq.Image, squota.Image) {
-		err.Add("image", squota.Image, used.Image, sreq.Image)
+		err.Add(used, "image", squota.Image, used.Image, sreq.Image)
 	}
 	if err.IsError() {
 		return err

pkg/keystone/models/identityquota.go

@@ -231,19 +231,19 @@ func (used *SIdentityQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) e
 	sreq := request.(*SIdentityQuota)
 	squota := quota.(*SIdentityQuota)
 	if quotas.Exceed(used.User, sreq.User, squota.User) {
-		err.Add("user", squota.User, used.User, sreq.User)
+		err.Add(used, "user", squota.User, used.User, sreq.User)
 	}
 	if quotas.Exceed(used.Group, sreq.Group, squota.Group) {
-		err.Add("group", squota.Group, used.Group, sreq.Group)
+		err.Add(used, "group", squota.Group, used.Group, sreq.Group)
 	}
 	if quotas.Exceed(used.Project, sreq.Project, squota.Project) {
-		err.Add("project", squota.Project, used.Project, sreq.Project)
+		err.Add(used, "project", squota.Project, used.Project, sreq.Project)
 	}
 	if quotas.Exceed(used.Role, sreq.Role, squota.Role) {
-		err.Add("role", squota.Role, used.Role, sreq.Role)
+		err.Add(used, "role", squota.Role, used.Role, sreq.Role)
 	}
 	if quotas.Exceed(used.Policy, sreq.Policy, squota.Policy) {
-		err.Add("policy", squota.Policy, used.Policy, sreq.Policy)
+		err.Add(used, "policy", squota.Policy, used.Policy, sreq.Policy)
 	}
 	if err.IsError() {
 		return err