Merge pull request #2284 from swordqiu/feature/qj-object-quota-limit

feature: 1. limit for bucket object count and size 2. quota for object

cmd/climc/shell/buckets.go

@@ -264,4 +264,20 @@ func init() {
 		return nil
 	})
 
+	type BucketLimitOptions struct {
+		ID          string `help:"ID or name of bucket" json:"-"`
+		SizeBytes   int64  `help:"size limit in bytes"`
+		ObjectCount int64  `help:"object count limit"`
+	}
+	R(&BucketLimitOptions{}, "bucket-limit", "Set limit of bucket", func(s *mcclient.ClientSession, args *BucketLimitOptions) error {
+		limit := jsonutils.Marshal(args)
+		params := jsonutils.NewDict()
+		params.Set("limit", limit)
+		result, err := modules.Buckets.PerformAction(s, args.ID, "limit", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/quotas.go

@@ -34,6 +34,9 @@ type QuotaBaseOptions struct {
 	Snapshot       int64 `help:"Snapshot count" json:"snapshot,omitzero"`
 	Image          int64 `help:"Template count" json:"image,omitzero"`
 	Secgroup       int64 `help:"Secgroup count" json:"secgroup,omitzero"`
+	Bucket         int64 `help:"bucket count" json:"bucket,omitzero"`
+	ObjectGB       int64 `help:"object size in GB" json:"object_gb,omitzero"`
+	ObjectCnt      int64 `help:"object count" json:"object_cnt,omitzero"`
 }
 
 func init() {

pkg/cloudprovider/objectstore.go

@@ -90,6 +90,8 @@ type ICloudBucket interface {
 	GetStorageClass() string
 	GetAccessUrls() []SBucketAccessUrl
 	GetStats() SBucketStats
+	GetLimit() SBucketStats
+	SetLimit(limit SBucketStats) error
 
 	SetAcl(acl TBucketACLType) error
 

pkg/compute/models/buckets.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"database/sql"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -29,7 +30,6 @@ import (
 	"yunion.io/x/pkg/util/compare"
 	"yunion.io/x/sqlchemy"
 
-	"strconv"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -76,6 +76,9 @@ type SBucket struct {
 	SizeBytes int64 `nullable:"false" default:"0" list:"user"`
 	ObjectCnt int   `nullable:"false" default:"0" list:"user"`
 
+	SizeBytesLimit int64 `nullable:"false" default:"0" list:"user"`
+	ObjectCntLimit int   `nullable:"false" default:"0" list:"user"`
+
 	AccessUrls jsonutils.JSONObject `nullable:"true" list:"user"`
 }
 
@@ -189,6 +192,10 @@ func (manager *SBucketManager) newFromCloudBucket(
 	bucket.SizeBytes = stats.SizeBytes
 	bucket.ObjectCnt = stats.ObjectCount
 
+	limit := extBucket.GetLimit()
+	bucket.SizeBytesLimit = limit.SizeBytes
+	bucket.ObjectCntLimit = limit.ObjectCount
+
 	bucket.AccessUrls = jsonutils.Marshal(extBucket.GetAccessUrls())
 
 	bucket.IsEmulated = false
@@ -240,6 +247,10 @@ func (bucket *SBucket) syncWithCloudBucket(
 		bucket.ObjectCnt = stats.ObjectCount
 
 		if !statsOnly {
+			limit := extBucket.GetLimit()
+			bucket.SizeBytesLimit = limit.SizeBytes
+			bucket.ObjectCntLimit = limit.ObjectCount
+
 			bucket.Acl = string(extBucket.GetAcl())
 			bucket.Location = extBucket.GetLocation()
 			bucket.StorageClass = extBucket.GetStorageClass()
@@ -369,9 +380,11 @@ func (manager *SBucketManager) ValidateCreateData(
 	query jsonutils.JSONObject,
 	data *jsonutils.JSONDict,
 ) (*jsonutils.JSONDict, error) {
+	cloudRegionV := validators.NewModelIdOrNameValidator("cloudregion", CloudregionManager.Keyword(), ownerId)
+	managerV := validators.NewModelIdOrNameValidator("manager", CloudproviderManager.Keyword(), ownerId)
 	for _, v := range []validators.IValidator{
-		validators.NewModelIdOrNameValidator("cloudregion", CloudregionManager.Keyword(), ownerId),
-		validators.NewModelIdOrNameValidator("manager", CloudproviderManager.Keyword(), ownerId),
+		cloudRegionV,
+		managerV,
 	} {
 		err := v.Validate(data)
 		if err != nil {
@@ -387,14 +400,7 @@ func (manager *SBucketManager) ValidateCreateData(
 		return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err)
 	}
 
-	managerId, _ := data.GetString("manager_id")
-	if len(managerId) == 0 {
-		return nil, httperrors.NewInternalServerError("empty manager_id???")
-	}
-	cloudprovider := CloudproviderManager.FetchCloudproviderById(managerId)
-	if cloudprovider == nil {
-		return nil, httperrors.NewInternalServerError("invalid cloudprovider???")
-	}
+	cloudprovider := managerV.Model.(*SCloudprovider)
 	quotaPlatformId := cloudprovider.GetQuotaPlatformID()
 	pendingUsage := SQuota{Bucket: 1}
 	if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatformId, &pendingUsage); err != nil {
@@ -751,6 +757,38 @@ func (bucket *SBucket) PerformUpload(
 		}
 	}
 
+	inc := cloudprovider.SBucketStats{}
+	obj, err := cloudprovider.GetIObject(iBucket, key)
+	if err == nil {
+		// replace
+		inc.SizeBytes = sizeBytes - obj.GetSizeBytes()
+		if inc.SizeBytes < 0 {
+			inc.SizeBytes = 0
+		}
+	} else if err == cloudprovider.ErrNotFound {
+		// new upload
+		inc.SizeBytes = sizeBytes
+		inc.ObjectCount = 1
+	} else {
+		return nil, httperrors.NewInternalServerError("GetIObject error %s", err)
+	}
+
+	if bucket.SizeBytesLimit > 0 && inc.SizeBytes > 0 && bucket.SizeBytesLimit < bucket.SizeBytes+inc.SizeBytes {
+		return nil, httperrors.NewOutOfQuotaError("object size limit exceeds")
+	}
+	if bucket.ObjectCntLimit > 0 && inc.ObjectCount > 0 && bucket.ObjectCntLimit < bucket.ObjectCnt+inc.ObjectCount {
+		return nil, httperrors.NewOutOfQuotaError("object count limit exceeds")
+	}
+
+	manager := bucket.GetCloudprovider()
+	quotaPlatformId := manager.GetQuotaPlatformID()
+	pendingUsage := SQuota{ObjectGB: int(inc.SizeBytes / 1000 / 1000 / 1000), ObjectCnt: inc.ObjectCount}
+	if !pendingUsage.IsEmpty() {
+		if err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, bucket.GetOwnerId(), quotaPlatformId, &pendingUsage); err != nil {
+			return nil, httperrors.NewOutOfQuotaError("%s", err)
+		}
+	}
+
 	err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, contType, cloudprovider.TBucketACLType(aclStr), storageClass, false)
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("put object error %s", err)
@@ -758,6 +796,10 @@ func (bucket *SBucket) PerformUpload(
 
 	bucket.syncWithCloudBucket(ctx, userCred, iBucket, nil, true)
 
+	if !pendingUsage.IsEmpty() {
+		QuotaManager.CancelPendingUsage(ctx, userCred, rbacutils.ScopeProject, bucket.GetOwnerId(), quotaPlatformId, &pendingUsage, &pendingUsage)
+	}
+
 	return nil, nil
 }
 
@@ -964,3 +1006,48 @@ func (manager *SBucketManager) TotalCount(scope rbacutils.TRbacScope, ownerId mc
 	}
 	return usage
 }
+
+func (bucket *SBucket) AllowPerformLimit(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) PerformLimit(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	limit := cloudprovider.SBucketStats{}
+	err := data.Unmarshal(&limit, "limit")
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("unmarshal limit error %s", err)
+	}
+
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+	}
+
+	err = iBucket.SetLimit(limit)
+	if err != nil && err != cloudprovider.ErrNotSupported {
+		return nil, httperrors.NewInternalServerError("SetLimit error %s", err)
+	}
+
+	diff, err := db.Update(bucket, func() error {
+		bucket.SizeBytesLimit = limit.SizeBytes
+		bucket.ObjectCntLimit = limit.ObjectCount
+		return nil
+	})
+
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("Update error %s", err)
+	}
+
+	db.OpsLog.LogEvent(bucket, db.ACT_UPDATE, diff, userCred)
+
+	return nil, nil
+}

pkg/compute/models/quotas.go

@@ -67,7 +67,9 @@ type SQuota struct {
 	IsolatedDevice int
 	Snapshot       int
 
-	Bucket int
+	Bucket    int
+	ObjectGB  int
+	ObjectCnt int
 }
 
 func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider) {
@@ -90,6 +92,8 @@ func (self *SQuota) FetchSystemQuota(scope rbacutils.TRbacScope, ownerId mcclien
 	self.IsolatedDevice = options.Options.DefaultIsolatedDeviceQuota * base
 	self.Snapshot = options.Options.DefaultSnapshotQuota * base
 	self.Bucket = options.Options.DefaultBucketQuota * base
+	self.ObjectGB = options.Options.DefaultObjectGBQuota * base
+	self.ObjectCnt = options.Options.DefaultObjectCntQuota * base
 }
 
 func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope, ownerId mcclient.IIdentityProvider, name []string) error {
@@ -125,6 +129,8 @@ func (self *SQuota) FetchUsage(ctx context.Context, scope rbacutils.TRbacScope,
 	self.IsolatedDevice = guest.TotalIsolatedCount
 	self.Snapshot = snapshotCount
 	self.Bucket = bucketUsage.Buckets
+	self.ObjectGB = int(bucketUsage.Bytes / 1000 / 1000 / 1000)
+	self.ObjectCnt = bucketUsage.Objects
 	return nil
 }
 
@@ -168,6 +174,12 @@ func (self *SQuota) IsEmpty() bool {
 	if self.Bucket > 0 {
 		return false
 	}
+	if self.ObjectGB > 0 {
+		return false
+	}
+	if self.ObjectCnt > 0 {
+		return false
+	}
 	return true
 }
 
@@ -186,6 +198,8 @@ func (self *SQuota) Add(quota quotas.IQuota) {
 	self.IsolatedDevice = self.IsolatedDevice + squota.IsolatedDevice
 	self.Snapshot = self.Snapshot + squota.Snapshot
 	self.Bucket = self.Bucket + squota.Bucket
+	self.ObjectGB = self.ObjectGB + squota.ObjectGB
+	self.ObjectCnt = self.ObjectCnt + squota.ObjectCnt
 }
 
 func nonNegative(val int) int {
@@ -207,6 +221,8 @@ func (self *SQuota) Sub(quota quotas.IQuota) {
 	self.IsolatedDevice = nonNegative(self.IsolatedDevice - squota.IsolatedDevice)
 	self.Snapshot = nonNegative(self.Snapshot - squota.Snapshot)
 	self.Bucket = nonNegative(self.Bucket - squota.Bucket)
+	self.ObjectGB = nonNegative(self.ObjectGB - squota.ObjectGB)
+	self.ObjectCnt = nonNegative(self.ObjectCnt - squota.ObjectCnt)
 }
 
 func (self *SQuota) Update(quota quotas.IQuota) {
@@ -250,6 +266,12 @@ func (self *SQuota) Update(quota quotas.IQuota) {
 	if squota.Bucket > 0 {
 		self.Bucket = squota.Bucket
 	}
+	if squota.ObjectGB > 0 {
+		self.ObjectGB = squota.ObjectGB
+	}
+	if squota.ObjectCnt > 0 {
+		self.ObjectCnt = squota.ObjectCnt
+	}
 }
 
 func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error {
@@ -295,6 +317,12 @@ func (self *SQuota) Exceed(request quotas.IQuota, quota quotas.IQuota) error {
 	if sreq.Bucket > 0 && self.Bucket > squota.Bucket {
 		err.Add("bucket", squota.Bucket, self.Bucket)
 	}
+	if sreq.ObjectGB > 0 && self.ObjectGB > squota.ObjectGB {
+		err.Add("object_gb", squota.ObjectGB, self.ObjectGB)
+	}
+	if sreq.ObjectCnt > 0 && self.ObjectCnt > squota.ObjectCnt {
+		err.Add("object_cnt", squota.ObjectCnt, self.ObjectCnt)
+	}
 	if err.IsError() {
 		return err
 	} else {
@@ -325,5 +353,7 @@ func (self *SQuota) ToJSON(prefix string) jsonutils.JSONObject {
 	ret.Add(jsonutils.NewInt(int64(self.IsolatedDevice)), keyName(prefix, "isolated_device"))
 	ret.Add(jsonutils.NewInt(int64(self.Snapshot)), keyName(prefix, "snapshot"))
 	ret.Add(jsonutils.NewInt(int64(self.Bucket)), keyName(prefix, "bucket"))
+	ret.Add(jsonutils.NewInt(int64(self.ObjectGB)), keyName(prefix, "object_gb"))
+	ret.Add(jsonutils.NewInt(int64(self.ObjectCnt)), keyName(prefix, "object_cnt"))
 	return ret
 }

pkg/compute/options/options.go

@@ -64,6 +64,8 @@ type ComputeOptions struct {
 	DefaultIsolatedDeviceQuota int `default:"200" help:"Common isolated device quota per tenant, default 200"`
 	DefaultSnapshotQuota       int `default:"10" help:"Common snapshot quota per tenant, default 10"`
 	DefaultBucketQuota         int `default:"100" help:"Common bucket quota per tenant, default 100"`
+	DefaultObjectGBQuota       int `default:"100" help:"Common object size quota per tenant in GB, default 100GB"`
+	DefaultObjectCntQuota      int `default:"500" help:"Common object count quota per tenant, default 500"`
 
 	SystemAdminQuotaCheck bool `help:"Enable quota check for system admin, default False" default:"false"`
 

pkg/multicloud/bucket_base.go

@@ -14,7 +14,10 @@
 
 package multicloud
 
-import "yunion.io/x/jsonutils"
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
 
 type SBaseBucket struct{}
 
@@ -53,3 +56,11 @@ func (b *SBaseBucket) IsEmulated() bool {
 func (b *SBaseBucket) GetMetadata() *jsonutils.JSONDict {
 	return nil
 }
+
+func (b *SBaseBucket) GetLimit() cloudprovider.SBucketStats {
+	return cloudprovider.SBucketStats{}
+}
+
+func (b *SBaseBucket) SetLimit(limit cloudprovider.SBucketStats) error {
+	return nil
+}

pkg/multicloud/huawei/bucket.go

@@ -390,3 +390,33 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s
 	output, err := obscli.CreateSignedUrl(&input)
 	return output.SignedUrl, nil
 }
+
+func (b *SBucket) GetLimit() cloudprovider.SBucketStats {
+	stats := cloudprovider.SBucketStats{}
+	obscli, err := b.region.getOBSClient()
+	if err != nil {
+		log.Errorf("getOBSClient error %s", err)
+		return stats
+	}
+	output, err := obscli.GetBucketQuota(b.Name)
+	if err != nil {
+		return stats
+	}
+	stats.SizeBytes = output.Quota
+	return stats
+}
+
+func (b *SBucket) SetLimit(limit cloudprovider.SBucketStats) error {
+	obscli, err := b.region.getOBSClient()
+	if err != nil {
+		return errors.Wrap(err, "getOBSClient")
+	}
+	input := &obs.SetBucketQuotaInput{}
+	input.Bucket = b.Name
+	input.Quota = limit.SizeBytes
+	_, err = obscli.SetBucketQuota(input)
+	if err != nil {
+		return errors.Wrap(err, "SetBucketQuota")
+	}
+	return nil
+}

pkg/multicloud/qcloud/storagecache.go

@@ -167,7 +167,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To
 		return "", err
 	}
 
-	bucketName := strings.ReplaceAll(strings.ToLower(self.region.GetId()+image.ImageId), "-", "")
+	bucketName := strings.Replace(strings.ToLower(self.region.GetId()+image.ImageId), "-", "", -1)
 	if len(bucketName) > 40 {
 		bucketName = bucketName[:40]
 	}