From 7433134cb4076dee4ff813183089f84ee6a82f38 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Wed, 25 Dec 2019 23:06:16 +0800 Subject: [PATCH] fix: bucket list objects with marker and limit --- cmd/climc/shell/buckets.go | 15 +++-- pkg/cloudprovider/objectstore.go | 81 ++++++++++++++++---------- pkg/compute/models/buckets.go | 16 ++++- pkg/multicloud/aliyun/bucket.go | 4 -- pkg/multicloud/aws/bucket.go | 4 -- pkg/multicloud/azure/storageaccount.go | 4 -- pkg/multicloud/huawei/bucket.go | 4 -- pkg/multicloud/objectstore/shell.go | 18 +++++- pkg/multicloud/qcloud/bucket.go | 4 -- pkg/multicloud/ucloud/ufile.go | 4 -- 10 files changed, 87 insertions(+), 67 deletions(-) diff --git a/cmd/climc/shell/buckets.go b/cmd/climc/shell/buckets.go index 55698c0c95..f3fef34d73 100644 --- a/cmd/climc/shell/buckets.go +++ b/cmd/climc/shell/buckets.go @@ -107,9 +107,11 @@ func init() { }) type BucketListObjectsOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - Prefix string `help:"List objects with prefix"` - Recursive bool `help:"List objects recursively"` + ID string `help:"ID or name of bucket" json:"-"` + Prefix string `help:"List objects with prefix"` + Recursive bool `help:"List objects recursively"` + Limit int `help:"maximal items per request"` + PagingMarker string `help:"paging marker"` } R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error { params, err := options.StructToParams(args) @@ -121,8 +123,11 @@ func init() { return err } - arrays, _ := result.GetArray("objects") - listResult := modulebase.ListResult{Data: arrays} + listResult := modulebase.ListResult{} + err = result.Unmarshal(&listResult) + if err != nil { + return err + } printList(&listResult, []string{}) return nil }) diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index 89bef462f9..058bfb11a3 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -28,6 +28,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/s3cli" + "yunion.io/x/onecloud/pkg/httperrors" ) type TBucketACLType string @@ -148,7 +149,6 @@ type ICloudBucket interface { SetAcl(acl TBucketACLType) error ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error) - GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl TBucketACLType, storageClassStr string, meta http.Header) error GetObject(ctx context.Context, key string, rangeOpt *SGetObjectRange) (io.ReadCloser, error) @@ -257,57 +257,74 @@ func GetIBucketByName(region ICloudRegion, name string) (ICloudBucket, error) { } func GetIBucketStats(bucket ICloudBucket) (SBucketStats, error) { - stats := SBucketStats{} - objs, err := bucket.GetIObjects("", true) + stats := SBucketStats{ + ObjectCount: -1, + SizeBytes: -1, + } + objs, err := bucket.ListObjects("", "", "", 1000) if err != nil { - stats.ObjectCount = -1 - stats.SizeBytes = -1 return stats, errors.Wrap(err, "GetIObjects") } - for _, obj := range objs { + if objs.IsTruncated { + return stats, errors.Wrap(httperrors.ErrTooLarge, "too many objects") + } + for _, obj := range objs.Objects { stats.SizeBytes += obj.GetSizeBytes() stats.ObjectCount += 1 } return stats, nil } -func GetIObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) { +func GetPagedObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool, marker string, maxCount int) ([]ICloudObject, string, error) { delimiter := "/" if isRecursive { delimiter = "" } + if maxCount > 1000 || maxCount <= 0 { + maxCount = 1000 + } + ret := make([]ICloudObject, 0) + result, err := bucket.ListObjects(objectPrefix, marker, delimiter, maxCount) + if err != nil { + return nil, "", errors.Wrap(err, "bucket.ListObjects") + } + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + if len(result.CommonPrefixes) > 0 { + ret = append(ret, result.CommonPrefixes...) + } + // Send all objects + for i := range result.Objects { + // if delimited, skip the first object + if !isRecursive && result.Objects[i].GetKey() == objectPrefix { + continue + } + ret = append(ret, result.Objects[i]) + marker = result.Objects[i].GetKey() + } + // If next marker present, save it for next request. + if result.NextMarker != "" { + marker = result.NextMarker + } + // If not truncated, no more objects + if !result.IsTruncated { + marker = "" + } + return ret, marker, nil +} + +func GetAllObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) { ret := make([]ICloudObject, 0) // Save marker for next request. var marker string for { // Get list of objects a maximum of 1000 per request. - result, err := bucket.ListObjects(objectPrefix, marker, delimiter, 1000) + result, marker, err := GetPagedObjects(bucket, objectPrefix, isRecursive, marker, 1000) if err != nil { return nil, errors.Wrap(err, "bucket.ListObjects") } - - // Send all objects - for i := range result.Objects { - if !isRecursive && result.Objects[i].GetKey() == objectPrefix { - continue - } - ret = append(ret, result.Objects[i]) - marker = result.Objects[i].GetKey() - } - - // Send all common prefixes if any. - // NOTE: prefixes are only present if the request is delimited. - if len(result.CommonPrefixes) > 0 { - ret = append(ret, result.CommonPrefixes...) - } - - // If next marker present, save it for next request. - if result.NextMarker != "" { - marker = result.NextMarker - } - - // Listing ends result is not truncated, break the loop - if !result.IsTruncated { + ret = append(ret, result...) + if marker == "" { break } } @@ -411,7 +428,7 @@ func UploadObject(ctx context.Context, bucket ICloudBucket, key string, blocksz } func DeletePrefix(ctx context.Context, bucket ICloudBucket, prefix string) error { - objs, err := bucket.GetIObjects(prefix, true) + objs, err := GetAllObjects(bucket, prefix, true) if err != nil { return errors.Wrap(err, "bucket.GetIObjects") } diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 1959fc807e..5d449effba 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -644,7 +644,14 @@ func (bucket *SBucket) GetDetailsObjects( } prefix, _ := query.GetString("prefix") isRecursive := jsonutils.QueryBoolean(query, "recursive", false) - objects, err := iBucket.GetIObjects(prefix, isRecursive) + marker, _ := query.GetString("paging_marker") + limit, _ := query.Int("limit") + if limit <= 0 { + limit = 50 + } else if limit > 1000 { + limit = 1000 + } + objects, nextMarker, err := cloudprovider.GetPagedObjects(iBucket, prefix, isRecursive, marker, int(limit)) if err != nil { return nil, httperrors.NewInternalServerError("fail to get objects: %s", err) } @@ -653,7 +660,10 @@ func (bucket *SBucket) GetDetailsObjects( retArray.Add(cloudprovider.ICloudObject2JSONObject(objects[i])) } ret := jsonutils.NewDict() - ret.Add(retArray, "objects") + ret.Add(retArray, "data") + if len(nextMarker) > 0 { + ret.Add(jsonutils.NewString(nextMarker), "next_marker") + } return ret, nil } @@ -1286,7 +1296,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn objects := make([]cloudprovider.ICloudObject, 0) for _, key := range input.Key { if strings.HasSuffix(key, "/") { - objs, err := cloudprovider.GetIObjects(iBucket, key, true) + objs, err := cloudprovider.GetAllObjects(iBucket, key, true) if err != nil { return nil, nil, httperrors.NewInternalServerError("iBucket.GetIObjects error %s", err) } diff --git a/pkg/multicloud/aliyun/bucket.go b/pkg/multicloud/aliyun/bucket.go index 459b9c2d3c..e3833538ed 100644 --- a/pkg/multicloud/aliyun/bucket.go +++ b/pkg/multicloud/aliyun/bucket.go @@ -179,10 +179,6 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma return result, nil } -func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { - return cloudprovider.GetIObjects(b, prefix, isRecursive) -} - func metaOpts(opts []oss.Option, meta http.Header) []oss.Option { for k, v := range meta { if len(v) == 0 { diff --git a/pkg/multicloud/aws/bucket.go b/pkg/multicloud/aws/bucket.go index e9d7d63b0e..70cfc2b9e2 100644 --- a/pkg/multicloud/aws/bucket.go +++ b/pkg/multicloud/aws/bucket.go @@ -206,10 +206,6 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma return result, nil } -func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { - return cloudprovider.GetIObjects(b, prefix, isRecursive) -} - func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { if sizeBytes < 0 { return errors.Error("content length expected") diff --git a/pkg/multicloud/azure/storageaccount.go b/pkg/multicloud/azure/storageaccount.go index 49cfc1f4f0..5213477c68 100644 --- a/pkg/multicloud/azure/storageaccount.go +++ b/pkg/multicloud/azure/storageaccount.go @@ -863,10 +863,6 @@ func (b *SStorageAccount) GetStats() cloudprovider.SBucketStats { return stats } -func (b *SStorageAccount) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { - return cloudprovider.GetIObjects(b, prefix, isRecursive) -} - func getBlobRefMeta(blob *storage.Blob) http.Header { meta := http.Header{} for k, v := range blob.Metadata { diff --git a/pkg/multicloud/huawei/bucket.go b/pkg/multicloud/huawei/bucket.go index 8a3cd45139..bc43465ee8 100644 --- a/pkg/multicloud/huawei/bucket.go +++ b/pkg/multicloud/huawei/bucket.go @@ -169,10 +169,6 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats { return stats } -func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { - return cloudprovider.GetIObjects(b, prefix, isRecursive) -} - func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { result := cloudprovider.SListObjectResult{} obscli, err := b.region.getOBSClient() diff --git a/pkg/multicloud/objectstore/shell.go b/pkg/multicloud/objectstore/shell.go index a8acfe7b4e..f5ea0e2df6 100644 --- a/pkg/multicloud/objectstore/shell.go +++ b/pkg/multicloud/objectstore/shell.go @@ -191,17 +191,22 @@ func S3Shell() { type BucketListObjectsOptions struct { BUCKET string `help:"name of bucket to list objects"` Prefix string `help:"prefix"` + Limit int `help:"limit per page request" default:"20"` + Marker string `help:"offset marker"` } shellutils.R(&BucketListObjectsOptions{}, "bucket-list-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error { bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } - objects, err := bucket.GetIObjects(args.Prefix, true) + objects, marker, err := cloudprovider.GetPagedObjects(bucket, args.Prefix, true, args.Marker, args.Limit) if err != nil { return err } printutils.PrintGetterList(objects, []string{"key", "size_bytes"}) + if len(marker) > 0 { + fmt.Println("Next marker:", marker) + } return nil }) @@ -210,11 +215,14 @@ func S3Shell() { if err != nil { return err } - objects, err := bucket.GetIObjects(args.Prefix, false) + objects, marker, err := cloudprovider.GetPagedObjects(bucket, args.Prefix, false, args.Marker, args.Limit) if err != nil { return err } printutils.PrintGetterList(objects, []string{"key", "size_bytes"}) + if len(marker) > 0 { + fmt.Println("Next marker:", marker) + } return nil }) @@ -289,7 +297,11 @@ func S3Shell() { if err != nil { return err } - err = bucket.DeleteObject(context.Background(), args.KEY) + if strings.HasSuffix(args.KEY, "/") { + err = cloudprovider.DeletePrefix(context.Background(), bucket, args.KEY) + } else { + err = bucket.DeleteObject(context.Background(), args.KEY) + } if err != nil { return err } diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index a6de26ec0b..c6e54bba26 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -165,10 +165,6 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats { return stats } -func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { - return cloudprovider.GetIObjects(b, prefix, isRecursive) -} - func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { result := cloudprovider.SListObjectResult{} coscli, err := b.region.GetCosClient(b) diff --git a/pkg/multicloud/ucloud/ufile.go b/pkg/multicloud/ucloud/ufile.go index f08071a330..f2793d1521 100644 --- a/pkg/multicloud/ucloud/ufile.go +++ b/pkg/multicloud/ucloud/ufile.go @@ -326,10 +326,6 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats { return stats } -func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { - return cloudprovider.GetIObjects(b, prefix, isRecursive) -} - func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { result := cloudprovider.SListObjectResult{}