opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-19 02:37:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: bucket list objects with marker and limit

Browse Source
This commit is contained in:
Qiu Jian
2019-12-25 23:06:16 +08:00
parent e617a3e404
commit 7433134cb4
10 changed files with 87 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/climc/shell/buckets.go
+10 -5
View File
@@ -107,9 +107,11 @@ func init() {
})
type BucketListObjectsOptions struct {
ID string `help:"ID or name of bucket" json:"-"`
Prefix string `help:"List objects with prefix"`
Recursive bool `help:"List objects recursively"`
ID string `help:"ID or name of bucket" json:"-"`
Prefix string `help:"List objects with prefix"`
Recursive bool `help:"List objects recursively"`
Limit int `help:"maximal items per request"`
PagingMarker string `help:"paging marker"`
}
R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error {
params, err := options.StructToParams(args)
@@ -121,8 +123,11 @@ func init() {
return err
}
arrays, _ := result.GetArray("objects")
listResult := modulebase.ListResult{Data: arrays}
listResult := modulebase.ListResult{}
err = result.Unmarshal(&listResult)
if err != nil {
return err
}
printList(&listResult, []string{})
return nil
})
pkg/cloudprovider/objectstore.go
+49 -32
View File
@@ -28,6 +28,7 @@ import (
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/s3cli"
"yunion.io/x/onecloud/pkg/httperrors"
)
type TBucketACLType string
@@ -148,7 +149,6 @@ type ICloudBucket interface {
SetAcl(acl TBucketACLType) error
ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error)
GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error)
CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl TBucketACLType, storageClassStr string, meta http.Header) error
GetObject(ctx context.Context, key string, rangeOpt *SGetObjectRange) (io.ReadCloser, error)
@@ -257,57 +257,74 @@ func GetIBucketByName(region ICloudRegion, name string) (ICloudBucket, error) {
}
func GetIBucketStats(bucket ICloudBucket) (SBucketStats, error) {
stats := SBucketStats{}
objs, err := bucket.GetIObjects("", true)
stats := SBucketStats{
ObjectCount: -1,
SizeBytes: -1,
}
objs, err := bucket.ListObjects("", "", "", 1000)
if err != nil {
stats.ObjectCount = -1
stats.SizeBytes = -1
return stats, errors.Wrap(err, "GetIObjects")
}
for _, obj := range objs {
if objs.IsTruncated {
return stats, errors.Wrap(httperrors.ErrTooLarge, "too many objects")
}
for _, obj := range objs.Objects {
stats.SizeBytes += obj.GetSizeBytes()
stats.ObjectCount += 1
}
return stats, nil
}
func GetIObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) {
func GetPagedObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool, marker string, maxCount int) ([]ICloudObject, string, error) {
delimiter := "/"
if isRecursive {
delimiter = ""
}
if maxCount > 1000 || maxCount <= 0 {
maxCount = 1000
}
ret := make([]ICloudObject, 0)
result, err := bucket.ListObjects(objectPrefix, marker, delimiter, maxCount)
if err != nil {
return nil, "", errors.Wrap(err, "bucket.ListObjects")
}
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
if len(result.CommonPrefixes) > 0 {
ret = append(ret, result.CommonPrefixes...)
}
// Send all objects
for i := range result.Objects {
// if delimited, skip the first object
if !isRecursive && result.Objects[i].GetKey() == objectPrefix {
continue
}
ret = append(ret, result.Objects[i])
marker = result.Objects[i].GetKey()
}
// If next marker present, save it for next request.
if result.NextMarker != "" {
marker = result.NextMarker
}
// If not truncated, no more objects
if !result.IsTruncated {
marker = ""
}
return ret, marker, nil
}
func GetAllObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) {
ret := make([]ICloudObject, 0)
// Save marker for next request.
var marker string
for {
// Get list of objects a maximum of 1000 per request.
result, err := bucket.ListObjects(objectPrefix, marker, delimiter, 1000)
result, marker, err := GetPagedObjects(bucket, objectPrefix, isRecursive, marker, 1000)
if err != nil {
return nil, errors.Wrap(err, "bucket.ListObjects")
}
// Send all objects
for i := range result.Objects {
if !isRecursive && result.Objects[i].GetKey() == objectPrefix {
continue
}
ret = append(ret, result.Objects[i])
marker = result.Objects[i].GetKey()
}
// Send all common prefixes if any.
// NOTE: prefixes are only present if the request is delimited.
if len(result.CommonPrefixes) > 0 {
ret = append(ret, result.CommonPrefixes...)
}
// If next marker present, save it for next request.
if result.NextMarker != "" {
marker = result.NextMarker
}
// Listing ends result is not truncated, break the loop
if !result.IsTruncated {
ret = append(ret, result...)
if marker == "" {
break
}
}
@@ -411,7 +428,7 @@ func UploadObject(ctx context.Context, bucket ICloudBucket, key string, blocksz
}
func DeletePrefix(ctx context.Context, bucket ICloudBucket, prefix string) error {
objs, err := bucket.GetIObjects(prefix, true)
objs, err := GetAllObjects(bucket, prefix, true)
if err != nil {
return errors.Wrap(err, "bucket.GetIObjects")
}
pkg/compute/models/buckets.go
+13 -3
View File
@@ -644,7 +644,14 @@ func (bucket *SBucket) GetDetailsObjects(
}
prefix, _ := query.GetString("prefix")
isRecursive := jsonutils.QueryBoolean(query, "recursive", false)
objects, err := iBucket.GetIObjects(prefix, isRecursive)
marker, _ := query.GetString("paging_marker")
limit, _ := query.Int("limit")
if limit <= 0 {
limit = 50
} else if limit > 1000 {
limit = 1000
}
objects, nextMarker, err := cloudprovider.GetPagedObjects(iBucket, prefix, isRecursive, marker, int(limit))
if err != nil {
return nil, httperrors.NewInternalServerError("fail to get objects: %s", err)
}
@@ -653,7 +660,10 @@ func (bucket *SBucket) GetDetailsObjects(
retArray.Add(cloudprovider.ICloudObject2JSONObject(objects[i]))
}
ret := jsonutils.NewDict()
ret.Add(retArray, "objects")
ret.Add(retArray, "data")
if len(nextMarker) > 0 {
ret.Add(jsonutils.NewString(nextMarker), "next_marker")
}
return ret, nil
}
@@ -1286,7 +1296,7 @@ func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionIn
objects := make([]cloudprovider.ICloudObject, 0)
for _, key := range input.Key {
if strings.HasSuffix(key, "/") {
objs, err := cloudprovider.GetIObjects(iBucket, key, true)
objs, err := cloudprovider.GetAllObjects(iBucket, key, true)
if err != nil {
return nil, nil, httperrors.NewInternalServerError("iBucket.GetIObjects error %s", err)
}
pkg/multicloud/aliyun/bucket.go
-4
View File
@@ -179,10 +179,6 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma
return result, nil
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func metaOpts(opts []oss.Option, meta http.Header) []oss.Option {
for k, v := range meta {
if len(v) == 0 {
pkg/multicloud/aws/bucket.go
-4
View File
@@ -206,10 +206,6 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma
return result, nil
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error {
if sizeBytes < 0 {
return errors.Error("content length expected")
pkg/multicloud/azure/storageaccount.go
-4
View File
@@ -863,10 +863,6 @@ func (b *SStorageAccount) GetStats() cloudprovider.SBucketStats {
return stats
}
func (b *SStorageAccount) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func getBlobRefMeta(blob *storage.Blob) http.Header {
meta := http.Header{}
for k, v := range blob.Metadata {
pkg/multicloud/huawei/bucket.go
-4
View File
@@ -169,10 +169,6 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats {
return stats
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
obscli, err := b.region.getOBSClient()
pkg/multicloud/objectstore/shell.go
+15 -3
View File
@@ -191,17 +191,22 @@ func S3Shell() {
type BucketListObjectsOptions struct {
BUCKET string `help:"name of bucket to list objects"`
Prefix string `help:"prefix"`
Limit int `help:"limit per page request" default:"20"`
Marker string `help:"offset marker"`
}
shellutils.R(&BucketListObjectsOptions{}, "bucket-list-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
objects, err := bucket.GetIObjects(args.Prefix, true)
objects, marker, err := cloudprovider.GetPagedObjects(bucket, args.Prefix, true, args.Marker, args.Limit)
if err != nil {
return err
}
printutils.PrintGetterList(objects, []string{"key", "size_bytes"})
if len(marker) > 0 {
fmt.Println("Next marker:", marker)
}
return nil
})
@@ -210,11 +215,14 @@ func S3Shell() {
if err != nil {
return err
}
objects, err := bucket.GetIObjects(args.Prefix, false)
objects, marker, err := cloudprovider.GetPagedObjects(bucket, args.Prefix, false, args.Marker, args.Limit)
if err != nil {
return err
}
printutils.PrintGetterList(objects, []string{"key", "size_bytes"})
if len(marker) > 0 {
fmt.Println("Next marker:", marker)
}
return nil
})
@@ -289,7 +297,11 @@ func S3Shell() {
if err != nil {
return err
}
err = bucket.DeleteObject(context.Background(), args.KEY)
if strings.HasSuffix(args.KEY, "/") {
err = cloudprovider.DeletePrefix(context.Background(), bucket, args.KEY)
} else {
err = bucket.DeleteObject(context.Background(), args.KEY)
}
if err != nil {
return err
}
pkg/multicloud/qcloud/bucket.go
-4
View File
@@ -165,10 +165,6 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats {
return stats
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}
coscli, err := b.region.GetCosClient(b)
pkg/multicloud/ucloud/ufile.go
-4
View File
@@ -326,10 +326,6 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats {
return stats
}
func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
return cloudprovider.GetIObjects(b, prefix, isRecursive)
}
func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
result := cloudprovider.SListObjectResult{}