From 4b2316208c92aea0b69f5ef2dd655feb8a989378 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 12 Aug 2019 10:34:51 +0800 Subject: [PATCH 1/3] feature: s3upload support multipart upload --- docs/bucket/uploadobject.yaml | 2 + docs/parameters/bucket.yaml | 18 ++- docs/schemas/bucket.yaml | 4 +- pkg/apis/compute/bucket.go | 1 + pkg/cloudprovider/objectstore.go | 86 +++++++++++++- pkg/compute/models/buckets.go | 28 ++++- pkg/image/models/images.go | 11 +- pkg/mcclient/modules/mod_images.go | 12 +- pkg/multicloud/aliyun/bucket.go | 124 +++++++++++++++++++- pkg/multicloud/aliyun/objects.go | 6 +- pkg/multicloud/aliyun/storagecache.go | 21 ++-- pkg/multicloud/aws/bucket.go | 127 ++++++++++++++++++--- pkg/multicloud/aws/s3object.go | 2 +- pkg/multicloud/aws/storagecache.go | 106 ++++++----------- pkg/multicloud/azure/blobobject.go | 2 +- pkg/multicloud/azure/storageaccount.go | 138 ++++++++++++++++++++++- pkg/multicloud/azure/storagecache.go | 6 +- pkg/multicloud/bucket_base.go | 8 ++ pkg/multicloud/huawei/bucket.go | 103 ++++++++++++++++- pkg/multicloud/huawei/object.go | 2 +- pkg/multicloud/huawei/storagecache.go | 50 ++++---- pkg/multicloud/objectstore/buckets.go | 72 +++++++++++- pkg/multicloud/objectstore/shell.go | 14 ++- pkg/multicloud/openstack/storagecache.go | 2 +- pkg/multicloud/qcloud/bucket.go | 91 ++++++++++++++- pkg/multicloud/qcloud/object.go | 8 +- pkg/multicloud/qcloud/storagecache.go | 6 +- pkg/multicloud/ucloud/storagecache.go | 4 +- pkg/multicloud/ucloud/ufile.go | 20 +++- pkg/multicloud/zstack/storagecache.go | 4 +- pkg/util/fileutils2/seeker.go | 81 +++++++++++++ pkg/util/fileutils2/seeker_test.go | 31 +++++ pkg/util/httputils/httputils.go | 6 +- 33 files changed, 1008 insertions(+), 188 deletions(-) create mode 100644 pkg/util/fileutils2/seeker.go create mode 100644 pkg/util/fileutils2/seeker_test.go diff --git a/docs/bucket/uploadobject.yaml b/docs/bucket/uploadobject.yaml index 17c58f4a9e..a078a49729 100644 --- a/docs/bucket/uploadobject.yaml +++ b/docs/bucket/uploadobject.yaml @@ -4,7 +4,9 @@ post: - $ref: "../parameters/bucket.yaml#/bucket_name" - $ref: "../parameters/bucket.yaml#/x-bucket-object-key" - $ref: "../parameters/bucket.yaml#/x-bucket-content-type" + - $ref: "../parameters/bucket.yaml#/x-bucket-content-length" - $ref: "../parameters/bucket.yaml#/x-bucket-storage-class" + - $ref: "../parameters/bucket.yaml#/x-bucket-acl" responses: 200: description: 上传成功 diff --git a/docs/parameters/bucket.yaml b/docs/parameters/bucket.yaml index aee480aa02..e7654fe77e 100644 --- a/docs/parameters/bucket.yaml +++ b/docs/parameters/bucket.yaml @@ -18,19 +18,31 @@ recursive: description: 是否展开对象列表，false则只显示当前目录层级下的对象，true则显示匹配前缀的所有对象 x-bucket-object-key: - name: x-bucket-object-key + name: X-Yunion-Bucket-Upload-Key type: string in: header description: 对象的key x-bucket-content-type: - name: x-bucket-content-type + name: Content-Type type: string in: header description: 对象的content-type +x-bucket-content-length: + name: Content-Length + type: string + in: header + description: 对象的content-length + x-bucket-storage-class: - name: x-bucket-storage-class + name: X-Yunion-Bucket-Upload-Storageclass type: string in: header description: 对象的storage_class + +x-bucket-acl: + name: X-Yunion-Bucket-Upload-Acl + type: string + in: header + description: 对象的acl，可能值为：private, public-read, public-read-write diff --git a/docs/schemas/bucket.yaml b/docs/schemas/bucket.yaml index e4e718fbc6..79ac6a7ee5 100644 --- a/docs/schemas/bucket.yaml +++ b/docs/schemas/bucket.yaml @@ -187,7 +187,7 @@ BucketGetACLResponse: properties: acl: type: string - description: bucket或者对象的ACL字串，可能为private, public-read, public-read-write, default(仅object支持) + description: bucket或者对象的ACL字串，可能为private, public-read, public-read-write, authenticated-read BucketMakedirInput: type: object @@ -202,7 +202,7 @@ BucketSetACLInput: acl: type: string required: true - description: bucket或者对象的ACL字串，可能为private, public-read, public-read-write, default(仅object支持) + description: bucket或者对象的ACL字串，可能为private, public-read, public-read-write key: type: string description: 如果设置对象的ACL，则此字段指定对象的key diff --git a/pkg/apis/compute/bucket.go b/pkg/apis/compute/bucket.go index 683ab7e820..4b1fd5ec81 100644 --- a/pkg/apis/compute/bucket.go +++ b/pkg/apis/compute/bucket.go @@ -27,5 +27,6 @@ const ( BUCKET_STATUS_DELETE_FAIL = "delete_fail" BUCKET_UPLOAD_OBJECT_KEY_HEADER = "X-Yunion-Bucket-Upload-Key" + BUCKET_UPLOAD_OBJECT_ACL_HEADER = "X-Yunion-Bucket-Upload-Acl" BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER = "X-Yunion-Bucket-Upload-Storageclass" ) diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index 95f75a5547..221b44b08b 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -17,10 +17,10 @@ package cloudprovider import ( "context" "io" + "strings" "time" - "strings" - + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/s3cli" ) @@ -28,7 +28,10 @@ import ( type TBucketACLType string const ( - ACLDefault = TBucketACLType("default") + // 100 MB + MAX_PUT_OBJECT_SIZEBYTES = int64(1000 * 1000 * 100) + + // ACLDefault = TBucketACLType("default") ACLPrivate = TBucketACLType(s3cli.CANNED_ACL_PRIVATE) ACLAuthRead = TBucketACLType(s3cli.CANNED_ACL_AUTH_READ) @@ -75,6 +78,9 @@ type SListObjectResult struct { type ICloudBucket interface { IVirtualResource + MaxPartCount() int + MaxPartSizeBytes() int64 + //GetGlobalId() string //GetName() string GetAcl() TBucketACLType @@ -89,10 +95,15 @@ type ICloudBucket interface { ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error) GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error) - PutObject(ctx context.Context, key string, input io.Reader, contType string, storageClass string) error + DeleteObject(ctx context.Context, keys string) error GetTempUrl(method string, key string, expire time.Duration) (string, error) - // ObjectExist(key string) (bool, error) + + PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, contType string, cannedAcl TBucketACLType, storageClassStr string) error + NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl TBucketACLType, storageClassStr string) (string, error) + UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) + CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error + AbortMultipartUpload(ctx context.Context, key string, uploadId string) error } type ICloudObject interface { @@ -247,9 +258,72 @@ func Makedir(ctx context.Context, bucket ICloudBucket, key string) error { } } path := strings.Join(segs, "/") + "/" - err := bucket.PutObject(ctx, path, strings.NewReader(""), "", "") + err := bucket.PutObject(ctx, path, strings.NewReader(""), 0, "", ACLPrivate, "") if err != nil { return errors.Wrap(err, "PutObject") } return nil } + +func UploadObject(ctx context.Context, bucket ICloudBucket, key string, blocksz int64, input io.Reader, sizeBytes int64, contType string, cannedAcl TBucketACLType, storageClass string, debug bool) error { + if blocksz <= 0 { + blocksz = MAX_PUT_OBJECT_SIZEBYTES + } + if sizeBytes < blocksz { + if debug { + log.Debugf("too small, put object in one shot") + } + return bucket.PutObject(ctx, key, input, sizeBytes, contType, cannedAcl, storageClass) + } + partSize := blocksz + partCount := sizeBytes / partSize + if partCount*partSize < sizeBytes { + partCount += 1 + } + if partCount > int64(bucket.MaxPartCount()) { + partCount = int64(bucket.MaxPartCount()) + partSize = sizeBytes / partCount + if partSize*partCount < sizeBytes { + partSize += 1 + } + if partSize > bucket.MaxPartSizeBytes() { + return errors.Error("too larget object") + } + } + if debug { + log.Debugf("multipart upload part count %d part size %d", partCount, partSize) + } + uploadId, err := bucket.NewMultipartUpload(ctx, key, contType, cannedAcl, storageClass) + if err != nil { + return errors.Wrap(err, "bucket.NewMultipartUpload") + } + etags := make([]string, partCount) + // offset := int64(0) + for i := 0; i < int(partCount); i += 1 { + if i == int(partCount)-1 { + partSize = sizeBytes - partSize*(partCount-1) + } + if debug { + log.Debugf("UploadPart %d %d", i+1, partSize) + } + etag, err := bucket.UploadPart(ctx, key, uploadId, i+1, io.LimitReader(input, partSize), partSize) + if err != nil { + err2 := bucket.AbortMultipartUpload(ctx, key, uploadId) + if err2 != nil { + log.Errorf("bucket.AbortMultipartUpload error %s", err2) + } + return errors.Wrap(err, "bucket.UploadPart") + } + // offset += partSize + etags[i] = etag + } + err = bucket.CompleteMultipartUpload(ctx, key, uploadId, etags) + if err != nil { + err2 := bucket.AbortMultipartUpload(ctx, key, uploadId) + if err2 != nil { + log.Errorf("bucket.AbortMultipartUpload error %s", err2) + } + return errors.Wrap(err, "CompleteMultipartUpload") + } + return nil +} diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 085c09ae56..b24cf36762 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "strconv" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" @@ -709,8 +710,27 @@ func (bucket *SBucket) PerformUpload( } contType := appParams.Request.Header.Get("Content-Type") + sizeStr := appParams.Request.Header.Get("Content-Length") + if len(sizeStr) == 0 { + return nil, httperrors.NewInputParameterError("missing Content-Length") + } + sizeBytes, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return nil, httperrors.NewInputParameterError("Illegal Content-Length %s", sizeStr) + } + if sizeBytes <= 0 { + return nil, httperrors.NewInputParameterError("Content-Length not positive %d", sizeBytes) + } storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER) - err = iBucket.PutObject(ctx, key, appParams.Request.Body, contType, storageClass) + aclStr := cloudprovider.TBucketACLType(appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_ACL_HEADER)) + switch aclStr { + case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite: + // do nothing + default: + return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr) + } + + err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, contType, aclStr, storageClass, false) if err != nil { return nil, httperrors.NewInternalServerError("put object error %s", err) } @@ -739,12 +759,8 @@ func (bucket *SBucket) PerformAcl( switch cloudprovider.TBucketACLType(aclStr) { case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite: // do nothing - case cloudprovider.ACLDefault: - if len(objKey) == 0 { - return nil, httperrors.NewInputParameterError("invalud acl") - } default: - return nil, httperrors.NewInputParameterError("invalud acl") + return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr) } iBucket, err := bucket.GetIBucket() diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 0e95994e9f..8738e2395a 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -23,11 +23,13 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" @@ -211,9 +213,16 @@ func (self *SImage) CustomizedGetDetailsBody(ctx context.Context, userCred mccli appParams := appsrv.AppContextGetParams(ctx) + fstat, err := os.Stat(filePath) + if err != nil { + return nil, errors.Wrap(err, "os.Stat") + } + + appParams.Response.Header().Set("Content-Length", strconv.FormatInt(fstat.Size(), 10)) + fp, err := os.Open(filePath) if err != nil { - return nil, err + return nil, errors.Wrap(err, "os.Open") } defer fp.Close() diff --git a/pkg/mcclient/modules/mod_images.go b/pkg/mcclient/modules/mod_images.go index b4a58789c6..a72df037de 100644 --- a/pkg/mcclient/modules/mod_images.go +++ b/pkg/mcclient/modules/mod_images.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/utils" + "strconv" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/httputils" @@ -538,7 +539,7 @@ func (this *ImageManager) _update(s *mcclient.ClientSession, id string, params j return json.Get("image") } -func (this *ImageManager) Download(s *mcclient.ClientSession, id string, format string, torrent bool) (jsonutils.JSONObject, io.Reader, error) { +func (this *ImageManager) Download(s *mcclient.ClientSession, id string, format string, torrent bool) (jsonutils.JSONObject, io.Reader, int64, error) { query := jsonutils.NewDict() if len(format) > 0 { query.Add(jsonutils.NewString(format), "format") @@ -553,10 +554,15 @@ func (this *ImageManager) Download(s *mcclient.ClientSession, id string, format } resp, err := this.rawRequest(s, "GET", path, nil, nil) if err == nil && resp.StatusCode >= 200 && resp.StatusCode < 300 { - return FetchImageMeta(resp.Header), resp.Body, nil + sizeBytes, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) + if err != nil { + log.Errorf("Download image unknown size") + sizeBytes = -1 + } + return FetchImageMeta(resp.Header), resp.Body, sizeBytes, nil } else { _, _, err = s.ParseJSONResponse(resp, err) - return nil, nil, err + return nil, nil, -1, err } } diff --git a/pkg/multicloud/aliyun/bucket.go b/pkg/multicloud/aliyun/bucket.go index 0a38e48363..a6577c3b79 100644 --- a/pkg/multicloud/aliyun/bucket.go +++ b/pkg/multicloud/aliyun/bucket.go @@ -22,9 +22,9 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" + "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" ) @@ -183,7 +183,7 @@ func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider. return cloudprovider.GetIObjects(b, prefix, isRecursive) } -func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, contType string, storageClassStr string) error { +func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { osscli, err := b.region.GetOssClient() if err != nil { return errors.Wrap(err, "GetOssClient") @@ -193,9 +193,19 @@ func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, co return errors.Wrap(err, "Bucket") } opts := make([]oss.Option, 0) + if sizeBytes > 0 { + opts = append(opts, oss.ContentLength(sizeBytes)) + } if len(contType) > 0 { opts = append(opts, oss.ContentType(contType)) } + if len(cannedAcl) > 0 { + acl, err := str2Acl(string(cannedAcl)) + if err != nil { + return errors.Wrap(err, "") + } + opts = append(opts, oss.ObjectACL(acl)) + } if len(storageClassStr) > 0 { storageClass, err := str2StorageClass(storageClassStr) if err != nil { @@ -206,6 +216,116 @@ func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, co return bucket.PutObject(key, input, opts...) } +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + osscli, err := b.region.GetOssClient() + if err != nil { + return "", errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return "", errors.Wrap(err, "Bucket") + } + opts := make([]oss.Option, 0) + if len(contType) > 0 { + opts = append(opts, oss.ContentType(contType)) + } + if len(cannedAcl) > 0 { + acl, err := str2Acl(string(cannedAcl)) + if err != nil { + return "", errors.Wrap(err, "") + } + opts = append(opts, oss.ObjectACL(acl)) + } + if len(storageClassStr) > 0 { + storageClass, err := str2StorageClass(storageClassStr) + if err != nil { + return "", errors.Wrap(err, "str2StorageClass") + } + opts = append(opts, oss.ObjectStorageClass(storageClass)) + } + result, err := bucket.InitiateMultipartUpload(key, opts...) + if err != nil { + return "", errors.Wrap(err, "bucket.InitiateMultipartUpload") + } + return result.UploadID, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) { + osscli, err := b.region.GetOssClient() + if err != nil { + return "", errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return "", errors.Wrap(err, "Bucket") + } + imur := oss.InitiateMultipartUploadResult{ + Bucket: b.Name, + Key: key, + UploadID: uploadId, + } + part, err := bucket.UploadPart(imur, input, partSize, partIndex) + if err != nil { + return "", errors.Wrap(err, "bucket.UploadPart") + } + if b.region.client.Debug { + log.Debugf("upload part key:%s uploadId:%s partIndex:%d etag:%s", key, uploadId, partIndex, part.ETag) + } + return part.ETag, nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return errors.Wrap(err, "Bucket") + } + imur := oss.InitiateMultipartUploadResult{ + Bucket: b.Name, + Key: key, + UploadID: uploadId, + } + parts := make([]oss.UploadPart, len(partEtags)) + for i := range partEtags { + parts[i] = oss.UploadPart{ + PartNumber: i + 1, + ETag: partEtags[i], + } + } + result, err := bucket.CompleteMultipartUpload(imur, parts) + if err != nil { + return errors.Wrap(err, "bucket.CompleteMultipartUpload") + } + if b.region.client.Debug { + log.Debugf("CompleteMultipartUpload bucket:%s key:%s etag:%s location:%s", result.Bucket, result.Key, result.ETag, result.Location) + } + return nil +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return errors.Wrap(err, "Bucket") + } + imur := oss.InitiateMultipartUploadResult{ + Bucket: b.Name, + Key: key, + UploadID: uploadId, + } + err = bucket.AbortMultipartUpload(imur) + if err != nil { + return errors.Wrap(err, "AbortMultipartUpload") + } + return nil +} + func (b *SBucket) DeleteObject(ctx context.Context, key string) error { osscli, err := b.region.GetOssClient() if err != nil { diff --git a/pkg/multicloud/aliyun/objects.go b/pkg/multicloud/aliyun/objects.go index 330759bd6b..2488dbac1c 100644 --- a/pkg/multicloud/aliyun/objects.go +++ b/pkg/multicloud/aliyun/objects.go @@ -18,6 +18,7 @@ import ( "github.com/pkg/errors" "yunion.io/x/log" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -32,7 +33,7 @@ func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { } func (o *SObject) GetAcl() cloudprovider.TBucketACLType { - acl := cloudprovider.ACLDefault + acl := cloudprovider.ACLPrivate osscli, err := o.bucket.region.GetOssClient() if err != nil { log.Errorf("o.bucket.region.GetOssClient error %s", err) @@ -48,6 +49,9 @@ func (o *SObject) GetAcl() cloudprovider.TBucketACLType { log.Errorf("bucket.GetObjectACL error %s", err) return acl } + if result.ACL == string(oss.ACLDefault) { + return o.bucket.GetAcl() + } acl = cloudprovider.TBucketACLType(result.ACL) return acl } diff --git a/pkg/multicloud/aliyun/storagecache.go b/pkg/multicloud/aliyun/storagecache.go index 11dd017378..cfba5732ab 100644 --- a/pkg/multicloud/aliyun/storagecache.go +++ b/pkg/multicloud/aliyun/storagecache.go @@ -148,25 +148,21 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To // first upload image to oss s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.QCOW2), false) + meta, reader, sizeByte, err := modules.Images.Download(s, image.ImageId, string(qemuimg.QCOW2), false) if err != nil { return "", err } log.Infof("meta data %s", meta) - oss, err := self.region.GetOssClient() - if err != nil { - log.Errorf("GetOssClient err %s", err) - return "", err - } + bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s-%s", self.region.GetId(), image.ImageId)) - exist, err := oss.IsBucketExist(bucketName) + exist, err := self.region.IBucketExist(bucketName) if err != nil { log.Errorf("IsBucketExist err %s", err) return "", err } if !exist { log.Debugf("Bucket %s not exists, to create ...", bucketName) - err = oss.CreateBucket(bucketName) + err = self.region.CreateIBucket(bucketName, "", "") if err != nil { log.Errorf("Create bucket error %s", err) return "", err @@ -175,21 +171,22 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To log.Debugf("Bucket %s exists", bucketName) } - defer oss.DeleteBucket(bucketName) // remove bucket + defer self.region.DeleteIBucket(bucketName) // remove bucket - bucket, err := oss.Bucket(bucketName) + bucket, err := self.region.GetIBucketByName(bucketName) if err != nil { log.Errorf("Bucket error %s %s", bucketName, err) return "", err } log.Debugf("To upload image to bucket %s ...", bucketName) - err = bucket.PutObject(image.ImageId, reader) + err = cloudprovider.UploadObject(context.Background(), bucket, image.ImageId, 0, reader, sizeByte, "", "", "", false) + // err = bucket.PutObject(image.ImageId, reader) if err != nil { log.Errorf("PutObject error %s %s", image.ImageId, err) return "", err } - defer bucket.DeleteObject(image.ImageId) // remove object + defer bucket.DeleteObject(context.Background(), image.ImageId) // remove object imageBaseName := image.ImageId if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { diff --git a/pkg/multicloud/aws/bucket.go b/pkg/multicloud/aws/bucket.go index 4c93bcb26d..a3486bb63f 100644 --- a/pkg/multicloud/aws/bucket.go +++ b/pkg/multicloud/aws/bucket.go @@ -20,10 +20,8 @@ import ( "io" "time" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" "yunion.io/x/log" "yunion.io/x/pkg/errors" @@ -31,6 +29,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/fileutils2" ) type SBucket struct { @@ -210,30 +209,126 @@ func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider. return cloudprovider.GetIObjects(b, prefix, isRecursive) } -func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, contType string, storageClassStr string) error { - sess, err := b.region.getAwsSession() +func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + if sizeBytes <= 0 { + return errors.Error("content length expected") + } + s3cli, err := b.region.GetS3Client() if err != nil { - return errors.Wrap(err, "session.NewSession") + return errors.Wrap(err, "GetS3Client") } - - svc := s3manager.NewUploader(sess) - input := &s3manager.UploadInput{ - Bucket: aws.String(b.Name), - Key: aws.String(key), - Body: reader, + input := &s3.PutObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + seeker, err := fileutils2.NewReadSeeker(body, sizeBytes) + if err != nil { + return errors.Wrap(err, "newFakeSeeker") } + defer seeker.Close() + input.SetBody(seeker) + input.SetContentLength(sizeBytes) if len(contType) > 0 { - input.ContentType = aws.String(contType) + input.SetContentType(contType) + } + if len(cannedAcl) > 0 { + input.SetACL(string(cannedAcl)) } if len(storageClassStr) > 0 { - input.StorageClass = aws.String(storageClassStr) + input.SetStorageClass(storageClassStr) } - - _, err = svc.Upload(input) + _, err = s3cli.PutObjectWithContext(ctx, input) if err != nil { - return errors.Wrap(err, "svc.Upload") + return errors.Wrap(err, "PutObjectWithContext") } + return nil +} +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return "", errors.Wrap(err, "GetS3Client") + } + input := &s3.CreateMultipartUploadInput{} + input.SetBucket(b.Name) + input.SetKey(key) + if len(contType) > 0 { + input.SetContentType(contType) + } + if len(cannedAcl) > 0 { + input.SetACL(string(cannedAcl)) + } + if len(storageClassStr) > 0 { + input.SetStorageClass(storageClassStr) + } + output, err := s3cli.CreateMultipartUploadWithContext(ctx, input) + if err != nil { + return "", errors.Wrap(err, "CreateMultipartUpload") + } + return *output.UploadId, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, part io.Reader, partSize int64) (string, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return "", errors.Wrap(err, "GetS3Client") + } + input := &s3.UploadPartInput{} + input.SetBucket(b.Name) + input.SetKey(key) + input.SetUploadId(uploadId) + input.SetPartNumber(int64(partIndex)) + seeker, err := fileutils2.NewReadSeeker(part, partSize) + if err != nil { + return "", errors.Wrap(err, "newFakeSeeker") + } + defer seeker.Close() + input.SetBody(seeker) + input.SetContentLength(partSize) + output, err := s3cli.UploadPartWithContext(ctx, input) + if err != nil { + return "", errors.Wrap(err, "UploadPartWithContext") + } + return *output.ETag, nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + input := &s3.CompleteMultipartUploadInput{} + input.SetBucket(b.Name) + input.SetKey(key) + input.SetUploadId(uploadId) + uploads := &s3.CompletedMultipartUpload{} + parts := make([]*s3.CompletedPart, len(partEtags)) + for i := range partEtags { + parts[i] = &s3.CompletedPart{} + parts[i].SetPartNumber(int64(i + 1)) + parts[i].SetETag(partEtags[i]) + } + uploads.SetParts(parts) + input.SetMultipartUpload(uploads) + _, err = s3cli.CompleteMultipartUploadWithContext(ctx, input) + if err != nil { + return errors.Wrap(err, "CompleteMultipartUploadWithContext") + } + return nil +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + input := &s3.AbortMultipartUploadInput{} + input.SetBucket(b.Name) + input.SetKey(key) + input.SetUploadId(uploadId) + _, err = s3cli.AbortMultipartUploadWithContext(ctx, input) + if err != nil { + return errors.Wrap(err, "AbortMultipartUploadWithContext") + } return nil } diff --git a/pkg/multicloud/aws/s3object.go b/pkg/multicloud/aws/s3object.go index 7c0c6876d0..29728037dc 100644 --- a/pkg/multicloud/aws/s3object.go +++ b/pkg/multicloud/aws/s3object.go @@ -34,7 +34,7 @@ func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { } func (o *SObject) GetAcl() cloudprovider.TBucketACLType { - acl := cloudprovider.ACLDefault + acl := cloudprovider.ACLPrivate s3cli, err := o.bucket.region.GetS3Client() if err != nil { log.Errorf("o.bucket.region.GetS3Client error %s", err) diff --git a/pkg/multicloud/aws/storagecache.go b/pkg/multicloud/aws/storagecache.go index 256eaaa47d..5b61cdc9fe 100644 --- a/pkg/multicloud/aws/storagecache.go +++ b/pkg/multicloud/aws/storagecache.go @@ -23,10 +23,10 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -153,65 +153,46 @@ func (self *SStoragecache) fetchImages() error { } func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { + err := self.region.initVmimport() + if err != nil { + return "", errors.Wrap(err, "initVmimport") + } + bucketName := GetBucketName(self.region.GetId(), image.ImageId) - err := self.region.initVmimport(bucketName) + + exist, err := self.region.IBucketExist(bucketName) if err != nil { - return "", err + return "", errors.Wrap(err, "IBucketExist") } - // checking remote - s3client, err := self.region.GetS3Client() - if err != nil { - return "", err + if !exist { + err = self.region.CreateIBucket(bucketName, "", "") + if err != nil { + return "", errors.Wrap(err, "CreateIBucket") + } } - defer s3client.DeleteBucket(&s3.DeleteBucketInput{Bucket: &bucketName}) // remove bucket + defer self.region.DeleteIBucket(bucketName) - var diskFormat string s := auth.GetAdminSession(ctx, options.Options.Region, "") - _, err = s3client.GetObject(&s3.GetObjectInput{Bucket: &bucketName, Key: &image.ImageId}) + meta, reader, sizeBytes, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) if err != nil { - // first upload image to oss - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) - if err != nil { - return "", err - } - log.Debugf("Images meta data %s", meta) - - diskFormat, err = meta.GetString("disk_format") - if err != nil { - return "", err - } - - // uploader to aws s3 - input := &s3manager.UploadInput{ - Bucket: &bucketName, - Key: &image.ImageId, - Body: reader, - } - - s3Session, err := self.region.getAwsSession() - if err != nil { - return "", err - } - - uploader := s3manager.NewUploader(s3Session) - _, err = uploader.Upload(input) - if err != nil { - return "", err - } - defer s3client.DeleteObject(&s3.DeleteObjectInput{Bucket: &bucketName, Key: &image.ImageId}) // remove object - } else { - meta, _, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) - if err != nil { - return "", err - } - - diskFormat, err = meta.GetString("disk_format") - if err != nil { - return "", err - } + return "", errors.Wrap(err, "Images.Download") } + log.Debugf("Images meta data %s", meta) + + diskFormat, _ := meta.GetString("disk_format") + + bucket, err := self.region.GetIBucketByName(bucketName) + if err != nil { + return "", errors.Wrap(err, "GetIBucketByName") + } + err = cloudprovider.UploadObject(ctx, bucket, image.ImageId, 0, reader, sizeBytes, "", "", "", false) + if err != nil { + return "", errors.Wrap(err, "cloudprovider.UploadObject") + } + + defer bucket.DeleteObject(ctx, image.ImageId) imageBaseName := image.ImageId if imageBaseName[0] >= '0' && imageBaseName[0] <= '9' { @@ -477,26 +458,7 @@ func (self *SRegion) initVmimportRolePolicy() error { } } -func (self *SRegion) initVmimportBucket(bucketName string) error { - exists, err := self.IsBucketExist(bucketName) - if err != nil { - return err - } - - if exists { - return nil - } - - s3Client, err := self.GetS3Client() - if err != nil { - return err - } - - _, err = s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: &bucketName}) - return err -} - -func (self *SRegion) initVmimport(bucketName string) error { +func (self *SRegion) initVmimport() error { if err := self.initVmimportRole(); err != nil { return err } @@ -505,10 +467,6 @@ func (self *SRegion) initVmimport(bucketName string) error { return err } - if err := self.initVmimportBucket(bucketName); err != nil { - return err - } - return nil } diff --git a/pkg/multicloud/azure/blobobject.go b/pkg/multicloud/azure/blobobject.go index e36807d6ba..f2ec965c13 100644 --- a/pkg/multicloud/azure/blobobject.go +++ b/pkg/multicloud/azure/blobobject.go @@ -29,7 +29,7 @@ func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { } func (o *SObject) GetAcl() cloudprovider.TBucketACLType { - return cloudprovider.ACLDefault + return o.container.getAcl() } func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error { diff --git a/pkg/multicloud/azure/storageaccount.go b/pkg/multicloud/azure/storageaccount.go index 61b4a5fec4..5de0a7dec1 100644 --- a/pkg/multicloud/azure/storageaccount.go +++ b/pkg/multicloud/azure/storageaccount.go @@ -32,6 +32,8 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + "encoding/base64" + "strconv" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" ) @@ -689,6 +691,14 @@ func (self *SStorageAccount) UploadStream(containerName string, key string, read return container.UploadStream(key, reader, contType) } +func (b *SStorageAccount) MaxPartSizeBytes() int64 { + return 100 * 1000 * 1000 +} + +func (b *SStorageAccount) MaxPartCount() int { + return 50000 +} + func (b *SStorageAccount) GetProjectId() string { return "" } @@ -927,7 +937,7 @@ func splitKey(key string) (string, string, error) { return containerName, key, nil } -func (b *SStorageAccount) PutObject(ctx context.Context, key string, reader io.Reader, contType string, storageClassStr string) error { +func (b *SStorageAccount) PutObject(ctx context.Context, key string, reader io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { containerName, blob, err := splitKey(key) if err != nil { return errors.Wrap(err, "splitKey") @@ -939,6 +949,132 @@ func (b *SStorageAccount) PutObject(ctx context.Context, key string, reader io.R return nil } +func (b *SStorageAccount) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + containerName, blob, err := splitKey(key) + if err != nil { + return "", errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, true) + if err != nil { + return "", errors.Wrap(err, "getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return "", errors.Wrap(err, "getContainerRef") + } + blobRef := containerRef.GetBlobReference(blob) + + err = blobRef.CreateBlockBlob(&storage.PutBlobOptions{}) + if err != nil { + return "", errors.Wrap(err, "CreateBlockBlob") + } + + uploadId, err := blobRef.AcquireLease(-1, "", nil) + if err != nil { + return "", errors.Wrap(err, "blobRef.AcquireLease") + } + + return uploadId, nil +} + +func (b *SStorageAccount) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) { + containerName, blob, err := splitKey(key) + if err != nil { + return "", errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, true) + if err != nil { + return "", errors.Wrap(err, "getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return "", errors.Wrap(err, "getContainerRef") + } + blobRef := containerRef.GetBlobReference(blob) + + opts := &storage.PutBlockOptions{} + opts.LeaseID = uploadId + + blockId := base64.URLEncoding.EncodeToString([]byte(strconv.FormatInt(int64(partIndex), 10))) + err = blobRef.PutBlockWithLength(blockId, uint64(partSize), input, opts) + if err != nil { + return "", errors.Wrap(err, "PutBlockWithLength") + } + + return blockId, nil +} + +func (b *SStorageAccount) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, blockIds []string) error { + containerName, blob, err := splitKey(key) + if err != nil { + return errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, true) + if err != nil { + return errors.Wrap(err, "getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return errors.Wrap(err, "getContainerRef") + } + blobRef := containerRef.GetBlobReference(blob) + + blocks := make([]storage.Block, len(blockIds)) + for i := range blockIds { + blocks[i] = storage.Block{ + ID: blockIds[i], + Status: storage.BlockStatusLatest, + } + } + opts := &storage.PutBlockListOptions{} + opts.LeaseID = uploadId + err = blobRef.PutBlockList(blocks, opts) + if err != nil { + return errors.Wrap(err, "PutBlockList") + } + + err = blobRef.ReleaseLease(uploadId, nil) + if err != nil { + return errors.Wrap(err, "ReleaseLease") + } + + return nil +} + +func (b *SStorageAccount) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + containerName, blob, err := splitKey(key) + if err != nil { + return errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, false) + if err != nil { + if err == cloudprovider.ErrNotFound { + return nil + } + return errors.Wrap(err, "getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return errors.Wrap(err, "getContainerRef") + } + + blobRef := containerRef.GetBlobReference(blob) + err = blobRef.ReleaseLease(uploadId, nil) + if err != nil { + return errors.Wrap(err, "ReleaseLease") + } + + opts := &storage.DeleteBlobOptions{} + deleteSnapshots := true + opts.DeleteSnapshots = &deleteSnapshots + _, err = blobRef.DeleteIfExists(opts) + if err != nil { + return errors.Wrap(err, "DeleteIfExists") + } + + return nil +} + func (b *SStorageAccount) DeleteObject(ctx context.Context, key string) error { containerName, blob, err := splitKey(key) if err != nil { diff --git a/pkg/multicloud/azure/storagecache.go b/pkg/multicloud/azure/storagecache.go index b4d5fe5e24..c000a14e3b 100644 --- a/pkg/multicloud/azure/storagecache.go +++ b/pkg/multicloud/azure/storagecache.go @@ -160,7 +160,7 @@ func (self *SStoragecache) checkStorageAccount() (*SStorageAccount, error) { func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool, tmpPath string) (string, error) { s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VHD), false) + meta, reader, sizeBytes, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VHD), false) if err != nil { return "", err } @@ -214,9 +214,9 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To return "", err } - size, _ := meta.Int("size") + // size, _ := meta.Int("size") - img, err := self.region.CreateImageByBlob(image.ImageId, image.OsType, blobURI, int32(size>>30)) + img, err := self.region.CreateImageByBlob(image.ImageId, image.OsType, blobURI, int32(sizeBytes>>30)) if err != nil { return "", err } diff --git a/pkg/multicloud/bucket_base.go b/pkg/multicloud/bucket_base.go index 7c9b876c58..cf0185b375 100644 --- a/pkg/multicloud/bucket_base.go +++ b/pkg/multicloud/bucket_base.go @@ -4,6 +4,14 @@ import "yunion.io/x/jsonutils" type SBaseBucket struct{} +func (b *SBaseBucket) MaxPartCount() int { + return 10000 +} + +func (b *SBaseBucket) MaxPartSizeBytes() int64 { + return 5 * 1000 * 1000 * 1000 +} + func (b *SBaseBucket) GetId() string { return "" } diff --git a/pkg/multicloud/huawei/bucket.go b/pkg/multicloud/huawei/bucket.go index 0bc81d4809..7f47df8942 100644 --- a/pkg/multicloud/huawei/bucket.go +++ b/pkg/multicloud/huawei/bucket.go @@ -227,7 +227,7 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma return result, nil } -func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, contType string, storageClassStr string) error { +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { obscli, err := b.region.getOBSClient() if err != nil { return errors.Wrap(err, "GetOBSClient") @@ -236,12 +236,19 @@ func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, c input.Bucket = b.Name input.Key = key input.Body = reader + + if sizeBytes > 0 { + input.ContentLength = sizeBytes + } if len(storageClassStr) > 0 { input.StorageClass, err = str2StorageClass(storageClassStr) if err != nil { return err } } + if len(cannedAcl) > 0 { + input.ACL = obs.AclType(string(cannedAcl)) + } if len(contType) > 0 { input.ContentType = contType } @@ -252,6 +259,100 @@ func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, c return nil } +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + + input := &obs.InitiateMultipartUploadInput{} + input.Bucket = b.Name + input.Key = key + if len(contType) > 0 { + input.ContentType = contType + } + if len(cannedAcl) > 0 { + input.ACL = obs.AclType(string(cannedAcl)) + } + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return "", errors.Wrap(err, "str2StorageClass") + } + } + output, err := obscli.InitiateMultipartUpload(input) + if err != nil { + return "", errors.Wrap(err, "InitiateMultipartUpload") + } + + return output.UploadId, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, part io.Reader, partSize int64) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + + input := &obs.UploadPartInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + input.PartNumber = partIndex + input.PartSize = partSize + input.Body = part + output, err := obscli.UploadPart(input) + if err != nil { + return "", errors.Wrap(err, "UploadPart") + } + + return output.ETag, nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.CompleteMultipartUploadInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + parts := make([]obs.Part, len(partEtags)) + for i := range partEtags { + parts[i] = obs.Part{ + PartNumber: i + 1, + ETag: partEtags[i], + } + } + input.Parts = parts + _, err = obscli.CompleteMultipartUpload(input) + if err != nil { + return errors.Wrap(err, "CompleteMultipartUpload") + } + + return nil +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + input := &obs.AbortMultipartUploadInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + + _, err = obscli.AbortMultipartUpload(input) + if err != nil { + return errors.Wrap(err, "AbortMultipartUpload") + } + + return nil +} + func (b *SBucket) DeleteObject(ctx context.Context, key string) error { obscli, err := b.region.getOBSClient() if err != nil { diff --git a/pkg/multicloud/huawei/object.go b/pkg/multicloud/huawei/object.go index bb5f705090..04c63aa901 100644 --- a/pkg/multicloud/huawei/object.go +++ b/pkg/multicloud/huawei/object.go @@ -33,7 +33,7 @@ func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { } func (o *SObject) GetAcl() cloudprovider.TBucketACLType { - acl := cloudprovider.ACLDefault + acl := cloudprovider.ACLPrivate obscli, err := o.bucket.region.getOBSClient() if err != nil { log.Errorf("o.bucket.region.GetOssClient error %s", err) diff --git a/pkg/multicloud/huawei/storagecache.go b/pkg/multicloud/huawei/storagecache.go index eda57631aa..3867874dac 100644 --- a/pkg/multicloud/huawei/storagecache.go +++ b/pkg/multicloud/huawei/storagecache.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -30,7 +31,6 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/onecloud/pkg/multicloud/huawei/obs" "yunion.io/x/onecloud/pkg/util/qemuimg" ) @@ -162,34 +162,28 @@ func (self *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.To func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { bucketName := GetBucketName(self.region.GetId(), image.ImageId) - obsClient, err := self.region.getOBSClient() - if err != nil { - return "", err - } - // create bucket - input := &obs.CreateBucketInput{} - input.Bucket = bucketName - input.Location = self.region.GetId() - _, err = obsClient.CreateBucket(input) + exist, err := self.region.IBucketExist(bucketName) if err != nil { - return "", err + return "", errors.Wrap(err, "self.region.IBucketExist") } - defer obsClient.DeleteBucket(bucketName) + if !exist { + err = self.region.CreateIBucket(bucketName, "", "") + if err != nil { + return "", errors.Wrap(err, "CreateIBucket") + } + } + defer self.region.DeleteIBucket(bucketName) // upload to huawei cloud s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) + meta, reader, sizeByte, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) if err != nil { return "", err } log.Debugf("Images meta data %s", meta) - _image, err := modules.Images.Get(s, image.ImageId, nil) - if err != nil { - return "", err - } - minDiskMB, _ := _image.Int("min_disk") + minDiskMB, _ := meta.Int("min_disk") minDiskGB := int64(math.Ceil(float64(minDiskMB) / 1024)) // 在使用OBS桶的外部镜像文件制作镜像时生效且为必选字段。取值为40～1024GB。 if minDiskGB < 40 { @@ -198,21 +192,17 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To minDiskGB = 1024 } - // upload to huawei cloud - obj := &obs.PutObjectInput{} - obj.Bucket = bucketName - obj.Key = image.ImageId - obj.Body = reader - - _, err = obsClient.PutObject(obj) + bucket, err := self.region.GetIBucketByName(bucketName) if err != nil { - return "", err + return "", errors.Wrap(err, "GetIBucketByName") } - objDelete := &obs.DeleteObjectInput{} - objDelete.Bucket = bucketName - objDelete.Key = image.ImageId - defer obsClient.DeleteObject(objDelete) // remove object + err = cloudprovider.UploadObject(context.Background(), bucket, image.ImageId, 0, reader, sizeByte, "", "", "", false) + if err != nil { + return "", errors.Wrap(err, "cloudprovider.UploadObject") + } + + defer bucket.DeleteObject(context.Background(), image.ImageId) // check image name, avoid name conflict imageBaseName := image.ImageId diff --git a/pkg/multicloud/objectstore/buckets.go b/pkg/multicloud/objectstore/buckets.go index 7b664962d2..f6f2506683 100644 --- a/pkg/multicloud/objectstore/buckets.go +++ b/pkg/multicloud/objectstore/buckets.go @@ -26,9 +26,12 @@ import ( "yunion.io/x/s3cli" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" ) type SBucket struct { + multicloud.SBaseBucket + client *SObjectStoreClient Name string @@ -150,16 +153,75 @@ func (bucket *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprov return ret, nil } -func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.Reader, contType string, storageClass string) error { +func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { opts := s3cli.PutObjectOptions{} if len(contType) > 0 { opts.ContentType = contType } - if len(storageClass) > 0 { - opts.StorageClass = storageClass + if len(storageClassStr) > 0 { + opts.StorageClass = storageClassStr } - _, err := bucket.client.client.PutObjectWithContext(ctx, bucket.Name, key, input, -1, opts) - return err + opts.PartSize = uint64(cloudprovider.MAX_PUT_OBJECT_SIZEBYTES) + _, err := bucket.client.client.PutObjectDo(ctx, bucket.Name, key, input, "", "", sizeBytes, opts) + if err != nil { + return errors.Wrap(err, "PutObjectWithContext") + } + obj, err := cloudprovider.GetIObject(bucket, key) + if err != nil { + return errors.Wrap(err, "GetIObject") + } + err = obj.SetAcl(cannedAcl) + if err != nil { + return errors.Wrap(err, "obj.SetAcl") + } + return nil +} + +func (bucket *SBucket) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + opts := s3cli.PutObjectOptions{} + if len(contType) > 0 { + opts.ContentType = contType + } + if len(storageClassStr) > 0 { + opts.StorageClass = storageClassStr + } + result, err := bucket.client.client.InitiateMultipartUpload(ctx, bucket.Name, key, opts) + if err != nil { + return "", errors.Wrap(err, "InitiateMultipartUpload") + } + return result.UploadID, nil +} + +func (bucket *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) { + part, err := bucket.client.client.UploadPart(ctx, bucket.Name, key, uploadId, input, partIndex, "", "", partSize, nil) + if err != nil { + return "", errors.Wrap(err, "UploadPart") + } + return part.ETag, nil +} + +func (bucket *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + complete := s3cli.CompleteMultipartUpload{} + complete.Parts = make([]s3cli.CompletePart, len(partEtags)) + for i := 0; i < len(partEtags); i += 1 { + complete.Parts[i] = s3cli.CompletePart{ + PartNumber: i + 1, + ETag: partEtags[i], + } + } + _, err := bucket.client.client.CompleteMultipartUpload(ctx, bucket.Name, key, uploadId, complete) + if err != nil { + return errors.Wrap(err, "CompleteMultipartUpload") + } + return nil +} + +func (bucket *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + err := bucket.client.client.AbortMultipartUpload(ctx, bucket.Name, key, uploadId) + if err != nil { + return errors.Wrap(err, "AbortMultipartUpload") + } + return nil } func (bucket *SBucket) DeleteObject(ctx context.Context, key string) error { diff --git a/pkg/multicloud/objectstore/shell.go b/pkg/multicloud/objectstore/shell.go index 01b29d62bb..560feb0d0e 100644 --- a/pkg/multicloud/objectstore/shell.go +++ b/pkg/multicloud/objectstore/shell.go @@ -21,6 +21,7 @@ import ( "os" "time" + "github.com/pkg/errors" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/shellutils" @@ -156,6 +157,9 @@ func S3Shell() { KEY string `help:"key of object"` Path string `help:"Path of file to upload"` + BlockSize int64 `help:"blocksz in MB" default:"100"` + + Acl string `help:"acl" choices:"private|public-read|public-read-write"` ContentType string `help:"content-type"` StorageClass string `help:"storage class"` } @@ -165,10 +169,16 @@ func S3Shell() { return err } var input io.ReadSeeker + var fSize int64 if len(args.Path) > 0 { + finfo, err := os.Stat(args.Path) + if err != nil { + return errors.Wrap(err, "os.Stat") + } + fSize = finfo.Size() file, err := os.Open(args.Path) if err != nil { - return err + return errors.Wrap(err, "os.Open") } defer file.Close() @@ -176,7 +186,7 @@ func S3Shell() { } else { input = os.Stdout } - err = bucket.PutObject(context.Background(), args.KEY, input, args.ContentType, args.StorageClass) + err = cloudprovider.UploadObject(context.Background(), bucket, args.KEY, args.BlockSize*1000*1000, input, fSize, args.ContentType, cloudprovider.TBucketACLType(args.Acl), args.StorageClass, true) if err != nil { return err } diff --git a/pkg/multicloud/openstack/storagecache.go b/pkg/multicloud/openstack/storagecache.go index 2527ecc13c..afdbe1010f 100644 --- a/pkg/multicloud/openstack/storagecache.go +++ b/pkg/multicloud/openstack/storagecache.go @@ -119,7 +119,7 @@ func (cache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient.T func (cache *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) + meta, reader, _, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) if err != nil { return "", err } diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index 58bcaaa952..8369562934 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -216,15 +216,24 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma return result, nil } -func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, contType string, storageClassStr string) error { +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { coscli, err := b.region.GetCosClient(b) if err != nil { return errors.Wrap(err, "GetCosClient") } - opts := &cos.ObjectPutOptions{} + opts := &cos.ObjectPutOptions{ + ACLHeaderOptions: &cos.ACLHeaderOptions{}, + ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{}, + } + if sizeBytes > 0 { + opts.ContentLength = int(sizeBytes) + } if len(contType) > 0 { opts.ContentType = contType } + if len(cannedAcl) > 0 { + opts.XCosACL = string(cannedAcl) + } if len(storageClassStr) > 0 { opts.XCosStorageClass = storageClassStr } @@ -235,6 +244,84 @@ func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, c return nil } +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return "", errors.Wrap(err, "GetCosClient") + } + opts := &cos.InitiateMultipartUploadOptions{ + ACLHeaderOptions: &cos.ACLHeaderOptions{}, + ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{}, + } + if len(contType) > 0 { + opts.ContentType = contType + } + if len(cannedAcl) > 0 { + opts.XCosACL = string(cannedAcl) + } + if len(storageClassStr) > 0 { + opts.XCosStorageClass = storageClassStr + } + result, _, err := coscli.Object.InitiateMultipartUpload(ctx, key, opts) + if err != nil { + return "", errors.Wrap(err, "InitiateMultipartUpload") + } + + return result.UploadID, nil +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return "", errors.Wrap(err, "GetCosClient") + } + opts := &cos.ObjectUploadPartOptions{} + opts.ContentLength = int(partSize) + resp, err := coscli.Object.UploadPart(ctx, key, uploadId, partIndex, input, opts) + if err != nil { + return "", errors.Wrap(err, "UploadPart") + } + + return resp.Header.Get("Etag"), nil +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + opts := &cos.CompleteMultipartUploadOptions{} + parts := make([]cos.Object, len(partEtags)) + for i := range partEtags { + parts[i] = cos.Object{ + PartNumber: i + 1, + ETag: partEtags[i], + } + } + opts.Parts = parts + _, _, err = coscli.Object.CompleteMultipartUpload(ctx, key, uploadId, opts) + + if err != nil { + return errors.Wrap(err, "CompleteMultipartUpload") + } + + return nil +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + + _, err = coscli.Object.AbortMultipartUpload(ctx, key, uploadId) + if err != nil { + return errors.Wrap(err, "AbortMultipartUpload") + } + + return nil +} + func (b *SBucket) DeleteObject(ctx context.Context, key string) error { coscli, err := b.region.GetCosClient(b) if err != nil { diff --git a/pkg/multicloud/qcloud/object.go b/pkg/multicloud/qcloud/object.go index d9fa896d39..677694726e 100644 --- a/pkg/multicloud/qcloud/object.go +++ b/pkg/multicloud/qcloud/object.go @@ -15,11 +15,13 @@ package qcloud import ( + "context" + + "github.com/tencentyun/cos-go-sdk-v5" + "yunion.io/x/log" "yunion.io/x/pkg/errors" - "context" - "github.com/tencentyun/cos-go-sdk-v5" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -34,7 +36,7 @@ func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { } func (o *SObject) GetAcl() cloudprovider.TBucketACLType { - acl := cloudprovider.ACLDefault + acl := cloudprovider.ACLPrivate coscli, err := o.bucket.region.GetCosClient(o.bucket) if err != nil { log.Errorf("o.bucket.region.GetOssClient error %s", err) diff --git a/pkg/multicloud/qcloud/storagecache.go b/pkg/multicloud/qcloud/storagecache.go index 57de5605d3..bc0852329e 100644 --- a/pkg/multicloud/qcloud/storagecache.go +++ b/pkg/multicloud/qcloud/storagecache.go @@ -17,7 +17,6 @@ package qcloud import ( "context" "fmt" - "io" "strings" "time" @@ -163,7 +162,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To // first upload image to oss s := auth.GetAdminSession(ctx, options.Options.Region, "") - _, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) + _, reader, sizeBytes, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) if err != nil { return "", err } @@ -186,7 +185,8 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To if err != nil { return "", errors.Wrap(err, "GetIBucketByName") } - err = bucket.PutObject(context.Background(), image.ImageId, reader.(io.ReadSeeker), "", "") + err = cloudprovider.UploadObject(context.Background(), bucket, image.ImageId, 0, reader, sizeBytes, "", "", "", false) + // err = bucket.PutObject(context.Background(), image.ImageId, reader, sizeBytes, "", "", "") if err != nil { log.Errorf("UploadObject error %s %s", image.ImageId, err) return "", errors.Wrap(err, "bucket.PutObject") diff --git a/pkg/multicloud/ucloud/storagecache.go b/pkg/multicloud/ucloud/storagecache.go index 36ff41ed09..99b6ae70d8 100644 --- a/pkg/multicloud/ucloud/storagecache.go +++ b/pkg/multicloud/ucloud/storagecache.go @@ -162,7 +162,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To // upload to ucloud s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) + meta, reader, size, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) if err != nil { return "", err } @@ -175,7 +175,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To } else if minDiskGB > 1024 { minDiskGB = 1024 } - size, _ := meta.Int("size") + // size, _ := meta.Int("size") md5, _ := meta.GetString("checksum") diskFormat, _ := meta.GetString("disk_format") // upload to ucloud diff --git a/pkg/multicloud/ucloud/ufile.go b/pkg/multicloud/ucloud/ufile.go index 4288702f05..9a1249805b 100644 --- a/pkg/multicloud/ucloud/ufile.go +++ b/pkg/multicloud/ucloud/ufile.go @@ -175,7 +175,7 @@ func (self *SFile) GetContentType() string { } func (self *SFile) GetAcl() cloudprovider.TBucketACLType { - return cloudprovider.ACLDefault + return self.bucket.GetAcl() } func (self *SFile) SetAcl(cloudprovider.TBucketACLType) error { @@ -343,7 +343,23 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma return result, nil } -func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, contType string, storageClassStr string) error { +func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + return cloudprovider.ErrNotSupported +} + +func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) (string, error) { + return "", cloudprovider.ErrNotSupported +} + +func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) { + return "", cloudprovider.ErrNotSupported +} + +func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { + return cloudprovider.ErrNotSupported +} + +func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { return cloudprovider.ErrNotSupported } diff --git a/pkg/multicloud/zstack/storagecache.go b/pkg/multicloud/zstack/storagecache.go index bf0c656821..8c50ff361b 100644 --- a/pkg/multicloud/zstack/storagecache.go +++ b/pkg/multicloud/zstack/storagecache.go @@ -118,13 +118,13 @@ func (scache *SStoragecache) UploadImage(ctx context.Context, userCred mcclient. func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.TokenCredential, image *cloudprovider.SImageCreateOption, isForce bool) (string, error) { s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.QCOW2), false) + meta, reader, size, err := modules.Images.Download(s, image.ImageId, string(qemuimg.QCOW2), false) if err != nil { return "", err } log.Infof("meta data %s", meta) - size, _ := meta.Int("size") + // size, _ := meta.Int("size") img, err := self.region.CreateImage(self.ZoneId, image.ImageName, string(qemuimg.QCOW2), image.OsType, "", reader, size) if err != nil { return "", err diff --git a/pkg/util/fileutils2/seeker.go b/pkg/util/fileutils2/seeker.go new file mode 100644 index 0000000000..5c3920884b --- /dev/null +++ b/pkg/util/fileutils2/seeker.go @@ -0,0 +1,81 @@ +package fileutils2 + +import ( + "io" + "io/ioutil" + "os" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" +) + +type SReadSeeker struct { + reader io.Reader + + offset int64 + + readerOffset int64 + readerSize int64 + + tmpFile *os.File +} + +func NewReadSeeker(reader io.Reader, size int64) (*SReadSeeker, error) { + tmpfile, err := ioutil.TempFile("", "fakeseeker") + if err != nil { + return nil, errors.Wrap(err, "TempFile") + } + log.Debugf("use tmpfile %s", tmpfile.Name()) + return &SReadSeeker{ + reader: reader, + readerOffset: 0, + readerSize: size, + + tmpFile: tmpfile, + }, nil +} + +func (s *SReadSeeker) Read(p []byte) (int, error) { + if s.offset == s.readerOffset && s.offset < s.readerSize { + n, err := s.reader.Read(p) + if n > 0 { + wn, werr := s.tmpFile.Write(p[:n]) + if werr != nil { + return n, werr + } + if wn < n { + return n, errors.Error("sFakeSeeker write less bytes") + } + s.offset += int64(n) + s.readerOffset += int64(n) + } + return n, err + } else { + n, err := s.tmpFile.ReadAt(p, s.offset) + if n > 0 { + s.offset += int64(n) + } + return n, err + } +} + +func (s *SReadSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + // offset = offset + case io.SeekCurrent: + offset = s.offset + offset + case io.SeekEnd: + offset = s.readerSize + offset + } + if offset < 0 || offset > s.readerSize { + log.Debugf("offset out of range: %d", offset) + return -1, io.ErrUnexpectedEOF + } + s.offset = offset + return offset, nil +} + +func (s *SReadSeeker) Close() error { + return os.Remove(s.tmpFile.Name()) +} diff --git a/pkg/util/fileutils2/seeker_test.go b/pkg/util/fileutils2/seeker_test.go new file mode 100644 index 0000000000..88f50694c8 --- /dev/null +++ b/pkg/util/fileutils2/seeker_test.go @@ -0,0 +1,31 @@ +package fileutils2 + +import ( + "io" + "strings" + "testing" +) + +func TestNewReadSeeker(t *testing.T) { + testStr := "This is a test reader string" + seeker, err := NewReadSeeker(strings.NewReader(testStr), int64(len(testStr))) + if err != nil { + t.Fatalf("NewReadSeeker error %s", err) + } + defer seeker.Close() + buf1 := make([]byte, 1024) + n, err := seeker.Read(buf1) + if n != len(testStr) { + t.Fatalf("read buf1 error %s", err) + } + buf2 := make([]byte, 1024) + n, err = seeker.Read(buf2) + if n != 0 { + t.Fatalf("read buf2 should fail") + } + seeker.Seek(0, io.SeekStart) + n, err = seeker.Read(buf2) + if n != len(testStr) { + t.Fatalf("read buf2 error %s", err) + } +} diff --git a/pkg/util/httputils/httputils.go b/pkg/util/httputils/httputils.go index 9641a29f35..f7438c46d2 100644 --- a/pkg/util/httputils/httputils.go +++ b/pkg/util/httputils/httputils.go @@ -130,12 +130,14 @@ func GetAddrPort(urlStr string) (string, int, error) { func GetTransport(insecure bool, timeout time.Duration) *http.Transport { return &http.Transport{ DialContext: (&net.Dialer{ - Timeout: 5 * time.Second, + Timeout: timeout, + KeepAlive: timeout, }).DialContext, - IdleConnTimeout: 5 * time.Second, + IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, + DisableCompression: true, } } From 17979f289d790b4a94eb907feecb478c1b2da5a9 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 12 Aug 2019 11:02:53 +0800 Subject: [PATCH 2/3] update vendor --- go.mod | 2 +- go.sum | 4 +- .../service/s3/s3iface/interface.go | 443 ----------- .../aws-sdk-go/service/s3/s3manager/batch.go | 529 ------------- .../service/s3/s3manager/bucket_region.go | 88 --- .../aws-sdk-go/service/s3/s3manager/doc.go | 3 - .../service/s3/s3manager/download.go | 557 -------------- .../aws-sdk-go/service/s3/s3manager/upload.go | 727 ------------------ .../service/s3/s3manager/upload_input.go | 129 ---- vendor/modules.txt | 4 +- vendor/yunion.io/x/s3cli/README.md | 2 +- .../yunion.io/x/s3cli/api-compose-object.go | 4 +- .../x/s3cli/api-put-object-common.go | 2 +- .../x/s3cli/api-put-object-multipart.go | 46 +- .../x/s3cli/api-put-object-streaming.go | 20 +- vendor/yunion.io/x/s3cli/api-put-object.go | 8 +- vendor/yunion.io/x/s3cli/api-remove.go | 4 +- vendor/yunion.io/x/s3cli/api-s3-datatypes.go | 26 +- vendor/yunion.io/x/s3cli/core.go | 153 ---- 19 files changed, 62 insertions(+), 2689 deletions(-) delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go delete mode 100644 vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go delete mode 100644 vendor/yunion.io/x/s3cli/core.go diff --git a/go.mod b/go.mod index 51e446f8c0..c7157fe4b3 100644 --- a/go.mod +++ b/go.mod @@ -158,7 +158,7 @@ require ( yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224 - yunion.io/x/s3cli v0.0.0-20190806141932-17a2c973cbc5 + yunion.io/x/s3cli v0.0.0-20190812030153-814f1e636a0a yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 ) diff --git a/go.sum b/go.sum index bfd4178d24..6d13b1a118 100644 --- a/go.sum +++ b/go.sum @@ -515,8 +515,8 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRi yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224 h1:dAeUov/CtKcPaOapGVG7+NRqmUF2fr2O3Onb+ID5HS4= yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/s3cli v0.0.0-20190806141932-17a2c973cbc5 h1:GzM18jK4RuFk+hyPus47u8PiyDr/RahhfGF8sDFXABs= -yunion.io/x/s3cli v0.0.0-20190806141932-17a2c973cbc5/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= +yunion.io/x/s3cli v0.0.0-20190812030153-814f1e636a0a h1:7E013HqvvP2Q7PExgdfTPGEhYcfpz/0L2WotfCG70dg= +yunion.io/x/s3cli v0.0.0-20190812030153-814f1e636a0a/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda h1:i+/3Hh+kVmPZM1P+j3wfViEzb0XlttWyNvDSYA5DDDU= yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE= diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go deleted file mode 100644 index 2646a42722..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3iface/interface.go +++ /dev/null @@ -1,443 +0,0 @@ -// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. - -// Package s3iface provides an interface to enable mocking the Amazon Simple Storage Service service client -// for testing your code. -// -// It is important to note that this interface will have breaking changes -// when the service model is updated and adds new API operations, paginators, -// and waiters. -package s3iface - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" -) - -// S3API provides an interface to enable mocking the -// s3.S3 service client's API operation, -// paginators, and waiters. This make unit testing your code that calls out -// to the SDK's service client's calls easier. -// -// The best way to use this interface is so the SDK's service client's calls -// can be stubbed out for unit testing your code with the SDK without needing -// to inject custom request handlers into the SDK's request pipeline. -// -// // myFunc uses an SDK service client to make a request to -// // Amazon Simple Storage Service. -// func myFunc(svc s3iface.S3API) bool { -// // Make svc.AbortMultipartUpload request -// } -// -// func main() { -// sess := session.New() -// svc := s3.New(sess) -// -// myFunc(svc) -// } -// -// In your _test.go file: -// -// // Define a mock struct to be used in your unit tests of myFunc. -// type mockS3Client struct { -// s3iface.S3API -// } -// func (m *mockS3Client) AbortMultipartUpload(input *s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) { -// // mock response/functionality -// } -// -// func TestMyFunc(t *testing.T) { -// // Setup Test -// mockSvc := &mockS3Client{} -// -// myfunc(mockSvc) -// -// // Verify myFunc's functionality -// } -// -// It is important to note that this interface will have breaking changes -// when the service model is updated and adds new API operations, paginators, -// and waiters. Its suggested to use the pattern above for testing, or using -// tooling to generate mocks to satisfy the interfaces. -type S3API interface { - AbortMultipartUpload(*s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) - AbortMultipartUploadWithContext(aws.Context, *s3.AbortMultipartUploadInput, ...request.Option) (*s3.AbortMultipartUploadOutput, error) - AbortMultipartUploadRequest(*s3.AbortMultipartUploadInput) (*request.Request, *s3.AbortMultipartUploadOutput) - - CompleteMultipartUpload(*s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) - CompleteMultipartUploadWithContext(aws.Context, *s3.CompleteMultipartUploadInput, ...request.Option) (*s3.CompleteMultipartUploadOutput, error) - CompleteMultipartUploadRequest(*s3.CompleteMultipartUploadInput) (*request.Request, *s3.CompleteMultipartUploadOutput) - - CopyObject(*s3.CopyObjectInput) (*s3.CopyObjectOutput, error) - CopyObjectWithContext(aws.Context, *s3.CopyObjectInput, ...request.Option) (*s3.CopyObjectOutput, error) - CopyObjectRequest(*s3.CopyObjectInput) (*request.Request, *s3.CopyObjectOutput) - - CreateBucket(*s3.CreateBucketInput) (*s3.CreateBucketOutput, error) - CreateBucketWithContext(aws.Context, *s3.CreateBucketInput, ...request.Option) (*s3.CreateBucketOutput, error) - CreateBucketRequest(*s3.CreateBucketInput) (*request.Request, *s3.CreateBucketOutput) - - CreateMultipartUpload(*s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) - CreateMultipartUploadWithContext(aws.Context, *s3.CreateMultipartUploadInput, ...request.Option) (*s3.CreateMultipartUploadOutput, error) - CreateMultipartUploadRequest(*s3.CreateMultipartUploadInput) (*request.Request, *s3.CreateMultipartUploadOutput) - - DeleteBucket(*s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error) - DeleteBucketWithContext(aws.Context, *s3.DeleteBucketInput, ...request.Option) (*s3.DeleteBucketOutput, error) - DeleteBucketRequest(*s3.DeleteBucketInput) (*request.Request, *s3.DeleteBucketOutput) - - DeleteBucketAnalyticsConfiguration(*s3.DeleteBucketAnalyticsConfigurationInput) (*s3.DeleteBucketAnalyticsConfigurationOutput, error) - DeleteBucketAnalyticsConfigurationWithContext(aws.Context, *s3.DeleteBucketAnalyticsConfigurationInput, ...request.Option) (*s3.DeleteBucketAnalyticsConfigurationOutput, error) - DeleteBucketAnalyticsConfigurationRequest(*s3.DeleteBucketAnalyticsConfigurationInput) (*request.Request, *s3.DeleteBucketAnalyticsConfigurationOutput) - - DeleteBucketCors(*s3.DeleteBucketCorsInput) (*s3.DeleteBucketCorsOutput, error) - DeleteBucketCorsWithContext(aws.Context, *s3.DeleteBucketCorsInput, ...request.Option) (*s3.DeleteBucketCorsOutput, error) - DeleteBucketCorsRequest(*s3.DeleteBucketCorsInput) (*request.Request, *s3.DeleteBucketCorsOutput) - - DeleteBucketEncryption(*s3.DeleteBucketEncryptionInput) (*s3.DeleteBucketEncryptionOutput, error) - DeleteBucketEncryptionWithContext(aws.Context, *s3.DeleteBucketEncryptionInput, ...request.Option) (*s3.DeleteBucketEncryptionOutput, error) - DeleteBucketEncryptionRequest(*s3.DeleteBucketEncryptionInput) (*request.Request, *s3.DeleteBucketEncryptionOutput) - - DeleteBucketInventoryConfiguration(*s3.DeleteBucketInventoryConfigurationInput) (*s3.DeleteBucketInventoryConfigurationOutput, error) - DeleteBucketInventoryConfigurationWithContext(aws.Context, *s3.DeleteBucketInventoryConfigurationInput, ...request.Option) (*s3.DeleteBucketInventoryConfigurationOutput, error) - DeleteBucketInventoryConfigurationRequest(*s3.DeleteBucketInventoryConfigurationInput) (*request.Request, *s3.DeleteBucketInventoryConfigurationOutput) - - DeleteBucketLifecycle(*s3.DeleteBucketLifecycleInput) (*s3.DeleteBucketLifecycleOutput, error) - DeleteBucketLifecycleWithContext(aws.Context, *s3.DeleteBucketLifecycleInput, ...request.Option) (*s3.DeleteBucketLifecycleOutput, error) - DeleteBucketLifecycleRequest(*s3.DeleteBucketLifecycleInput) (*request.Request, *s3.DeleteBucketLifecycleOutput) - - DeleteBucketMetricsConfiguration(*s3.DeleteBucketMetricsConfigurationInput) (*s3.DeleteBucketMetricsConfigurationOutput, error) - DeleteBucketMetricsConfigurationWithContext(aws.Context, *s3.DeleteBucketMetricsConfigurationInput, ...request.Option) (*s3.DeleteBucketMetricsConfigurationOutput, error) - DeleteBucketMetricsConfigurationRequest(*s3.DeleteBucketMetricsConfigurationInput) (*request.Request, *s3.DeleteBucketMetricsConfigurationOutput) - - DeleteBucketPolicy(*s3.DeleteBucketPolicyInput) (*s3.DeleteBucketPolicyOutput, error) - DeleteBucketPolicyWithContext(aws.Context, *s3.DeleteBucketPolicyInput, ...request.Option) (*s3.DeleteBucketPolicyOutput, error) - DeleteBucketPolicyRequest(*s3.DeleteBucketPolicyInput) (*request.Request, *s3.DeleteBucketPolicyOutput) - - DeleteBucketReplication(*s3.DeleteBucketReplicationInput) (*s3.DeleteBucketReplicationOutput, error) - DeleteBucketReplicationWithContext(aws.Context, *s3.DeleteBucketReplicationInput, ...request.Option) (*s3.DeleteBucketReplicationOutput, error) - DeleteBucketReplicationRequest(*s3.DeleteBucketReplicationInput) (*request.Request, *s3.DeleteBucketReplicationOutput) - - DeleteBucketTagging(*s3.DeleteBucketTaggingInput) (*s3.DeleteBucketTaggingOutput, error) - DeleteBucketTaggingWithContext(aws.Context, *s3.DeleteBucketTaggingInput, ...request.Option) (*s3.DeleteBucketTaggingOutput, error) - DeleteBucketTaggingRequest(*s3.DeleteBucketTaggingInput) (*request.Request, *s3.DeleteBucketTaggingOutput) - - DeleteBucketWebsite(*s3.DeleteBucketWebsiteInput) (*s3.DeleteBucketWebsiteOutput, error) - DeleteBucketWebsiteWithContext(aws.Context, *s3.DeleteBucketWebsiteInput, ...request.Option) (*s3.DeleteBucketWebsiteOutput, error) - DeleteBucketWebsiteRequest(*s3.DeleteBucketWebsiteInput) (*request.Request, *s3.DeleteBucketWebsiteOutput) - - DeleteObject(*s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) - DeleteObjectWithContext(aws.Context, *s3.DeleteObjectInput, ...request.Option) (*s3.DeleteObjectOutput, error) - DeleteObjectRequest(*s3.DeleteObjectInput) (*request.Request, *s3.DeleteObjectOutput) - - DeleteObjectTagging(*s3.DeleteObjectTaggingInput) (*s3.DeleteObjectTaggingOutput, error) - DeleteObjectTaggingWithContext(aws.Context, *s3.DeleteObjectTaggingInput, ...request.Option) (*s3.DeleteObjectTaggingOutput, error) - DeleteObjectTaggingRequest(*s3.DeleteObjectTaggingInput) (*request.Request, *s3.DeleteObjectTaggingOutput) - - DeleteObjects(*s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) - DeleteObjectsWithContext(aws.Context, *s3.DeleteObjectsInput, ...request.Option) (*s3.DeleteObjectsOutput, error) - DeleteObjectsRequest(*s3.DeleteObjectsInput) (*request.Request, *s3.DeleteObjectsOutput) - - DeletePublicAccessBlock(*s3.DeletePublicAccessBlockInput) (*s3.DeletePublicAccessBlockOutput, error) - DeletePublicAccessBlockWithContext(aws.Context, *s3.DeletePublicAccessBlockInput, ...request.Option) (*s3.DeletePublicAccessBlockOutput, error) - DeletePublicAccessBlockRequest(*s3.DeletePublicAccessBlockInput) (*request.Request, *s3.DeletePublicAccessBlockOutput) - - GetBucketAccelerateConfiguration(*s3.GetBucketAccelerateConfigurationInput) (*s3.GetBucketAccelerateConfigurationOutput, error) - GetBucketAccelerateConfigurationWithContext(aws.Context, *s3.GetBucketAccelerateConfigurationInput, ...request.Option) (*s3.GetBucketAccelerateConfigurationOutput, error) - GetBucketAccelerateConfigurationRequest(*s3.GetBucketAccelerateConfigurationInput) (*request.Request, *s3.GetBucketAccelerateConfigurationOutput) - - GetBucketAcl(*s3.GetBucketAclInput) (*s3.GetBucketAclOutput, error) - GetBucketAclWithContext(aws.Context, *s3.GetBucketAclInput, ...request.Option) (*s3.GetBucketAclOutput, error) - GetBucketAclRequest(*s3.GetBucketAclInput) (*request.Request, *s3.GetBucketAclOutput) - - GetBucketAnalyticsConfiguration(*s3.GetBucketAnalyticsConfigurationInput) (*s3.GetBucketAnalyticsConfigurationOutput, error) - GetBucketAnalyticsConfigurationWithContext(aws.Context, *s3.GetBucketAnalyticsConfigurationInput, ...request.Option) (*s3.GetBucketAnalyticsConfigurationOutput, error) - GetBucketAnalyticsConfigurationRequest(*s3.GetBucketAnalyticsConfigurationInput) (*request.Request, *s3.GetBucketAnalyticsConfigurationOutput) - - GetBucketCors(*s3.GetBucketCorsInput) (*s3.GetBucketCorsOutput, error) - GetBucketCorsWithContext(aws.Context, *s3.GetBucketCorsInput, ...request.Option) (*s3.GetBucketCorsOutput, error) - GetBucketCorsRequest(*s3.GetBucketCorsInput) (*request.Request, *s3.GetBucketCorsOutput) - - GetBucketEncryption(*s3.GetBucketEncryptionInput) (*s3.GetBucketEncryptionOutput, error) - GetBucketEncryptionWithContext(aws.Context, *s3.GetBucketEncryptionInput, ...request.Option) (*s3.GetBucketEncryptionOutput, error) - GetBucketEncryptionRequest(*s3.GetBucketEncryptionInput) (*request.Request, *s3.GetBucketEncryptionOutput) - - GetBucketInventoryConfiguration(*s3.GetBucketInventoryConfigurationInput) (*s3.GetBucketInventoryConfigurationOutput, error) - GetBucketInventoryConfigurationWithContext(aws.Context, *s3.GetBucketInventoryConfigurationInput, ...request.Option) (*s3.GetBucketInventoryConfigurationOutput, error) - GetBucketInventoryConfigurationRequest(*s3.GetBucketInventoryConfigurationInput) (*request.Request, *s3.GetBucketInventoryConfigurationOutput) - - GetBucketLifecycle(*s3.GetBucketLifecycleInput) (*s3.GetBucketLifecycleOutput, error) - GetBucketLifecycleWithContext(aws.Context, *s3.GetBucketLifecycleInput, ...request.Option) (*s3.GetBucketLifecycleOutput, error) - GetBucketLifecycleRequest(*s3.GetBucketLifecycleInput) (*request.Request, *s3.GetBucketLifecycleOutput) - - GetBucketLifecycleConfiguration(*s3.GetBucketLifecycleConfigurationInput) (*s3.GetBucketLifecycleConfigurationOutput, error) - GetBucketLifecycleConfigurationWithContext(aws.Context, *s3.GetBucketLifecycleConfigurationInput, ...request.Option) (*s3.GetBucketLifecycleConfigurationOutput, error) - GetBucketLifecycleConfigurationRequest(*s3.GetBucketLifecycleConfigurationInput) (*request.Request, *s3.GetBucketLifecycleConfigurationOutput) - - GetBucketLocation(*s3.GetBucketLocationInput) (*s3.GetBucketLocationOutput, error) - GetBucketLocationWithContext(aws.Context, *s3.GetBucketLocationInput, ...request.Option) (*s3.GetBucketLocationOutput, error) - GetBucketLocationRequest(*s3.GetBucketLocationInput) (*request.Request, *s3.GetBucketLocationOutput) - - GetBucketLogging(*s3.GetBucketLoggingInput) (*s3.GetBucketLoggingOutput, error) - GetBucketLoggingWithContext(aws.Context, *s3.GetBucketLoggingInput, ...request.Option) (*s3.GetBucketLoggingOutput, error) - GetBucketLoggingRequest(*s3.GetBucketLoggingInput) (*request.Request, *s3.GetBucketLoggingOutput) - - GetBucketMetricsConfiguration(*s3.GetBucketMetricsConfigurationInput) (*s3.GetBucketMetricsConfigurationOutput, error) - GetBucketMetricsConfigurationWithContext(aws.Context, *s3.GetBucketMetricsConfigurationInput, ...request.Option) (*s3.GetBucketMetricsConfigurationOutput, error) - GetBucketMetricsConfigurationRequest(*s3.GetBucketMetricsConfigurationInput) (*request.Request, *s3.GetBucketMetricsConfigurationOutput) - - GetBucketNotification(*s3.GetBucketNotificationConfigurationRequest) (*s3.NotificationConfigurationDeprecated, error) - GetBucketNotificationWithContext(aws.Context, *s3.GetBucketNotificationConfigurationRequest, ...request.Option) (*s3.NotificationConfigurationDeprecated, error) - GetBucketNotificationRequest(*s3.GetBucketNotificationConfigurationRequest) (*request.Request, *s3.NotificationConfigurationDeprecated) - - GetBucketNotificationConfiguration(*s3.GetBucketNotificationConfigurationRequest) (*s3.NotificationConfiguration, error) - GetBucketNotificationConfigurationWithContext(aws.Context, *s3.GetBucketNotificationConfigurationRequest, ...request.Option) (*s3.NotificationConfiguration, error) - GetBucketNotificationConfigurationRequest(*s3.GetBucketNotificationConfigurationRequest) (*request.Request, *s3.NotificationConfiguration) - - GetBucketPolicy(*s3.GetBucketPolicyInput) (*s3.GetBucketPolicyOutput, error) - GetBucketPolicyWithContext(aws.Context, *s3.GetBucketPolicyInput, ...request.Option) (*s3.GetBucketPolicyOutput, error) - GetBucketPolicyRequest(*s3.GetBucketPolicyInput) (*request.Request, *s3.GetBucketPolicyOutput) - - GetBucketPolicyStatus(*s3.GetBucketPolicyStatusInput) (*s3.GetBucketPolicyStatusOutput, error) - GetBucketPolicyStatusWithContext(aws.Context, *s3.GetBucketPolicyStatusInput, ...request.Option) (*s3.GetBucketPolicyStatusOutput, error) - GetBucketPolicyStatusRequest(*s3.GetBucketPolicyStatusInput) (*request.Request, *s3.GetBucketPolicyStatusOutput) - - GetBucketReplication(*s3.GetBucketReplicationInput) (*s3.GetBucketReplicationOutput, error) - GetBucketReplicationWithContext(aws.Context, *s3.GetBucketReplicationInput, ...request.Option) (*s3.GetBucketReplicationOutput, error) - GetBucketReplicationRequest(*s3.GetBucketReplicationInput) (*request.Request, *s3.GetBucketReplicationOutput) - - GetBucketRequestPayment(*s3.GetBucketRequestPaymentInput) (*s3.GetBucketRequestPaymentOutput, error) - GetBucketRequestPaymentWithContext(aws.Context, *s3.GetBucketRequestPaymentInput, ...request.Option) (*s3.GetBucketRequestPaymentOutput, error) - GetBucketRequestPaymentRequest(*s3.GetBucketRequestPaymentInput) (*request.Request, *s3.GetBucketRequestPaymentOutput) - - GetBucketTagging(*s3.GetBucketTaggingInput) (*s3.GetBucketTaggingOutput, error) - GetBucketTaggingWithContext(aws.Context, *s3.GetBucketTaggingInput, ...request.Option) (*s3.GetBucketTaggingOutput, error) - GetBucketTaggingRequest(*s3.GetBucketTaggingInput) (*request.Request, *s3.GetBucketTaggingOutput) - - GetBucketVersioning(*s3.GetBucketVersioningInput) (*s3.GetBucketVersioningOutput, error) - GetBucketVersioningWithContext(aws.Context, *s3.GetBucketVersioningInput, ...request.Option) (*s3.GetBucketVersioningOutput, error) - GetBucketVersioningRequest(*s3.GetBucketVersioningInput) (*request.Request, *s3.GetBucketVersioningOutput) - - GetBucketWebsite(*s3.GetBucketWebsiteInput) (*s3.GetBucketWebsiteOutput, error) - GetBucketWebsiteWithContext(aws.Context, *s3.GetBucketWebsiteInput, ...request.Option) (*s3.GetBucketWebsiteOutput, error) - GetBucketWebsiteRequest(*s3.GetBucketWebsiteInput) (*request.Request, *s3.GetBucketWebsiteOutput) - - GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) - GetObjectWithContext(aws.Context, *s3.GetObjectInput, ...request.Option) (*s3.GetObjectOutput, error) - GetObjectRequest(*s3.GetObjectInput) (*request.Request, *s3.GetObjectOutput) - - GetObjectAcl(*s3.GetObjectAclInput) (*s3.GetObjectAclOutput, error) - GetObjectAclWithContext(aws.Context, *s3.GetObjectAclInput, ...request.Option) (*s3.GetObjectAclOutput, error) - GetObjectAclRequest(*s3.GetObjectAclInput) (*request.Request, *s3.GetObjectAclOutput) - - GetObjectLegalHold(*s3.GetObjectLegalHoldInput) (*s3.GetObjectLegalHoldOutput, error) - GetObjectLegalHoldWithContext(aws.Context, *s3.GetObjectLegalHoldInput, ...request.Option) (*s3.GetObjectLegalHoldOutput, error) - GetObjectLegalHoldRequest(*s3.GetObjectLegalHoldInput) (*request.Request, *s3.GetObjectLegalHoldOutput) - - GetObjectLockConfiguration(*s3.GetObjectLockConfigurationInput) (*s3.GetObjectLockConfigurationOutput, error) - GetObjectLockConfigurationWithContext(aws.Context, *s3.GetObjectLockConfigurationInput, ...request.Option) (*s3.GetObjectLockConfigurationOutput, error) - GetObjectLockConfigurationRequest(*s3.GetObjectLockConfigurationInput) (*request.Request, *s3.GetObjectLockConfigurationOutput) - - GetObjectRetention(*s3.GetObjectRetentionInput) (*s3.GetObjectRetentionOutput, error) - GetObjectRetentionWithContext(aws.Context, *s3.GetObjectRetentionInput, ...request.Option) (*s3.GetObjectRetentionOutput, error) - GetObjectRetentionRequest(*s3.GetObjectRetentionInput) (*request.Request, *s3.GetObjectRetentionOutput) - - GetObjectTagging(*s3.GetObjectTaggingInput) (*s3.GetObjectTaggingOutput, error) - GetObjectTaggingWithContext(aws.Context, *s3.GetObjectTaggingInput, ...request.Option) (*s3.GetObjectTaggingOutput, error) - GetObjectTaggingRequest(*s3.GetObjectTaggingInput) (*request.Request, *s3.GetObjectTaggingOutput) - - GetObjectTorrent(*s3.GetObjectTorrentInput) (*s3.GetObjectTorrentOutput, error) - GetObjectTorrentWithContext(aws.Context, *s3.GetObjectTorrentInput, ...request.Option) (*s3.GetObjectTorrentOutput, error) - GetObjectTorrentRequest(*s3.GetObjectTorrentInput) (*request.Request, *s3.GetObjectTorrentOutput) - - GetPublicAccessBlock(*s3.GetPublicAccessBlockInput) (*s3.GetPublicAccessBlockOutput, error) - GetPublicAccessBlockWithContext(aws.Context, *s3.GetPublicAccessBlockInput, ...request.Option) (*s3.GetPublicAccessBlockOutput, error) - GetPublicAccessBlockRequest(*s3.GetPublicAccessBlockInput) (*request.Request, *s3.GetPublicAccessBlockOutput) - - HeadBucket(*s3.HeadBucketInput) (*s3.HeadBucketOutput, error) - HeadBucketWithContext(aws.Context, *s3.HeadBucketInput, ...request.Option) (*s3.HeadBucketOutput, error) - HeadBucketRequest(*s3.HeadBucketInput) (*request.Request, *s3.HeadBucketOutput) - - HeadObject(*s3.HeadObjectInput) (*s3.HeadObjectOutput, error) - HeadObjectWithContext(aws.Context, *s3.HeadObjectInput, ...request.Option) (*s3.HeadObjectOutput, error) - HeadObjectRequest(*s3.HeadObjectInput) (*request.Request, *s3.HeadObjectOutput) - - ListBucketAnalyticsConfigurations(*s3.ListBucketAnalyticsConfigurationsInput) (*s3.ListBucketAnalyticsConfigurationsOutput, error) - ListBucketAnalyticsConfigurationsWithContext(aws.Context, *s3.ListBucketAnalyticsConfigurationsInput, ...request.Option) (*s3.ListBucketAnalyticsConfigurationsOutput, error) - ListBucketAnalyticsConfigurationsRequest(*s3.ListBucketAnalyticsConfigurationsInput) (*request.Request, *s3.ListBucketAnalyticsConfigurationsOutput) - - ListBucketInventoryConfigurations(*s3.ListBucketInventoryConfigurationsInput) (*s3.ListBucketInventoryConfigurationsOutput, error) - ListBucketInventoryConfigurationsWithContext(aws.Context, *s3.ListBucketInventoryConfigurationsInput, ...request.Option) (*s3.ListBucketInventoryConfigurationsOutput, error) - ListBucketInventoryConfigurationsRequest(*s3.ListBucketInventoryConfigurationsInput) (*request.Request, *s3.ListBucketInventoryConfigurationsOutput) - - ListBucketMetricsConfigurations(*s3.ListBucketMetricsConfigurationsInput) (*s3.ListBucketMetricsConfigurationsOutput, error) - ListBucketMetricsConfigurationsWithContext(aws.Context, *s3.ListBucketMetricsConfigurationsInput, ...request.Option) (*s3.ListBucketMetricsConfigurationsOutput, error) - ListBucketMetricsConfigurationsRequest(*s3.ListBucketMetricsConfigurationsInput) (*request.Request, *s3.ListBucketMetricsConfigurationsOutput) - - ListBuckets(*s3.ListBucketsInput) (*s3.ListBucketsOutput, error) - ListBucketsWithContext(aws.Context, *s3.ListBucketsInput, ...request.Option) (*s3.ListBucketsOutput, error) - ListBucketsRequest(*s3.ListBucketsInput) (*request.Request, *s3.ListBucketsOutput) - - ListMultipartUploads(*s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error) - ListMultipartUploadsWithContext(aws.Context, *s3.ListMultipartUploadsInput, ...request.Option) (*s3.ListMultipartUploadsOutput, error) - ListMultipartUploadsRequest(*s3.ListMultipartUploadsInput) (*request.Request, *s3.ListMultipartUploadsOutput) - - ListMultipartUploadsPages(*s3.ListMultipartUploadsInput, func(*s3.ListMultipartUploadsOutput, bool) bool) error - ListMultipartUploadsPagesWithContext(aws.Context, *s3.ListMultipartUploadsInput, func(*s3.ListMultipartUploadsOutput, bool) bool, ...request.Option) error - - ListObjectVersions(*s3.ListObjectVersionsInput) (*s3.ListObjectVersionsOutput, error) - ListObjectVersionsWithContext(aws.Context, *s3.ListObjectVersionsInput, ...request.Option) (*s3.ListObjectVersionsOutput, error) - ListObjectVersionsRequest(*s3.ListObjectVersionsInput) (*request.Request, *s3.ListObjectVersionsOutput) - - ListObjectVersionsPages(*s3.ListObjectVersionsInput, func(*s3.ListObjectVersionsOutput, bool) bool) error - ListObjectVersionsPagesWithContext(aws.Context, *s3.ListObjectVersionsInput, func(*s3.ListObjectVersionsOutput, bool) bool, ...request.Option) error - - ListObjects(*s3.ListObjectsInput) (*s3.ListObjectsOutput, error) - ListObjectsWithContext(aws.Context, *s3.ListObjectsInput, ...request.Option) (*s3.ListObjectsOutput, error) - ListObjectsRequest(*s3.ListObjectsInput) (*request.Request, *s3.ListObjectsOutput) - - ListObjectsPages(*s3.ListObjectsInput, func(*s3.ListObjectsOutput, bool) bool) error - ListObjectsPagesWithContext(aws.Context, *s3.ListObjectsInput, func(*s3.ListObjectsOutput, bool) bool, ...request.Option) error - - ListObjectsV2(*s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) - ListObjectsV2WithContext(aws.Context, *s3.ListObjectsV2Input, ...request.Option) (*s3.ListObjectsV2Output, error) - ListObjectsV2Request(*s3.ListObjectsV2Input) (*request.Request, *s3.ListObjectsV2Output) - - ListObjectsV2Pages(*s3.ListObjectsV2Input, func(*s3.ListObjectsV2Output, bool) bool) error - ListObjectsV2PagesWithContext(aws.Context, *s3.ListObjectsV2Input, func(*s3.ListObjectsV2Output, bool) bool, ...request.Option) error - - ListParts(*s3.ListPartsInput) (*s3.ListPartsOutput, error) - ListPartsWithContext(aws.Context, *s3.ListPartsInput, ...request.Option) (*s3.ListPartsOutput, error) - ListPartsRequest(*s3.ListPartsInput) (*request.Request, *s3.ListPartsOutput) - - ListPartsPages(*s3.ListPartsInput, func(*s3.ListPartsOutput, bool) bool) error - ListPartsPagesWithContext(aws.Context, *s3.ListPartsInput, func(*s3.ListPartsOutput, bool) bool, ...request.Option) error - - PutBucketAccelerateConfiguration(*s3.PutBucketAccelerateConfigurationInput) (*s3.PutBucketAccelerateConfigurationOutput, error) - PutBucketAccelerateConfigurationWithContext(aws.Context, *s3.PutBucketAccelerateConfigurationInput, ...request.Option) (*s3.PutBucketAccelerateConfigurationOutput, error) - PutBucketAccelerateConfigurationRequest(*s3.PutBucketAccelerateConfigurationInput) (*request.Request, *s3.PutBucketAccelerateConfigurationOutput) - - PutBucketAcl(*s3.PutBucketAclInput) (*s3.PutBucketAclOutput, error) - PutBucketAclWithContext(aws.Context, *s3.PutBucketAclInput, ...request.Option) (*s3.PutBucketAclOutput, error) - PutBucketAclRequest(*s3.PutBucketAclInput) (*request.Request, *s3.PutBucketAclOutput) - - PutBucketAnalyticsConfiguration(*s3.PutBucketAnalyticsConfigurationInput) (*s3.PutBucketAnalyticsConfigurationOutput, error) - PutBucketAnalyticsConfigurationWithContext(aws.Context, *s3.PutBucketAnalyticsConfigurationInput, ...request.Option) (*s3.PutBucketAnalyticsConfigurationOutput, error) - PutBucketAnalyticsConfigurationRequest(*s3.PutBucketAnalyticsConfigurationInput) (*request.Request, *s3.PutBucketAnalyticsConfigurationOutput) - - PutBucketCors(*s3.PutBucketCorsInput) (*s3.PutBucketCorsOutput, error) - PutBucketCorsWithContext(aws.Context, *s3.PutBucketCorsInput, ...request.Option) (*s3.PutBucketCorsOutput, error) - PutBucketCorsRequest(*s3.PutBucketCorsInput) (*request.Request, *s3.PutBucketCorsOutput) - - PutBucketEncryption(*s3.PutBucketEncryptionInput) (*s3.PutBucketEncryptionOutput, error) - PutBucketEncryptionWithContext(aws.Context, *s3.PutBucketEncryptionInput, ...request.Option) (*s3.PutBucketEncryptionOutput, error) - PutBucketEncryptionRequest(*s3.PutBucketEncryptionInput) (*request.Request, *s3.PutBucketEncryptionOutput) - - PutBucketInventoryConfiguration(*s3.PutBucketInventoryConfigurationInput) (*s3.PutBucketInventoryConfigurationOutput, error) - PutBucketInventoryConfigurationWithContext(aws.Context, *s3.PutBucketInventoryConfigurationInput, ...request.Option) (*s3.PutBucketInventoryConfigurationOutput, error) - PutBucketInventoryConfigurationRequest(*s3.PutBucketInventoryConfigurationInput) (*request.Request, *s3.PutBucketInventoryConfigurationOutput) - - PutBucketLifecycle(*s3.PutBucketLifecycleInput) (*s3.PutBucketLifecycleOutput, error) - PutBucketLifecycleWithContext(aws.Context, *s3.PutBucketLifecycleInput, ...request.Option) (*s3.PutBucketLifecycleOutput, error) - PutBucketLifecycleRequest(*s3.PutBucketLifecycleInput) (*request.Request, *s3.PutBucketLifecycleOutput) - - PutBucketLifecycleConfiguration(*s3.PutBucketLifecycleConfigurationInput) (*s3.PutBucketLifecycleConfigurationOutput, error) - PutBucketLifecycleConfigurationWithContext(aws.Context, *s3.PutBucketLifecycleConfigurationInput, ...request.Option) (*s3.PutBucketLifecycleConfigurationOutput, error) - PutBucketLifecycleConfigurationRequest(*s3.PutBucketLifecycleConfigurationInput) (*request.Request, *s3.PutBucketLifecycleConfigurationOutput) - - PutBucketLogging(*s3.PutBucketLoggingInput) (*s3.PutBucketLoggingOutput, error) - PutBucketLoggingWithContext(aws.Context, *s3.PutBucketLoggingInput, ...request.Option) (*s3.PutBucketLoggingOutput, error) - PutBucketLoggingRequest(*s3.PutBucketLoggingInput) (*request.Request, *s3.PutBucketLoggingOutput) - - PutBucketMetricsConfiguration(*s3.PutBucketMetricsConfigurationInput) (*s3.PutBucketMetricsConfigurationOutput, error) - PutBucketMetricsConfigurationWithContext(aws.Context, *s3.PutBucketMetricsConfigurationInput, ...request.Option) (*s3.PutBucketMetricsConfigurationOutput, error) - PutBucketMetricsConfigurationRequest(*s3.PutBucketMetricsConfigurationInput) (*request.Request, *s3.PutBucketMetricsConfigurationOutput) - - PutBucketNotification(*s3.PutBucketNotificationInput) (*s3.PutBucketNotificationOutput, error) - PutBucketNotificationWithContext(aws.Context, *s3.PutBucketNotificationInput, ...request.Option) (*s3.PutBucketNotificationOutput, error) - PutBucketNotificationRequest(*s3.PutBucketNotificationInput) (*request.Request, *s3.PutBucketNotificationOutput) - - PutBucketNotificationConfiguration(*s3.PutBucketNotificationConfigurationInput) (*s3.PutBucketNotificationConfigurationOutput, error) - PutBucketNotificationConfigurationWithContext(aws.Context, *s3.PutBucketNotificationConfigurationInput, ...request.Option) (*s3.PutBucketNotificationConfigurationOutput, error) - PutBucketNotificationConfigurationRequest(*s3.PutBucketNotificationConfigurationInput) (*request.Request, *s3.PutBucketNotificationConfigurationOutput) - - PutBucketPolicy(*s3.PutBucketPolicyInput) (*s3.PutBucketPolicyOutput, error) - PutBucketPolicyWithContext(aws.Context, *s3.PutBucketPolicyInput, ...request.Option) (*s3.PutBucketPolicyOutput, error) - PutBucketPolicyRequest(*s3.PutBucketPolicyInput) (*request.Request, *s3.PutBucketPolicyOutput) - - PutBucketReplication(*s3.PutBucketReplicationInput) (*s3.PutBucketReplicationOutput, error) - PutBucketReplicationWithContext(aws.Context, *s3.PutBucketReplicationInput, ...request.Option) (*s3.PutBucketReplicationOutput, error) - PutBucketReplicationRequest(*s3.PutBucketReplicationInput) (*request.Request, *s3.PutBucketReplicationOutput) - - PutBucketRequestPayment(*s3.PutBucketRequestPaymentInput) (*s3.PutBucketRequestPaymentOutput, error) - PutBucketRequestPaymentWithContext(aws.Context, *s3.PutBucketRequestPaymentInput, ...request.Option) (*s3.PutBucketRequestPaymentOutput, error) - PutBucketRequestPaymentRequest(*s3.PutBucketRequestPaymentInput) (*request.Request, *s3.PutBucketRequestPaymentOutput) - - PutBucketTagging(*s3.PutBucketTaggingInput) (*s3.PutBucketTaggingOutput, error) - PutBucketTaggingWithContext(aws.Context, *s3.PutBucketTaggingInput, ...request.Option) (*s3.PutBucketTaggingOutput, error) - PutBucketTaggingRequest(*s3.PutBucketTaggingInput) (*request.Request, *s3.PutBucketTaggingOutput) - - PutBucketVersioning(*s3.PutBucketVersioningInput) (*s3.PutBucketVersioningOutput, error) - PutBucketVersioningWithContext(aws.Context, *s3.PutBucketVersioningInput, ...request.Option) (*s3.PutBucketVersioningOutput, error) - PutBucketVersioningRequest(*s3.PutBucketVersioningInput) (*request.Request, *s3.PutBucketVersioningOutput) - - PutBucketWebsite(*s3.PutBucketWebsiteInput) (*s3.PutBucketWebsiteOutput, error) - PutBucketWebsiteWithContext(aws.Context, *s3.PutBucketWebsiteInput, ...request.Option) (*s3.PutBucketWebsiteOutput, error) - PutBucketWebsiteRequest(*s3.PutBucketWebsiteInput) (*request.Request, *s3.PutBucketWebsiteOutput) - - PutObject(*s3.PutObjectInput) (*s3.PutObjectOutput, error) - PutObjectWithContext(aws.Context, *s3.PutObjectInput, ...request.Option) (*s3.PutObjectOutput, error) - PutObjectRequest(*s3.PutObjectInput) (*request.Request, *s3.PutObjectOutput) - - PutObjectAcl(*s3.PutObjectAclInput) (*s3.PutObjectAclOutput, error) - PutObjectAclWithContext(aws.Context, *s3.PutObjectAclInput, ...request.Option) (*s3.PutObjectAclOutput, error) - PutObjectAclRequest(*s3.PutObjectAclInput) (*request.Request, *s3.PutObjectAclOutput) - - PutObjectLegalHold(*s3.PutObjectLegalHoldInput) (*s3.PutObjectLegalHoldOutput, error) - PutObjectLegalHoldWithContext(aws.Context, *s3.PutObjectLegalHoldInput, ...request.Option) (*s3.PutObjectLegalHoldOutput, error) - PutObjectLegalHoldRequest(*s3.PutObjectLegalHoldInput) (*request.Request, *s3.PutObjectLegalHoldOutput) - - PutObjectLockConfiguration(*s3.PutObjectLockConfigurationInput) (*s3.PutObjectLockConfigurationOutput, error) - PutObjectLockConfigurationWithContext(aws.Context, *s3.PutObjectLockConfigurationInput, ...request.Option) (*s3.PutObjectLockConfigurationOutput, error) - PutObjectLockConfigurationRequest(*s3.PutObjectLockConfigurationInput) (*request.Request, *s3.PutObjectLockConfigurationOutput) - - PutObjectRetention(*s3.PutObjectRetentionInput) (*s3.PutObjectRetentionOutput, error) - PutObjectRetentionWithContext(aws.Context, *s3.PutObjectRetentionInput, ...request.Option) (*s3.PutObjectRetentionOutput, error) - PutObjectRetentionRequest(*s3.PutObjectRetentionInput) (*request.Request, *s3.PutObjectRetentionOutput) - - PutObjectTagging(*s3.PutObjectTaggingInput) (*s3.PutObjectTaggingOutput, error) - PutObjectTaggingWithContext(aws.Context, *s3.PutObjectTaggingInput, ...request.Option) (*s3.PutObjectTaggingOutput, error) - PutObjectTaggingRequest(*s3.PutObjectTaggingInput) (*request.Request, *s3.PutObjectTaggingOutput) - - PutPublicAccessBlock(*s3.PutPublicAccessBlockInput) (*s3.PutPublicAccessBlockOutput, error) - PutPublicAccessBlockWithContext(aws.Context, *s3.PutPublicAccessBlockInput, ...request.Option) (*s3.PutPublicAccessBlockOutput, error) - PutPublicAccessBlockRequest(*s3.PutPublicAccessBlockInput) (*request.Request, *s3.PutPublicAccessBlockOutput) - - RestoreObject(*s3.RestoreObjectInput) (*s3.RestoreObjectOutput, error) - RestoreObjectWithContext(aws.Context, *s3.RestoreObjectInput, ...request.Option) (*s3.RestoreObjectOutput, error) - RestoreObjectRequest(*s3.RestoreObjectInput) (*request.Request, *s3.RestoreObjectOutput) - - SelectObjectContent(*s3.SelectObjectContentInput) (*s3.SelectObjectContentOutput, error) - SelectObjectContentWithContext(aws.Context, *s3.SelectObjectContentInput, ...request.Option) (*s3.SelectObjectContentOutput, error) - SelectObjectContentRequest(*s3.SelectObjectContentInput) (*request.Request, *s3.SelectObjectContentOutput) - - UploadPart(*s3.UploadPartInput) (*s3.UploadPartOutput, error) - UploadPartWithContext(aws.Context, *s3.UploadPartInput, ...request.Option) (*s3.UploadPartOutput, error) - UploadPartRequest(*s3.UploadPartInput) (*request.Request, *s3.UploadPartOutput) - - UploadPartCopy(*s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) - UploadPartCopyWithContext(aws.Context, *s3.UploadPartCopyInput, ...request.Option) (*s3.UploadPartCopyOutput, error) - UploadPartCopyRequest(*s3.UploadPartCopyInput) (*request.Request, *s3.UploadPartCopyOutput) - - WaitUntilBucketExists(*s3.HeadBucketInput) error - WaitUntilBucketExistsWithContext(aws.Context, *s3.HeadBucketInput, ...request.WaiterOption) error - - WaitUntilBucketNotExists(*s3.HeadBucketInput) error - WaitUntilBucketNotExistsWithContext(aws.Context, *s3.HeadBucketInput, ...request.WaiterOption) error - - WaitUntilObjectExists(*s3.HeadObjectInput) error - WaitUntilObjectExistsWithContext(aws.Context, *s3.HeadObjectInput, ...request.WaiterOption) error - - WaitUntilObjectNotExists(*s3.HeadObjectInput) error - WaitUntilObjectNotExistsWithContext(aws.Context, *s3.HeadObjectInput, ...request.WaiterOption) error -} - -var _ S3API = (*s3.S3)(nil) diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go deleted file mode 100644 index 22bd0b7ce5..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/batch.go +++ /dev/null @@ -1,529 +0,0 @@ -package s3manager - -import ( - "bytes" - "fmt" - "io" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" -) - -const ( - // DefaultBatchSize is the batch size we initialize when constructing a batch delete client. - // This value is used when calling DeleteObjects. This represents how many objects to delete - // per DeleteObjects call. - DefaultBatchSize = 100 -) - -// BatchError will contain the key and bucket of the object that failed to -// either upload or download. -type BatchError struct { - Errors Errors - code string - message string -} - -// Errors is a typed alias for a slice of errors to satisfy the error -// interface. -type Errors []Error - -func (errs Errors) Error() string { - buf := bytes.NewBuffer(nil) - for i, err := range errs { - buf.WriteString(err.Error()) - if i+1 < len(errs) { - buf.WriteString("\n") - } - } - return buf.String() -} - -// Error will contain the original error, bucket, and key of the operation that failed -// during batch operations. -type Error struct { - OrigErr error - Bucket *string - Key *string -} - -func newError(err error, bucket, key *string) Error { - return Error{ - err, - bucket, - key, - } -} - -func (err *Error) Error() string { - origErr := "" - if err.OrigErr != nil { - origErr = ":\n" + err.OrigErr.Error() - } - return fmt.Sprintf("failed to perform batch operation on %q to %q%s", - aws.StringValue(err.Key), - aws.StringValue(err.Bucket), - origErr, - ) -} - -// NewBatchError will return a BatchError that satisfies the awserr.Error interface. -func NewBatchError(code, message string, err []Error) awserr.Error { - return &BatchError{ - Errors: err, - code: code, - message: message, - } -} - -// Code will return the code associated with the batch error. -func (err *BatchError) Code() string { - return err.code -} - -// Message will return the message associated with the batch error. -func (err *BatchError) Message() string { - return err.message -} - -func (err *BatchError) Error() string { - return awserr.SprintError(err.Code(), err.Message(), "", err.Errors) -} - -// OrigErr will return the original error. Which, in this case, will always be nil -// for batched operations. -func (err *BatchError) OrigErr() error { - return err.Errors -} - -// BatchDeleteIterator is an interface that uses the scanner pattern to -// iterate through what needs to be deleted. -type BatchDeleteIterator interface { - Next() bool - Err() error - DeleteObject() BatchDeleteObject -} - -// DeleteListIterator is an alternative iterator for the BatchDelete client. This will -// iterate through a list of objects and delete the objects. -// -// Example: -// iter := &s3manager.DeleteListIterator{ -// Client: svc, -// Input: &s3.ListObjectsInput{ -// Bucket: aws.String("bucket"), -// MaxKeys: aws.Int64(5), -// }, -// Paginator: request.Pagination{ -// NewRequest: func() (*request.Request, error) { -// var inCpy *ListObjectsInput -// if input != nil { -// tmp := *input -// inCpy = &tmp -// } -// req, _ := c.ListObjectsRequest(inCpy) -// return req, nil -// }, -// }, -// } -// -// batcher := s3manager.NewBatchDeleteWithClient(svc) -// if err := batcher.Delete(aws.BackgroundContext(), iter); err != nil { -// return err -// } -type DeleteListIterator struct { - Bucket *string - Paginator request.Pagination - objects []*s3.Object -} - -// NewDeleteListIterator will return a new DeleteListIterator. -func NewDeleteListIterator(svc s3iface.S3API, input *s3.ListObjectsInput, opts ...func(*DeleteListIterator)) BatchDeleteIterator { - iter := &DeleteListIterator{ - Bucket: input.Bucket, - Paginator: request.Pagination{ - NewRequest: func() (*request.Request, error) { - var inCpy *s3.ListObjectsInput - if input != nil { - tmp := *input - inCpy = &tmp - } - req, _ := svc.ListObjectsRequest(inCpy) - return req, nil - }, - }, - } - - for _, opt := range opts { - opt(iter) - } - return iter -} - -// Next will use the S3API client to iterate through a list of objects. -func (iter *DeleteListIterator) Next() bool { - if len(iter.objects) > 0 { - iter.objects = iter.objects[1:] - } - - if len(iter.objects) == 0 && iter.Paginator.Next() { - iter.objects = iter.Paginator.Page().(*s3.ListObjectsOutput).Contents - } - - return len(iter.objects) > 0 -} - -// Err will return the last known error from Next. -func (iter *DeleteListIterator) Err() error { - return iter.Paginator.Err() -} - -// DeleteObject will return the current object to be deleted. -func (iter *DeleteListIterator) DeleteObject() BatchDeleteObject { - return BatchDeleteObject{ - Object: &s3.DeleteObjectInput{ - Bucket: iter.Bucket, - Key: iter.objects[0].Key, - }, - } -} - -// BatchDelete will use the s3 package's service client to perform a batch -// delete. -type BatchDelete struct { - Client s3iface.S3API - BatchSize int -} - -// NewBatchDeleteWithClient will return a new delete client that can delete a batched amount of -// objects. -// -// Example: -// batcher := s3manager.NewBatchDeleteWithClient(client, size) -// -// objects := []BatchDeleteObject{ -// { -// Object: &s3.DeleteObjectInput { -// Key: aws.String("key"), -// Bucket: aws.String("bucket"), -// }, -// }, -// } -// -// if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{ -// Objects: objects, -// }); err != nil { -// return err -// } -func NewBatchDeleteWithClient(client s3iface.S3API, options ...func(*BatchDelete)) *BatchDelete { - svc := &BatchDelete{ - Client: client, - BatchSize: DefaultBatchSize, - } - - for _, opt := range options { - opt(svc) - } - - return svc -} - -// NewBatchDelete will return a new delete client that can delete a batched amount of -// objects. -// -// Example: -// batcher := s3manager.NewBatchDelete(sess, size) -// -// objects := []BatchDeleteObject{ -// { -// Object: &s3.DeleteObjectInput { -// Key: aws.String("key"), -// Bucket: aws.String("bucket"), -// }, -// }, -// } -// -// if err := batcher.Delete(aws.BackgroundContext(), &s3manager.DeleteObjectsIterator{ -// Objects: objects, -// }); err != nil { -// return err -// } -func NewBatchDelete(c client.ConfigProvider, options ...func(*BatchDelete)) *BatchDelete { - client := s3.New(c) - return NewBatchDeleteWithClient(client, options...) -} - -// BatchDeleteObject is a wrapper object for calling the batch delete operation. -type BatchDeleteObject struct { - Object *s3.DeleteObjectInput - // After will run after each iteration during the batch process. This function will - // be executed whether or not the request was successful. - After func() error -} - -// DeleteObjectsIterator is an interface that uses the scanner pattern to iterate -// through a series of objects to be deleted. -type DeleteObjectsIterator struct { - Objects []BatchDeleteObject - index int - inc bool -} - -// Next will increment the default iterator's index and ensure that there -// is another object to iterator to. -func (iter *DeleteObjectsIterator) Next() bool { - if iter.inc { - iter.index++ - } else { - iter.inc = true - } - return iter.index < len(iter.Objects) -} - -// Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface -// this will only return nil. -func (iter *DeleteObjectsIterator) Err() error { - return nil -} - -// DeleteObject will return the BatchDeleteObject at the current batched index. -func (iter *DeleteObjectsIterator) DeleteObject() BatchDeleteObject { - object := iter.Objects[iter.index] - return object -} - -// Delete will use the iterator to queue up objects that need to be deleted. -// Once the batch size is met, this will call the deleteBatch function. -func (d *BatchDelete) Delete(ctx aws.Context, iter BatchDeleteIterator) error { - var errs []Error - objects := []BatchDeleteObject{} - var input *s3.DeleteObjectsInput - - for iter.Next() { - o := iter.DeleteObject() - - if input == nil { - input = initDeleteObjectsInput(o.Object) - } - - parity := hasParity(input, o) - if parity { - input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{ - Key: o.Object.Key, - VersionId: o.Object.VersionId, - }) - objects = append(objects, o) - } - - if len(input.Delete.Objects) == d.BatchSize || !parity { - if err := deleteBatch(ctx, d, input, objects); err != nil { - errs = append(errs, err...) - } - - objects = objects[:0] - input = nil - - if !parity { - objects = append(objects, o) - input = initDeleteObjectsInput(o.Object) - input.Delete.Objects = append(input.Delete.Objects, &s3.ObjectIdentifier{ - Key: o.Object.Key, - VersionId: o.Object.VersionId, - }) - } - } - } - - // iter.Next() could return false (above) plus populate iter.Err() - if iter.Err() != nil { - errs = append(errs, newError(iter.Err(), nil, nil)) - } - - if input != nil && len(input.Delete.Objects) > 0 { - if err := deleteBatch(ctx, d, input, objects); err != nil { - errs = append(errs, err...) - } - } - - if len(errs) > 0 { - return NewBatchError("BatchedDeleteIncomplete", "some objects have failed to be deleted.", errs) - } - return nil -} - -func initDeleteObjectsInput(o *s3.DeleteObjectInput) *s3.DeleteObjectsInput { - return &s3.DeleteObjectsInput{ - Bucket: o.Bucket, - MFA: o.MFA, - RequestPayer: o.RequestPayer, - Delete: &s3.Delete{}, - } -} - -const ( - // ErrDeleteBatchFailCode represents an error code which will be returned - // only when DeleteObjects.Errors has an error that does not contain a code. - ErrDeleteBatchFailCode = "DeleteBatchError" - errDefaultDeleteBatchMessage = "failed to delete" -) - -// deleteBatch will delete a batch of items in the objects parameters. -func deleteBatch(ctx aws.Context, d *BatchDelete, input *s3.DeleteObjectsInput, objects []BatchDeleteObject) []Error { - errs := []Error{} - - if result, err := d.Client.DeleteObjectsWithContext(ctx, input); err != nil { - for i := 0; i < len(input.Delete.Objects); i++ { - errs = append(errs, newError(err, input.Bucket, input.Delete.Objects[i].Key)) - } - } else if len(result.Errors) > 0 { - for i := 0; i < len(result.Errors); i++ { - code := ErrDeleteBatchFailCode - msg := errDefaultDeleteBatchMessage - if result.Errors[i].Message != nil { - msg = *result.Errors[i].Message - } - if result.Errors[i].Code != nil { - code = *result.Errors[i].Code - } - - errs = append(errs, newError(awserr.New(code, msg, err), input.Bucket, result.Errors[i].Key)) - } - } - for _, object := range objects { - if object.After == nil { - continue - } - if err := object.After(); err != nil { - errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key)) - } - } - - return errs -} - -func hasParity(o1 *s3.DeleteObjectsInput, o2 BatchDeleteObject) bool { - if o1.Bucket != nil && o2.Object.Bucket != nil { - if *o1.Bucket != *o2.Object.Bucket { - return false - } - } else if o1.Bucket != o2.Object.Bucket { - return false - } - - if o1.MFA != nil && o2.Object.MFA != nil { - if *o1.MFA != *o2.Object.MFA { - return false - } - } else if o1.MFA != o2.Object.MFA { - return false - } - - if o1.RequestPayer != nil && o2.Object.RequestPayer != nil { - if *o1.RequestPayer != *o2.Object.RequestPayer { - return false - } - } else if o1.RequestPayer != o2.Object.RequestPayer { - return false - } - - return true -} - -// BatchDownloadIterator is an interface that uses the scanner pattern to iterate -// through a series of objects to be downloaded. -type BatchDownloadIterator interface { - Next() bool - Err() error - DownloadObject() BatchDownloadObject -} - -// BatchDownloadObject contains all necessary information to run a batch operation once. -type BatchDownloadObject struct { - Object *s3.GetObjectInput - Writer io.WriterAt - // After will run after each iteration during the batch process. This function will - // be executed whether or not the request was successful. - After func() error -} - -// DownloadObjectsIterator implements the BatchDownloadIterator interface and allows for batched -// download of objects. -type DownloadObjectsIterator struct { - Objects []BatchDownloadObject - index int - inc bool -} - -// Next will increment the default iterator's index and ensure that there -// is another object to iterator to. -func (batcher *DownloadObjectsIterator) Next() bool { - if batcher.inc { - batcher.index++ - } else { - batcher.inc = true - } - return batcher.index < len(batcher.Objects) -} - -// DownloadObject will return the BatchDownloadObject at the current batched index. -func (batcher *DownloadObjectsIterator) DownloadObject() BatchDownloadObject { - object := batcher.Objects[batcher.index] - return object -} - -// Err will return an error. Since this is just used to satisfy the BatchDeleteIterator interface -// this will only return nil. -func (batcher *DownloadObjectsIterator) Err() error { - return nil -} - -// BatchUploadIterator is an interface that uses the scanner pattern to -// iterate through what needs to be uploaded. -type BatchUploadIterator interface { - Next() bool - Err() error - UploadObject() BatchUploadObject -} - -// UploadObjectsIterator implements the BatchUploadIterator interface and allows for batched -// upload of objects. -type UploadObjectsIterator struct { - Objects []BatchUploadObject - index int - inc bool -} - -// Next will increment the default iterator's index and ensure that there -// is another object to iterator to. -func (batcher *UploadObjectsIterator) Next() bool { - if batcher.inc { - batcher.index++ - } else { - batcher.inc = true - } - return batcher.index < len(batcher.Objects) -} - -// Err will return an error. Since this is just used to satisfy the BatchUploadIterator interface -// this will only return nil. -func (batcher *UploadObjectsIterator) Err() error { - return nil -} - -// UploadObject will return the BatchUploadObject at the current batched index. -func (batcher *UploadObjectsIterator) UploadObject() BatchUploadObject { - object := batcher.Objects[batcher.index] - return object -} - -// BatchUploadObject contains all necessary information to run a batch operation once. -type BatchUploadObject struct { - Object *UploadInput - // After will run after each iteration during the batch process. This function will - // be executed whether or not the request was successful. - After func() error -} diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go deleted file mode 100644 index f61665a58a..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/bucket_region.go +++ /dev/null @@ -1,88 +0,0 @@ -package s3manager - -import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" -) - -// GetBucketRegion will attempt to get the region for a bucket using the -// regionHint to determine which AWS partition to perform the query on. -// -// The request will not be signed, and will not use your AWS credentials. -// -// A "NotFound" error code will be returned if the bucket does not exist in the -// AWS partition the regionHint belongs to. If the regionHint parameter is an -// empty string GetBucketRegion will fallback to the ConfigProvider's region -// config. If the regionHint is empty, and the ConfigProvider does not have a -// region value, an error will be returned.. -// -// For example to get the region of a bucket which exists in "eu-central-1" -// you could provide a region hint of "us-west-2". -// -// sess := session.Must(session.NewSession()) -// -// bucket := "my-bucket" -// region, err := s3manager.GetBucketRegion(ctx, sess, bucket, "us-west-2") -// if err != nil { -// if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "NotFound" { -// fmt.Fprintf(os.Stderr, "unable to find bucket %s's region not found\n", bucket) -// } -// return err -// } -// fmt.Printf("Bucket %s is in %s region\n", bucket, region) -// -func GetBucketRegion(ctx aws.Context, c client.ConfigProvider, bucket, regionHint string, opts ...request.Option) (string, error) { - var cfg aws.Config - if len(regionHint) != 0 { - cfg.Region = aws.String(regionHint) - } - svc := s3.New(c, &cfg) - return GetBucketRegionWithClient(ctx, svc, bucket, opts...) -} - -const bucketRegionHeader = "X-Amz-Bucket-Region" - -// GetBucketRegionWithClient is the same as GetBucketRegion with the exception -// that it takes a S3 service client instead of a Session. The regionHint is -// derived from the region the S3 service client was created in. -// -// See GetBucketRegion for more information. -func GetBucketRegionWithClient(ctx aws.Context, svc s3iface.S3API, bucket string, opts ...request.Option) (string, error) { - req, _ := svc.HeadBucketRequest(&s3.HeadBucketInput{ - Bucket: aws.String(bucket), - }) - req.Config.S3ForcePathStyle = aws.Bool(true) - req.Config.Credentials = credentials.AnonymousCredentials - req.SetContext(ctx) - - // Disable HTTP redirects to prevent an invalid 301 from eating the response - // because Go's HTTP client will fail, and drop the response if an 301 is - // received without a location header. S3 will return a 301 without the - // location header for HeadObject API calls. - req.DisableFollowRedirects = true - - var bucketRegion string - req.Handlers.Send.PushBack(func(r *request.Request) { - bucketRegion = r.HTTPResponse.Header.Get(bucketRegionHeader) - if len(bucketRegion) == 0 { - return - } - r.HTTPResponse.StatusCode = 200 - r.HTTPResponse.Status = "OK" - r.Error = nil - }) - - req.ApplyOptions(opts...) - - if err := req.Send(); err != nil { - return "", err - } - - bucketRegion = s3.NormalizeBucketLocation(bucketRegion) - - return bucketRegion, nil -} diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go deleted file mode 100644 index 229c0d63bd..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// Package s3manager provides utilities to upload and download objects from -// S3 concurrently. Helpful for when working with large objects. -package s3manager diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go deleted file mode 100644 index 498f0b2b0a..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/download.go +++ /dev/null @@ -1,557 +0,0 @@ -package s3manager - -import ( - "fmt" - "io" - "net/http" - "strconv" - "strings" - "sync" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/awsutil" - "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" -) - -// DefaultDownloadPartSize is the default range of bytes to get at a time when -// using Download(). -const DefaultDownloadPartSize = 1024 * 1024 * 5 - -// DefaultDownloadConcurrency is the default number of goroutines to spin up -// when using Download(). -const DefaultDownloadConcurrency = 5 - -// The Downloader structure that calls Download(). It is safe to call Download() -// on this structure for multiple objects and across concurrent goroutines. -// Mutating the Downloader's properties is not safe to be done concurrently. -type Downloader struct { - // The buffer size (in bytes) to use when buffering data into chunks and - // sending them as parts to S3. The minimum allowed part size is 5MB, and - // if this value is set to zero, the DefaultDownloadPartSize value will be used. - // - // PartSize is ignored if the Range input parameter is provided. - PartSize int64 - - // The number of goroutines to spin up in parallel when sending parts. - // If this is set to zero, the DefaultDownloadConcurrency value will be used. - // - // Concurrency of 1 will download the parts sequentially. - // - // Concurrency is ignored if the Range input parameter is provided. - Concurrency int - - // An S3 client to use when performing downloads. - S3 s3iface.S3API - - // List of request options that will be passed down to individual API - // operation requests made by the downloader. - RequestOptions []request.Option -} - -// WithDownloaderRequestOptions appends to the Downloader's API request options. -func WithDownloaderRequestOptions(opts ...request.Option) func(*Downloader) { - return func(d *Downloader) { - d.RequestOptions = append(d.RequestOptions, opts...) - } -} - -// NewDownloader creates a new Downloader instance to downloads objects from -// S3 in concurrent chunks. Pass in additional functional options to customize -// the downloader behavior. Requires a client.ConfigProvider in order to create -// a S3 service client. The session.Session satisfies the client.ConfigProvider -// interface. -// -// Example: -// // The session the S3 Downloader will use -// sess := session.Must(session.NewSession()) -// -// // Create a downloader with the session and default options -// downloader := s3manager.NewDownloader(sess) -// -// // Create a downloader with the session and custom options -// downloader := s3manager.NewDownloader(sess, func(d *s3manager.Downloader) { -// d.PartSize = 64 * 1024 * 1024 // 64MB per part -// }) -func NewDownloader(c client.ConfigProvider, options ...func(*Downloader)) *Downloader { - d := &Downloader{ - S3: s3.New(c), - PartSize: DefaultDownloadPartSize, - Concurrency: DefaultDownloadConcurrency, - } - for _, option := range options { - option(d) - } - - return d -} - -// NewDownloaderWithClient creates a new Downloader instance to downloads -// objects from S3 in concurrent chunks. Pass in additional functional -// options to customize the downloader behavior. Requires a S3 service client -// to make S3 API calls. -// -// Example: -// // The session the S3 Downloader will use -// sess := session.Must(session.NewSession()) -// -// // The S3 client the S3 Downloader will use -// s3Svc := s3.new(sess) -// -// // Create a downloader with the s3 client and default options -// downloader := s3manager.NewDownloaderWithClient(s3Svc) -// -// // Create a downloader with the s3 client and custom options -// downloader := s3manager.NewDownloaderWithClient(s3Svc, func(d *s3manager.Downloader) { -// d.PartSize = 64 * 1024 * 1024 // 64MB per part -// }) -func NewDownloaderWithClient(svc s3iface.S3API, options ...func(*Downloader)) *Downloader { - d := &Downloader{ - S3: svc, - PartSize: DefaultDownloadPartSize, - Concurrency: DefaultDownloadConcurrency, - } - for _, option := range options { - option(d) - } - - return d -} - -type maxRetrier interface { - MaxRetries() int -} - -// Download downloads an object in S3 and writes the payload into w using -// concurrent GET requests. The n int64 returned is the size of the object downloaded -// in bytes. -// -// Additional functional options can be provided to configure the individual -// download. These options are copies of the Downloader instance Download is called from. -// Modifying the options will not impact the original Downloader instance. -// -// It is safe to call this method concurrently across goroutines. -// -// The w io.WriterAt can be satisfied by an os.File to do multipart concurrent -// downloads, or in memory []byte wrapper using aws.WriteAtBuffer. -// -// Specifying a Downloader.Concurrency of 1 will cause the Downloader to -// download the parts from S3 sequentially. -// -// If the GetObjectInput's Range value is provided that will cause the downloader -// to perform a single GetObjectInput request for that object's range. This will -// caused the part size, and concurrency configurations to be ignored. -func (d Downloader) Download(w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) { - return d.DownloadWithContext(aws.BackgroundContext(), w, input, options...) -} - -// DownloadWithContext downloads an object in S3 and writes the payload into w -// using concurrent GET requests. The n int64 returned is the size of the object downloaded -// in bytes. -// -// DownloadWithContext is the same as Download with the additional support for -// Context input parameters. The Context must not be nil. A nil Context will -// cause a panic. Use the Context to add deadlining, timeouts, etc. The -// DownloadWithContext may create sub-contexts for individual underlying -// requests. -// -// Additional functional options can be provided to configure the individual -// download. These options are copies of the Downloader instance Download is -// called from. Modifying the options will not impact the original Downloader -// instance. Use the WithDownloaderRequestOptions helper function to pass in request -// options that will be applied to all API operations made with this downloader. -// -// The w io.WriterAt can be satisfied by an os.File to do multipart concurrent -// downloads, or in memory []byte wrapper using aws.WriteAtBuffer. -// -// Specifying a Downloader.Concurrency of 1 will cause the Downloader to -// download the parts from S3 sequentially. -// -// It is safe to call this method concurrently across goroutines. -// -// If the GetObjectInput's Range value is provided that will cause the downloader -// to perform a single GetObjectInput request for that object's range. This will -// caused the part size, and concurrency configurations to be ignored. -func (d Downloader) DownloadWithContext(ctx aws.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*Downloader)) (n int64, err error) { - impl := downloader{w: w, in: input, cfg: d, ctx: ctx} - - for _, option := range options { - option(&impl.cfg) - } - impl.cfg.RequestOptions = append(impl.cfg.RequestOptions, request.WithAppendUserAgent("S3Manager")) - - if s, ok := d.S3.(maxRetrier); ok { - impl.partBodyMaxRetries = s.MaxRetries() - } - - impl.totalBytes = -1 - if impl.cfg.Concurrency == 0 { - impl.cfg.Concurrency = DefaultDownloadConcurrency - } - - if impl.cfg.PartSize == 0 { - impl.cfg.PartSize = DefaultDownloadPartSize - } - - return impl.download() -} - -// DownloadWithIterator will download a batched amount of objects in S3 and writes them -// to the io.WriterAt specificed in the iterator. -// -// Example: -// svc := s3manager.NewDownloader(session) -// -// fooFile, err := os.Open("/tmp/foo.file") -// if err != nil { -// return err -// } -// -// barFile, err := os.Open("/tmp/bar.file") -// if err != nil { -// return err -// } -// -// objects := []s3manager.BatchDownloadObject { -// { -// Object: &s3.GetObjectInput { -// Bucket: aws.String("bucket"), -// Key: aws.String("foo"), -// }, -// Writer: fooFile, -// }, -// { -// Object: &s3.GetObjectInput { -// Bucket: aws.String("bucket"), -// Key: aws.String("bar"), -// }, -// Writer: barFile, -// }, -// } -// -// iter := &s3manager.DownloadObjectsIterator{Objects: objects} -// if err := svc.DownloadWithIterator(aws.BackgroundContext(), iter); err != nil { -// return err -// } -func (d Downloader) DownloadWithIterator(ctx aws.Context, iter BatchDownloadIterator, opts ...func(*Downloader)) error { - var errs []Error - for iter.Next() { - object := iter.DownloadObject() - if _, err := d.DownloadWithContext(ctx, object.Writer, object.Object, opts...); err != nil { - errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key)) - } - - if object.After == nil { - continue - } - - if err := object.After(); err != nil { - errs = append(errs, newError(err, object.Object.Bucket, object.Object.Key)) - } - } - - if len(errs) > 0 { - return NewBatchError("BatchedDownloadIncomplete", "some objects have failed to download.", errs) - } - return nil -} - -// downloader is the implementation structure used internally by Downloader. -type downloader struct { - ctx aws.Context - cfg Downloader - - in *s3.GetObjectInput - w io.WriterAt - - wg sync.WaitGroup - m sync.Mutex - - pos int64 - totalBytes int64 - written int64 - err error - - partBodyMaxRetries int -} - -// download performs the implementation of the object download across ranged -// GETs. -func (d *downloader) download() (n int64, err error) { - // If range is specified fall back to single download of that range - // this enables the functionality of ranged gets with the downloader but - // at the cost of no multipart downloads. - if rng := aws.StringValue(d.in.Range); len(rng) > 0 { - d.downloadRange(rng) - return d.written, d.err - } - - // Spin off first worker to check additional header information - d.getChunk() - - if total := d.getTotalBytes(); total >= 0 { - // Spin up workers - ch := make(chan dlchunk, d.cfg.Concurrency) - - for i := 0; i < d.cfg.Concurrency; i++ { - d.wg.Add(1) - go d.downloadPart(ch) - } - - // Assign work - for d.getErr() == nil { - if d.pos >= total { - break // We're finished queuing chunks - } - - // Queue the next range of bytes to read. - ch <- dlchunk{w: d.w, start: d.pos, size: d.cfg.PartSize} - d.pos += d.cfg.PartSize - } - - // Wait for completion - close(ch) - d.wg.Wait() - } else { - // Checking if we read anything new - for d.err == nil { - d.getChunk() - } - - // We expect a 416 error letting us know we are done downloading the - // total bytes. Since we do not know the content's length, this will - // keep grabbing chunks of data until the range of bytes specified in - // the request is out of range of the content. Once, this happens, a - // 416 should occur. - e, ok := d.err.(awserr.RequestFailure) - if ok && e.StatusCode() == http.StatusRequestedRangeNotSatisfiable { - d.err = nil - } - } - - // Return error - return d.written, d.err -} - -// downloadPart is an individual goroutine worker reading from the ch channel -// and performing a GetObject request on the data with a given byte range. -// -// If this is the first worker, this operation also resolves the total number -// of bytes to be read so that the worker manager knows when it is finished. -func (d *downloader) downloadPart(ch chan dlchunk) { - defer d.wg.Done() - for { - chunk, ok := <-ch - if !ok { - break - } - if d.getErr() != nil { - // Drain the channel if there is an error, to prevent deadlocking - // of download producer. - continue - } - - if err := d.downloadChunk(chunk); err != nil { - d.setErr(err) - } - } -} - -// getChunk grabs a chunk of data from the body. -// Not thread safe. Should only used when grabbing data on a single thread. -func (d *downloader) getChunk() { - if d.getErr() != nil { - return - } - - chunk := dlchunk{w: d.w, start: d.pos, size: d.cfg.PartSize} - d.pos += d.cfg.PartSize - - if err := d.downloadChunk(chunk); err != nil { - d.setErr(err) - } -} - -// downloadRange downloads an Object given the passed in Byte-Range value. -// The chunk used down download the range will be configured for that range. -func (d *downloader) downloadRange(rng string) { - if d.getErr() != nil { - return - } - - chunk := dlchunk{w: d.w, start: d.pos} - // Ranges specified will short circuit the multipart download - chunk.withRange = rng - - if err := d.downloadChunk(chunk); err != nil { - d.setErr(err) - } - - // Update the position based on the amount of data received. - d.pos = d.written -} - -// downloadChunk downloads the chunk from s3 -func (d *downloader) downloadChunk(chunk dlchunk) error { - in := &s3.GetObjectInput{} - awsutil.Copy(in, d.in) - - // Get the next byte range of data - in.Range = aws.String(chunk.ByteRange()) - - var n int64 - var err error - for retry := 0; retry <= d.partBodyMaxRetries; retry++ { - var resp *s3.GetObjectOutput - resp, err = d.cfg.S3.GetObjectWithContext(d.ctx, in, d.cfg.RequestOptions...) - if err != nil { - return err - } - d.setTotalBytes(resp) // Set total if not yet set. - - n, err = io.Copy(&chunk, resp.Body) - resp.Body.Close() - if err == nil { - break - } - - chunk.cur = 0 - logMessage(d.cfg.S3, aws.LogDebugWithRequestRetries, - fmt.Sprintf("DEBUG: object part body download interrupted %s, err, %v, retrying attempt %d", - aws.StringValue(in.Key), err, retry)) - } - - d.incrWritten(n) - - return err -} - -func logMessage(svc s3iface.S3API, level aws.LogLevelType, msg string) { - s, ok := svc.(*s3.S3) - if !ok { - return - } - - if s.Config.Logger == nil { - return - } - - if s.Config.LogLevel.Matches(level) { - s.Config.Logger.Log(msg) - } -} - -// getTotalBytes is a thread-safe getter for retrieving the total byte status. -func (d *downloader) getTotalBytes() int64 { - d.m.Lock() - defer d.m.Unlock() - - return d.totalBytes -} - -// setTotalBytes is a thread-safe setter for setting the total byte status. -// Will extract the object's total bytes from the Content-Range if the file -// will be chunked, or Content-Length. Content-Length is used when the response -// does not include a Content-Range. Meaning the object was not chunked. This -// occurs when the full file fits within the PartSize directive. -func (d *downloader) setTotalBytes(resp *s3.GetObjectOutput) { - d.m.Lock() - defer d.m.Unlock() - - if d.totalBytes >= 0 { - return - } - - if resp.ContentRange == nil { - // ContentRange is nil when the full file contents is provided, and - // is not chunked. Use ContentLength instead. - if resp.ContentLength != nil { - d.totalBytes = *resp.ContentLength - return - } - } else { - parts := strings.Split(*resp.ContentRange, "/") - - total := int64(-1) - var err error - // Checking for whether or not a numbered total exists - // If one does not exist, we will assume the total to be -1, undefined, - // and sequentially download each chunk until hitting a 416 error - totalStr := parts[len(parts)-1] - if totalStr != "*" { - total, err = strconv.ParseInt(totalStr, 10, 64) - if err != nil { - d.err = err - return - } - } - - d.totalBytes = total - } -} - -func (d *downloader) incrWritten(n int64) { - d.m.Lock() - defer d.m.Unlock() - - d.written += n -} - -// getErr is a thread-safe getter for the error object -func (d *downloader) getErr() error { - d.m.Lock() - defer d.m.Unlock() - - return d.err -} - -// setErr is a thread-safe setter for the error object -func (d *downloader) setErr(e error) { - d.m.Lock() - defer d.m.Unlock() - - d.err = e -} - -// dlchunk represents a single chunk of data to write by the worker routine. -// This structure also implements an io.SectionReader style interface for -// io.WriterAt, effectively making it an io.SectionWriter (which does not -// exist). -type dlchunk struct { - w io.WriterAt - start int64 - size int64 - cur int64 - - // specifies the byte range the chunk should be downloaded with. - withRange string -} - -// Write wraps io.WriterAt for the dlchunk, writing from the dlchunk's start -// position to its end (or EOF). -// -// If a range is specified on the dlchunk the size will be ignored when writing. -// as the total size may not of be known ahead of time. -func (c *dlchunk) Write(p []byte) (n int, err error) { - if c.cur >= c.size && len(c.withRange) == 0 { - return 0, io.EOF - } - - n, err = c.w.WriteAt(p, c.start+c.cur) - c.cur += int64(n) - - return -} - -// ByteRange returns a HTTP Byte-Range header value that should be used by the -// client to request the chunk's range. -func (c *dlchunk) ByteRange() string { - if len(c.withRange) != 0 { - return c.withRange - } - - return fmt.Sprintf("bytes=%d-%d", c.start, c.start+c.size-1) -} diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go deleted file mode 100644 index 2cebf6180e..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload.go +++ /dev/null @@ -1,727 +0,0 @@ -package s3manager - -import ( - "bytes" - "fmt" - "io" - "sort" - "sync" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/awsutil" - "github.com/aws/aws-sdk-go/aws/client" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" -) - -// MaxUploadParts is the maximum allowed number of parts in a multi-part upload -// on Amazon S3. -const MaxUploadParts = 10000 - -// MinUploadPartSize is the minimum allowed part size when uploading a part to -// Amazon S3. -const MinUploadPartSize int64 = 1024 * 1024 * 5 - -// DefaultUploadPartSize is the default part size to buffer chunks of a -// payload into. -const DefaultUploadPartSize = MinUploadPartSize - -// DefaultUploadConcurrency is the default number of goroutines to spin up when -// using Upload(). -const DefaultUploadConcurrency = 5 - -// A MultiUploadFailure wraps a failed S3 multipart upload. An error returned -// will satisfy this interface when a multi part upload failed to upload all -// chucks to S3. In the case of a failure the UploadID is needed to operate on -// the chunks, if any, which were uploaded. -// -// Example: -// -// u := s3manager.NewUploader(opts) -// output, err := u.upload(input) -// if err != nil { -// if multierr, ok := err.(s3manager.MultiUploadFailure); ok { -// // Process error and its associated uploadID -// fmt.Println("Error:", multierr.Code(), multierr.Message(), multierr.UploadID()) -// } else { -// // Process error generically -// fmt.Println("Error:", err.Error()) -// } -// } -// -type MultiUploadFailure interface { - awserr.Error - - // Returns the upload id for the S3 multipart upload that failed. - UploadID() string -} - -// So that the Error interface type can be included as an anonymous field -// in the multiUploadError struct and not conflict with the error.Error() method. -type awsError awserr.Error - -// A multiUploadError wraps the upload ID of a failed s3 multipart upload. -// Composed of BaseError for code, message, and original error -// -// Should be used for an error that occurred failing a S3 multipart upload, -// and a upload ID is available. If an uploadID is not available a more relevant -type multiUploadError struct { - awsError - - // ID for multipart upload which failed. - uploadID string -} - -// Error returns the string representation of the error. -// -// See apierr.BaseError ErrorWithExtra for output format -// -// Satisfies the error interface. -func (m multiUploadError) Error() string { - extra := fmt.Sprintf("upload id: %s", m.uploadID) - return awserr.SprintError(m.Code(), m.Message(), extra, m.OrigErr()) -} - -// String returns the string representation of the error. -// Alias for Error to satisfy the stringer interface. -func (m multiUploadError) String() string { - return m.Error() -} - -// UploadID returns the id of the S3 upload which failed. -func (m multiUploadError) UploadID() string { - return m.uploadID -} - -// UploadOutput represents a response from the Upload() call. -type UploadOutput struct { - // The URL where the object was uploaded to. - Location string - - // The version of the object that was uploaded. Will only be populated if - // the S3 Bucket is versioned. If the bucket is not versioned this field - // will not be set. - VersionID *string - - // The ID for a multipart upload to S3. In the case of an error the error - // can be cast to the MultiUploadFailure interface to extract the upload ID. - UploadID string -} - -// WithUploaderRequestOptions appends to the Uploader's API request options. -func WithUploaderRequestOptions(opts ...request.Option) func(*Uploader) { - return func(u *Uploader) { - u.RequestOptions = append(u.RequestOptions, opts...) - } -} - -// The Uploader structure that calls Upload(). It is safe to call Upload() -// on this structure for multiple objects and across concurrent goroutines. -// Mutating the Uploader's properties is not safe to be done concurrently. -type Uploader struct { - // The buffer size (in bytes) to use when buffering data into chunks and - // sending them as parts to S3. The minimum allowed part size is 5MB, and - // if this value is set to zero, the DefaultUploadPartSize value will be used. - PartSize int64 - - // The number of goroutines to spin up in parallel per call to Upload when - // sending parts. If this is set to zero, the DefaultUploadConcurrency value - // will be used. - // - // The concurrency pool is not shared between calls to Upload. - Concurrency int - - // Setting this value to true will cause the SDK to avoid calling - // AbortMultipartUpload on a failure, leaving all successfully uploaded - // parts on S3 for manual recovery. - // - // Note that storing parts of an incomplete multipart upload counts towards - // space usage on S3 and will add additional costs if not cleaned up. - LeavePartsOnError bool - - // MaxUploadParts is the max number of parts which will be uploaded to S3. - // Will be used to calculate the partsize of the object to be uploaded. - // E.g: 5GB file, with MaxUploadParts set to 100, will upload the file - // as 100, 50MB parts. With a limited of s3.MaxUploadParts (10,000 parts). - // - // MaxUploadParts must not be used to limit the total number of bytes uploaded. - // Use a type like to io.LimitReader (https://golang.org/pkg/io/#LimitedReader) - // instead. An io.LimitReader is helpful when uploading an unbounded reader - // to S3, and you know its maximum size. Otherwise the reader's io.EOF returned - // error must be used to signal end of stream. - // - // Defaults to package const's MaxUploadParts value. - MaxUploadParts int - - // The client to use when uploading to S3. - S3 s3iface.S3API - - // List of request options that will be passed down to individual API - // operation requests made by the uploader. - RequestOptions []request.Option -} - -// NewUploader creates a new Uploader instance to upload objects to S3. Pass In -// additional functional options to customize the uploader's behavior. Requires a -// client.ConfigProvider in order to create a S3 service client. The session.Session -// satisfies the client.ConfigProvider interface. -// -// Example: -// // The session the S3 Uploader will use -// sess := session.Must(session.NewSession()) -// -// // Create an uploader with the session and default options -// uploader := s3manager.NewUploader(sess) -// -// // Create an uploader with the session and custom options -// uploader := s3manager.NewUploader(session, func(u *s3manager.Uploader) { -// u.PartSize = 64 * 1024 * 1024 // 64MB per part -// }) -func NewUploader(c client.ConfigProvider, options ...func(*Uploader)) *Uploader { - u := &Uploader{ - S3: s3.New(c), - PartSize: DefaultUploadPartSize, - Concurrency: DefaultUploadConcurrency, - LeavePartsOnError: false, - MaxUploadParts: MaxUploadParts, - } - - for _, option := range options { - option(u) - } - - return u -} - -// NewUploaderWithClient creates a new Uploader instance to upload objects to S3. Pass in -// additional functional options to customize the uploader's behavior. Requires -// a S3 service client to make S3 API calls. -// -// Example: -// // The session the S3 Uploader will use -// sess := session.Must(session.NewSession()) -// -// // S3 service client the Upload manager will use. -// s3Svc := s3.New(sess) -// -// // Create an uploader with S3 client and default options -// uploader := s3manager.NewUploaderWithClient(s3Svc) -// -// // Create an uploader with S3 client and custom options -// uploader := s3manager.NewUploaderWithClient(s3Svc, func(u *s3manager.Uploader) { -// u.PartSize = 64 * 1024 * 1024 // 64MB per part -// }) -func NewUploaderWithClient(svc s3iface.S3API, options ...func(*Uploader)) *Uploader { - u := &Uploader{ - S3: svc, - PartSize: DefaultUploadPartSize, - Concurrency: DefaultUploadConcurrency, - LeavePartsOnError: false, - MaxUploadParts: MaxUploadParts, - } - - for _, option := range options { - option(u) - } - - return u -} - -// Upload uploads an object to S3, intelligently buffering large files into -// smaller chunks and sending them in parallel across multiple goroutines. You -// can configure the buffer size and concurrency through the Uploader's parameters. -// -// Additional functional options can be provided to configure the individual -// upload. These options are copies of the Uploader instance Upload is called from. -// Modifying the options will not impact the original Uploader instance. -// -// Use the WithUploaderRequestOptions helper function to pass in request -// options that will be applied to all API operations made with this uploader. -// -// It is safe to call this method concurrently across goroutines. -// -// Example: -// // Upload input parameters -// upParams := &s3manager.UploadInput{ -// Bucket: &bucketName, -// Key: &keyName, -// Body: file, -// } -// -// // Perform an upload. -// result, err := uploader.Upload(upParams) -// -// // Perform upload with options different than the those in the Uploader. -// result, err := uploader.Upload(upParams, func(u *s3manager.Uploader) { -// u.PartSize = 10 * 1024 * 1024 // 10MB part size -// u.LeavePartsOnError = true // Don't delete the parts if the upload fails. -// }) -func (u Uploader) Upload(input *UploadInput, options ...func(*Uploader)) (*UploadOutput, error) { - return u.UploadWithContext(aws.BackgroundContext(), input, options...) -} - -// UploadWithContext uploads an object to S3, intelligently buffering large -// files into smaller chunks and sending them in parallel across multiple -// goroutines. You can configure the buffer size and concurrency through the -// Uploader's parameters. -// -// UploadWithContext is the same as Upload with the additional support for -// Context input parameters. The Context must not be nil. A nil Context will -// cause a panic. Use the context to add deadlining, timeouts, etc. The -// UploadWithContext may create sub-contexts for individual underlying requests. -// -// Additional functional options can be provided to configure the individual -// upload. These options are copies of the Uploader instance Upload is called from. -// Modifying the options will not impact the original Uploader instance. -// -// Use the WithUploaderRequestOptions helper function to pass in request -// options that will be applied to all API operations made with this uploader. -// -// It is safe to call this method concurrently across goroutines. -func (u Uploader) UploadWithContext(ctx aws.Context, input *UploadInput, opts ...func(*Uploader)) (*UploadOutput, error) { - i := uploader{in: input, cfg: u, ctx: ctx} - - for _, opt := range opts { - opt(&i.cfg) - } - i.cfg.RequestOptions = append(i.cfg.RequestOptions, request.WithAppendUserAgent("S3Manager")) - - return i.upload() -} - -// UploadWithIterator will upload a batched amount of objects to S3. This operation uses -// the iterator pattern to know which object to upload next. Since this is an interface this -// allows for custom defined functionality. -// -// Example: -// svc:= s3manager.NewUploader(sess) -// -// objects := []BatchUploadObject{ -// { -// Object: &s3manager.UploadInput { -// Key: aws.String("key"), -// Bucket: aws.String("bucket"), -// }, -// }, -// } -// -// iter := &s3manager.UploadObjectsIterator{Objects: objects} -// if err := svc.UploadWithIterator(aws.BackgroundContext(), iter); err != nil { -// return err -// } -func (u Uploader) UploadWithIterator(ctx aws.Context, iter BatchUploadIterator, opts ...func(*Uploader)) error { - var errs []Error - for iter.Next() { - object := iter.UploadObject() - if _, err := u.UploadWithContext(ctx, object.Object, opts...); err != nil { - s3Err := Error{ - OrigErr: err, - Bucket: object.Object.Bucket, - Key: object.Object.Key, - } - - errs = append(errs, s3Err) - } - - if object.After == nil { - continue - } - - if err := object.After(); err != nil { - s3Err := Error{ - OrigErr: err, - Bucket: object.Object.Bucket, - Key: object.Object.Key, - } - - errs = append(errs, s3Err) - } - } - - if len(errs) > 0 { - return NewBatchError("BatchedUploadIncomplete", "some objects have failed to upload.", errs) - } - return nil -} - -// internal structure to manage an upload to S3. -type uploader struct { - ctx aws.Context - cfg Uploader - - in *UploadInput - - readerPos int64 // current reader position - totalSize int64 // set to -1 if the size is not known - - bufferPool sync.Pool -} - -// internal logic for deciding whether to upload a single part or use a -// multipart upload. -func (u *uploader) upload() (*UploadOutput, error) { - if err := u.init(); err != nil { - return nil, awserr.New("ReadRequestBody", "unable to initialize upload", err) - } - - if u.cfg.PartSize < MinUploadPartSize { - msg := fmt.Sprintf("part size must be at least %d bytes", MinUploadPartSize) - return nil, awserr.New("ConfigError", msg, nil) - } - - // Do one read to determine if we have more than one part - reader, _, part, err := u.nextReader() - if err == io.EOF { // single part - return u.singlePart(reader) - } else if err != nil { - return nil, awserr.New("ReadRequestBody", "read upload data failed", err) - } - - mu := multiuploader{uploader: u} - return mu.upload(reader, part) -} - -// init will initialize all default options. -func (u *uploader) init() error { - if u.cfg.Concurrency == 0 { - u.cfg.Concurrency = DefaultUploadConcurrency - } - if u.cfg.PartSize == 0 { - u.cfg.PartSize = DefaultUploadPartSize - } - if u.cfg.MaxUploadParts == 0 { - u.cfg.MaxUploadParts = MaxUploadParts - } - - u.bufferPool = sync.Pool{ - New: func() interface{} { return make([]byte, u.cfg.PartSize) }, - } - - // Try to get the total size for some optimizations - return u.initSize() -} - -// initSize tries to detect the total stream size, setting u.totalSize. If -// the size is not known, totalSize is set to -1. -func (u *uploader) initSize() error { - u.totalSize = -1 - - switch r := u.in.Body.(type) { - case io.Seeker: - n, err := aws.SeekerLen(r) - if err != nil { - return err - } - u.totalSize = n - - // Try to adjust partSize if it is too small and account for - // integer division truncation. - if u.totalSize/u.cfg.PartSize >= int64(u.cfg.MaxUploadParts) { - // Add one to the part size to account for remainders - // during the size calculation. e.g odd number of bytes. - u.cfg.PartSize = (u.totalSize / int64(u.cfg.MaxUploadParts)) + 1 - } - } - - return nil -} - -// nextReader returns a seekable reader representing the next packet of data. -// This operation increases the shared u.readerPos counter, but note that it -// does not need to be wrapped in a mutex because nextReader is only called -// from the main thread. -func (u *uploader) nextReader() (io.ReadSeeker, int, []byte, error) { - type readerAtSeeker interface { - io.ReaderAt - io.ReadSeeker - } - switch r := u.in.Body.(type) { - case readerAtSeeker: - var err error - - n := u.cfg.PartSize - if u.totalSize >= 0 { - bytesLeft := u.totalSize - u.readerPos - - if bytesLeft <= u.cfg.PartSize { - err = io.EOF - n = bytesLeft - } - } - - reader := io.NewSectionReader(r, u.readerPos, n) - u.readerPos += n - - return reader, int(n), nil, err - - default: - part := u.bufferPool.Get().([]byte) - n, err := readFillBuf(r, part) - u.readerPos += int64(n) - - return bytes.NewReader(part[0:n]), n, part, err - } -} - -func readFillBuf(r io.Reader, b []byte) (offset int, err error) { - for offset < len(b) && err == nil { - var n int - n, err = r.Read(b[offset:]) - offset += n - } - - return offset, err -} - -// singlePart contains upload logic for uploading a single chunk via -// a regular PutObject request. Multipart requests require at least two -// parts, or at least 5MB of data. -func (u *uploader) singlePart(buf io.ReadSeeker) (*UploadOutput, error) { - params := &s3.PutObjectInput{} - awsutil.Copy(params, u.in) - params.Body = buf - - // Need to use request form because URL generated in request is - // used in return. - req, out := u.cfg.S3.PutObjectRequest(params) - req.SetContext(u.ctx) - req.ApplyOptions(u.cfg.RequestOptions...) - if err := req.Send(); err != nil { - return nil, err - } - - url := req.HTTPRequest.URL.String() - return &UploadOutput{ - Location: url, - VersionID: out.VersionId, - }, nil -} - -// internal structure to manage a specific multipart upload to S3. -type multiuploader struct { - *uploader - wg sync.WaitGroup - m sync.Mutex - err error - uploadID string - parts completedParts -} - -// keeps track of a single chunk of data being sent to S3. -type chunk struct { - buf io.ReadSeeker - part []byte - num int64 -} - -// completedParts is a wrapper to make parts sortable by their part number, -// since S3 required this list to be sent in sorted order. -type completedParts []*s3.CompletedPart - -func (a completedParts) Len() int { return len(a) } -func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a completedParts) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber } - -// upload will perform a multipart upload using the firstBuf buffer containing -// the first chunk of data. -func (u *multiuploader) upload(firstBuf io.ReadSeeker, firstPart []byte) (*UploadOutput, error) { - params := &s3.CreateMultipartUploadInput{} - awsutil.Copy(params, u.in) - - // Create the multipart - resp, err := u.cfg.S3.CreateMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...) - if err != nil { - return nil, err - } - u.uploadID = *resp.UploadId - - // Create the workers - ch := make(chan chunk, u.cfg.Concurrency) - for i := 0; i < u.cfg.Concurrency; i++ { - u.wg.Add(1) - go u.readChunk(ch) - } - - // Send part 1 to the workers - var num int64 = 1 - ch <- chunk{buf: firstBuf, part: firstPart, num: num} - - // Read and queue the rest of the parts - for u.geterr() == nil && err == nil { - var reader io.ReadSeeker - var nextChunkLen int - var part []byte - reader, nextChunkLen, part, err = u.nextReader() - - if err != nil && err != io.EOF { - u.seterr(awserr.New( - "ReadRequestBody", - "read multipart upload data failed", - err)) - break - } - - if nextChunkLen == 0 { - // No need to upload empty part, if file was empty to start - // with empty single part would of been created and never - // started multipart upload. - break - } - - num++ - // This upload exceeded maximum number of supported parts, error now. - if num > int64(u.cfg.MaxUploadParts) || num > int64(MaxUploadParts) { - var msg string - if num > int64(u.cfg.MaxUploadParts) { - msg = fmt.Sprintf("exceeded total allowed configured MaxUploadParts (%d). Adjust PartSize to fit in this limit", - u.cfg.MaxUploadParts) - } else { - msg = fmt.Sprintf("exceeded total allowed S3 limit MaxUploadParts (%d). Adjust PartSize to fit in this limit", - MaxUploadParts) - } - u.seterr(awserr.New("TotalPartsExceeded", msg, nil)) - break - } - - ch <- chunk{buf: reader, part: part, num: num} - } - - // Close the channel, wait for workers, and complete upload - close(ch) - u.wg.Wait() - complete := u.complete() - - if err := u.geterr(); err != nil { - return nil, &multiUploadError{ - awsError: awserr.New( - "MultipartUpload", - "upload multipart failed", - err), - uploadID: u.uploadID, - } - } - - // Create a presigned URL of the S3 Get Object in order to have parity with - // single part upload. - getReq, _ := u.cfg.S3.GetObjectRequest(&s3.GetObjectInput{ - Bucket: u.in.Bucket, - Key: u.in.Key, - }) - getReq.Config.Credentials = credentials.AnonymousCredentials - uploadLocation, _, _ := getReq.PresignRequest(1) - - return &UploadOutput{ - Location: uploadLocation, - VersionID: complete.VersionId, - UploadID: u.uploadID, - }, nil -} - -// readChunk runs in worker goroutines to pull chunks off of the ch channel -// and send() them as UploadPart requests. -func (u *multiuploader) readChunk(ch chan chunk) { - defer u.wg.Done() - for { - data, ok := <-ch - - if !ok { - break - } - - if u.geterr() == nil { - if err := u.send(data); err != nil { - u.seterr(err) - } - } - } -} - -// send performs an UploadPart request and keeps track of the completed -// part information. -func (u *multiuploader) send(c chunk) error { - params := &s3.UploadPartInput{ - Bucket: u.in.Bucket, - Key: u.in.Key, - Body: c.buf, - UploadId: &u.uploadID, - SSECustomerAlgorithm: u.in.SSECustomerAlgorithm, - SSECustomerKey: u.in.SSECustomerKey, - PartNumber: &c.num, - } - resp, err := u.cfg.S3.UploadPartWithContext(u.ctx, params, u.cfg.RequestOptions...) - // put the byte array back into the pool to conserve memory - u.bufferPool.Put(c.part) - if err != nil { - return err - } - - n := c.num - completed := &s3.CompletedPart{ETag: resp.ETag, PartNumber: &n} - - u.m.Lock() - u.parts = append(u.parts, completed) - u.m.Unlock() - - return nil -} - -// geterr is a thread-safe getter for the error object -func (u *multiuploader) geterr() error { - u.m.Lock() - defer u.m.Unlock() - - return u.err -} - -// seterr is a thread-safe setter for the error object -func (u *multiuploader) seterr(e error) { - u.m.Lock() - defer u.m.Unlock() - - u.err = e -} - -// fail will abort the multipart unless LeavePartsOnError is set to true. -func (u *multiuploader) fail() { - if u.cfg.LeavePartsOnError { - return - } - - params := &s3.AbortMultipartUploadInput{ - Bucket: u.in.Bucket, - Key: u.in.Key, - UploadId: &u.uploadID, - } - _, err := u.cfg.S3.AbortMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...) - if err != nil { - logMessage(u.cfg.S3, aws.LogDebug, fmt.Sprintf("failed to abort multipart upload, %v", err)) - } -} - -// complete successfully completes a multipart upload and returns the response. -func (u *multiuploader) complete() *s3.CompleteMultipartUploadOutput { - if u.geterr() != nil { - u.fail() - return nil - } - - // Parts must be sorted in PartNumber order. - sort.Sort(u.parts) - - params := &s3.CompleteMultipartUploadInput{ - Bucket: u.in.Bucket, - Key: u.in.Key, - UploadId: &u.uploadID, - MultipartUpload: &s3.CompletedMultipartUpload{Parts: u.parts}, - } - resp, err := u.cfg.S3.CompleteMultipartUploadWithContext(u.ctx, params, u.cfg.RequestOptions...) - if err != nil { - u.seterr(err) - u.fail() - } - - return resp -} diff --git a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go b/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go deleted file mode 100644 index 221c37f6ea..0000000000 --- a/vendor/github.com/aws/aws-sdk-go/service/s3/s3manager/upload_input.go +++ /dev/null @@ -1,129 +0,0 @@ -// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. - -package s3manager - -import ( - "io" - "time" -) - -// UploadInput provides the input parameters for uploading a stream or buffer -// to an object in an Amazon S3 bucket. This type is similar to the s3 -// package's PutObjectInput with the exception that the Body member is an -// io.Reader instead of an io.ReadSeeker. -type UploadInput struct { - _ struct{} `type:"structure" payload:"Body"` - - // The canned ACL to apply to the object. - ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"` - - // The readable body payload to send to S3. - Body io.Reader - - // Name of the bucket to which the PUT operation was initiated. - // - // Bucket is a required field - Bucket *string `location:"uri" locationName:"Bucket" type:"string" required:"true"` - - // Specifies caching behavior along the request/reply chain. - CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"` - - // Specifies presentational information for the object. - ContentDisposition *string `location:"header" locationName:"Content-Disposition" type:"string"` - - // Specifies what content encodings have been applied to the object and thus - // what decoding mechanisms must be applied to obtain the media-type referenced - // by the Content-Type header field. - ContentEncoding *string `location:"header" locationName:"Content-Encoding" type:"string"` - - // The language the content is in. - ContentLanguage *string `location:"header" locationName:"Content-Language" type:"string"` - - // The base64-encoded 128-bit MD5 digest of the part data. This parameter is - // auto-populated when using the command from the CLI. This parameted is required - // if object lock parameters are specified. - ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"` - - // A standard MIME type describing the format of the object data. - ContentType *string `location:"header" locationName:"Content-Type" type:"string"` - - // The date and time at which the object is no longer cacheable. - Expires *time.Time `location:"header" locationName:"Expires" type:"timestamp"` - - // Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object. - GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"` - - // Allows grantee to read the object data and its metadata. - GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"` - - // Allows grantee to read the object ACL. - GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"` - - // Allows grantee to write the ACL for the applicable object. - GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"` - - // Object key for which the PUT operation was initiated. - // - // Key is a required field - Key *string `location:"uri" locationName:"Key" min:"1" type:"string" required:"true"` - - // A map of metadata to store with the object in S3. - Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"` - - // The Legal Hold status that you want to apply to the specified object. - ObjectLockLegalHoldStatus *string `location:"header" locationName:"x-amz-object-lock-legal-hold" type:"string" enum:"ObjectLockLegalHoldStatus"` - - // The object lock mode that you want to apply to this object. - ObjectLockMode *string `location:"header" locationName:"x-amz-object-lock-mode" type:"string" enum:"ObjectLockMode"` - - // The date and time when you want this object's object lock to expire. - ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"` - - // Confirms that the requester knows that she or he will be charged for the - // request. Bucket owners need not specify this parameter in their requests. - // Documentation on downloading objects from requester pays buckets can be found - // at http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html - RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"` - - // Specifies the algorithm to use to when encrypting the object (e.g., AES256). - SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"` - - // Specifies the customer-provided encryption key for Amazon S3 to use in encrypting - // data. This value is used to store the object and then it is discarded; Amazon - // does not store the encryption key. The key must be appropriate for use with - // the algorithm specified in the x-amz-server-side​-encryption​-customer-algorithm - // header. - SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"` - - // Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. - // Amazon S3 uses this header for a message integrity check to ensure the encryption - // key was transmitted without error. - SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"` - - // Specifies the AWS KMS Encryption Context to use for object encryption. The - // value of this header is a base64-encoded UTF-8 string holding JSON with the - // encryption context key-value pairs. - SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"` - - // Specifies the AWS KMS key ID to use for object encryption. All GET and PUT - // requests for an object protected by AWS KMS will fail if not made via SSL - // or using SigV4. Documentation on configuring any of the officially supported - // AWS SDKs and CLI can be found at http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version - SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"` - - // The Server-side encryption algorithm used when storing this object in S3 - // (e.g., AES256, aws:kms). - ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"` - - // The type of storage to use for the object. Defaults to 'STANDARD'. - StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"` - - // The tag-set for the object. The tag-set must be encoded as URL Query parameters. - // (For example, "Key1=Value1") - Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"` - - // If the bucket is configured as a website, redirects requests for this object - // to another object in the same bucket or to an external URL. Amazon S3 stores - // the value of this header in the object metadata. - WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"` -} diff --git a/vendor/modules.txt b/vendor/modules.txt index 1cd7ecee94..c078cbba2e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -110,7 +110,6 @@ github.com/aws/aws-sdk-go/service/ec2 github.com/aws/aws-sdk-go/service/elbv2 github.com/aws/aws-sdk-go/service/iam github.com/aws/aws-sdk-go/service/s3 -github.com/aws/aws-sdk-go/service/s3/s3manager github.com/aws/aws-sdk-go/aws/endpoints github.com/aws/aws-sdk-go/internal/sdkio github.com/aws/aws-sdk-go/internal/sdkrand @@ -132,7 +131,6 @@ github.com/aws/aws-sdk-go/internal/s3err github.com/aws/aws-sdk-go/private/protocol/eventstream github.com/aws/aws-sdk-go/private/protocol/eventstream/eventstreamapi github.com/aws/aws-sdk-go/private/protocol/restxml -github.com/aws/aws-sdk-go/service/s3/s3iface github.com/aws/aws-sdk-go/service/sts github.com/aws/aws-sdk-go/service/sts/stsiface github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds @@ -783,7 +781,7 @@ yunion.io/x/pkg/util/prometheus yunion.io/x/pkg/prettytable yunion.io/x/pkg/util/runtime yunion.io/x/pkg/util/clock -# yunion.io/x/s3cli v0.0.0-20190806141932-17a2c973cbc5 +# yunion.io/x/s3cli v0.0.0-20190812030153-814f1e636a0a yunion.io/x/s3cli # yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda yunion.io/x/sqlchemy diff --git a/vendor/yunion.io/x/s3cli/README.md b/vendor/yunion.io/x/s3cli/README.md index 8f9731c322..824ec1b9c5 100644 --- a/vendor/yunion.io/x/s3cli/README.md +++ b/vendor/yunion.io/x/s3cli/README.md @@ -1,3 +1,3 @@ -# A fork of minio GO s3 client +# A Yunion fork of minio GO s3 client Please refer to https://github.com/minio/minio-go diff --git a/vendor/yunion.io/x/s3cli/api-compose-object.go b/vendor/yunion.io/x/s3cli/api-compose-object.go index 9c337ff56c..395d211f28 100644 --- a/vendor/yunion.io/x/s3cli/api-compose-object.go +++ b/vendor/yunion.io/x/s3cli/api-compose-object.go @@ -491,8 +491,8 @@ func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo } // 4. Make final complete-multipart request. - _, err = c.completeMultipartUpload(ctx, dst.bucket, dst.object, uploadID, - completeMultipartUpload{Parts: objParts}) + _, err = c.CompleteMultipartUpload(ctx, dst.bucket, dst.object, uploadID, + CompleteMultipartUpload{Parts: objParts}) if err != nil { return err } diff --git a/vendor/yunion.io/x/s3cli/api-put-object-common.go b/vendor/yunion.io/x/s3cli/api-put-object-common.go index 67a1b23886..73a6a20f7c 100644 --- a/vendor/yunion.io/x/s3cli/api-put-object-common.go +++ b/vendor/yunion.io/x/s3cli/api-put-object-common.go @@ -130,7 +130,7 @@ func (c Client) newUploadID(ctx context.Context, bucketName, objectName string, } // Initiate multipart upload for an object. - initMultipartUploadResult, err := c.initiateMultipartUpload(ctx, bucketName, objectName, opts) + initMultipartUploadResult, err := c.InitiateMultipartUpload(ctx, bucketName, objectName, opts) if err != nil { return "", err } diff --git a/vendor/yunion.io/x/s3cli/api-put-object-multipart.go b/vendor/yunion.io/x/s3cli/api-put-object-multipart.go index 0310c5641c..395d0f17ee 100644 --- a/vendor/yunion.io/x/s3cli/api-put-object-multipart.go +++ b/vendor/yunion.io/x/s3cli/api-put-object-multipart.go @@ -70,7 +70,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje var totalUploadedSize int64 // Complete multipart upload. - var complMultipartUpload completeMultipartUpload + var complMultipartUpload CompleteMultipartUpload // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) @@ -86,7 +86,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje defer func() { if err != nil { - c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + c.AbortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() @@ -138,7 +138,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Proceed to upload the part. var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, + objPart, err = c.UploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, md5Base64, sha256Hex, int64(length), opts.ServerSideEncryption) if err != nil { return totalUploadedSize, err @@ -175,7 +175,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { + if _, err = c.CompleteMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { return totalUploadedSize, err } @@ -184,13 +184,13 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje } // initiateMultipartUpload - Initiates a multipart upload and returns an upload ID. -func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (initiateMultipartUploadResult, error) { +func (c Client) InitiateMultipartUpload(ctx context.Context, bucketName, objectName string, opts PutObjectOptions) (InitiateMultipartUploadResult, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return initiateMultipartUploadResult{}, err + return InitiateMultipartUploadResult{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return initiateMultipartUploadResult{}, err + return InitiateMultipartUploadResult{}, err } // Initialize url queries. @@ -211,24 +211,24 @@ func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectN resp, err := c.executeMethod(ctx, "POST", reqMetadata) defer closeResponse(resp) if err != nil { - return initiateMultipartUploadResult{}, err + return InitiateMultipartUploadResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return initiateMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + return InitiateMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // Decode xml for new multipart upload. - initiateMultipartUploadResult := initiateMultipartUploadResult{} - err = xmlDecoder(resp.Body, &initiateMultipartUploadResult) + InitiateMultipartUploadResult := InitiateMultipartUploadResult{} + err = xmlDecoder(resp.Body, &InitiateMultipartUploadResult) if err != nil { - return initiateMultipartUploadResult, err + return InitiateMultipartUploadResult, err } - return initiateMultipartUploadResult, nil + return InitiateMultipartUploadResult, nil } // uploadPart - Uploads a part in a multipart upload. -func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID string, reader io.Reader, +func (c Client) UploadPart(ctx context.Context, bucketName, objectName, uploadID string, reader io.Reader, partNumber int, md5Base64, sha256Hex string, size int64, sse encrypt.ServerSide) (ObjectPart, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { @@ -300,14 +300,14 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID } // completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts. -func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, - complete completeMultipartUpload) (completeMultipartUploadResult, error) { +func (c Client) CompleteMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string, + complete CompleteMultipartUpload) (CompleteMultipartUploadResult, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { - return completeMultipartUploadResult{}, err + return CompleteMultipartUploadResult{}, err } if err := s3utils.CheckValidObjectName(objectName); err != nil { - return completeMultipartUploadResult{}, err + return CompleteMultipartUploadResult{}, err } // Initialize url queries. @@ -316,7 +316,7 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN // Marshal complete multipart body. completeMultipartUploadBytes, err := xml.Marshal(complete) if err != nil { - return completeMultipartUploadResult{}, err + return CompleteMultipartUploadResult{}, err } // Instantiate all the complete multipart buffer. @@ -334,11 +334,11 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN resp, err := c.executeMethod(ctx, "POST", reqMetadata) defer closeResponse(resp) if err != nil { - return completeMultipartUploadResult{}, err + return CompleteMultipartUploadResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return completeMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + return CompleteMultipartUploadResult{}, httpRespToErrorResponse(resp, bucketName, objectName) } } @@ -346,10 +346,10 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN var b []byte b, err = ioutil.ReadAll(resp.Body) if err != nil { - return completeMultipartUploadResult{}, err + return CompleteMultipartUploadResult{}, err } // Decode completed multipart upload response on success. - completeMultipartUploadResult := completeMultipartUploadResult{} + completeMultipartUploadResult := CompleteMultipartUploadResult{} err = xmlDecoder(bytes.NewReader(b), &completeMultipartUploadResult) if err != nil { // xml parsing failure due to presence an ill-formed xml fragment diff --git a/vendor/yunion.io/x/s3cli/api-put-object-streaming.go b/vendor/yunion.io/x/s3cli/api-put-object-streaming.go index c0ef14018f..8bb0c3afad 100644 --- a/vendor/yunion.io/x/s3cli/api-put-object-streaming.go +++ b/vendor/yunion.io/x/s3cli/api-put-object-streaming.go @@ -114,7 +114,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // to relinquish storage space. defer func() { if err != nil { - c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + c.AbortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() @@ -122,7 +122,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa var totalUploadedSize int64 // Complete multipart upload. - var complMultipartUpload completeMultipartUpload + var complMultipartUpload CompleteMultipartUpload // Declare a channel that sends the next part number to be uploaded. // Buffered to 10000 because thats the maximum number of parts allowed @@ -165,7 +165,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Proceed to upload the part. var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, + objPart, err = c.UploadPart(ctx, bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, "", "", partSize, opts.ServerSideEncryption) if err != nil { @@ -220,7 +220,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + _, err = c.CompleteMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) if err != nil { return totalUploadedSize, err } @@ -256,7 +256,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa // storage space. defer func() { if err != nil { - c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + c.AbortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() @@ -278,7 +278,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa partSize = lastPartSize } var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, + objPart, err = c.UploadPart(ctx, bucketName, objectName, uploadID, io.LimitReader(hookReader, partSize), partNumber, "", "", partSize, opts.ServerSideEncryption) if err != nil { @@ -300,7 +300,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa } // Complete multipart upload. - var complMultipartUpload completeMultipartUpload + var complMultipartUpload CompleteMultipartUpload // Loop over total uploaded parts to save them in // Parts array before completing the multipart request. @@ -317,7 +317,7 @@ func (c Client) putObjectMultipartStreamNoChecksum(ctx context.Context, bucketNa // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) + _, err = c.CompleteMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload) if err != nil { return totalUploadedSize, err } @@ -359,7 +359,7 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName // This function does not calculate sha256 and md5sum for payload. // Execute put object. - st, err := c.putObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts) + st, err := c.PutObjectDo(ctx, bucketName, objectName, readSeeker, "", "", size, opts) if err != nil { return 0, err } @@ -371,7 +371,7 @@ func (c Client) putObjectNoChecksum(ctx context.Context, bucketName, objectName // putObjectDo - executes the put object http operation. // NOTE: You must have WRITE permissions on a bucket to add an object to it. -func (c Client) putObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) { +func (c Client) PutObjectDo(ctx context.Context, bucketName, objectName string, reader io.Reader, md5Base64, sha256Hex string, size int64, opts PutObjectOptions) (ObjectInfo, error) { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ObjectInfo{}, err diff --git a/vendor/yunion.io/x/s3cli/api-put-object.go b/vendor/yunion.io/x/s3cli/api-put-object.go index d0bbc032cf..91e2d91882 100644 --- a/vendor/yunion.io/x/s3cli/api-put-object.go +++ b/vendor/yunion.io/x/s3cli/api-put-object.go @@ -186,7 +186,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName var totalUploadedSize int64 // Complete multipart upload. - var complMultipartUpload completeMultipartUpload + var complMultipartUpload CompleteMultipartUpload // Calculate the optimal parts info for a given size. totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize) @@ -201,7 +201,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName defer func() { if err != nil { - c.abortMultipartUpload(ctx, bucketName, objectName, uploadID) + c.AbortMultipartUpload(ctx, bucketName, objectName, uploadID) } }() @@ -229,7 +229,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Proceed to upload the part. var objPart ObjectPart - objPart, err = c.uploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, + objPart, err = c.UploadPart(ctx, bucketName, objectName, uploadID, rd, partNumber, "", "", int64(length), opts.ServerSideEncryption) if err != nil { return totalUploadedSize, err @@ -266,7 +266,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName // Sort all completed parts. sort.Sort(completedParts(complMultipartUpload.Parts)) - if _, err = c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { + if _, err = c.CompleteMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload); err != nil { return totalUploadedSize, err } diff --git a/vendor/yunion.io/x/s3cli/api-remove.go b/vendor/yunion.io/x/s3cli/api-remove.go index 5bb4edc956..c8bc70f821 100644 --- a/vendor/yunion.io/x/s3cli/api-remove.go +++ b/vendor/yunion.io/x/s3cli/api-remove.go @@ -241,7 +241,7 @@ func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { for _, uploadID := range uploadIDs { // abort incomplete multipart upload, based on the upload id passed. - err := c.abortMultipartUpload(context.Background(), bucketName, objectName, uploadID) + err := c.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID) if err != nil { return err } @@ -252,7 +252,7 @@ func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { // abortMultipartUpload aborts a multipart upload for the given // uploadID, all previously uploaded parts are deleted. -func (c Client) abortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error { +func (c Client) AbortMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string) error { // Input validation. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return err diff --git a/vendor/yunion.io/x/s3cli/api-s3-datatypes.go b/vendor/yunion.io/x/s3cli/api-s3-datatypes.go index 7ffadd0abc..5fbcaacc20 100644 --- a/vendor/yunion.io/x/s3cli/api-s3-datatypes.go +++ b/vendor/yunion.io/x/s3cli/api-s3-datatypes.go @@ -171,19 +171,23 @@ type ListObjectPartsResult struct { // initiateMultipartUploadResult container for InitiateMultiPartUpload // response. -type initiateMultipartUploadResult struct { - Bucket string - Key string +type InitiateMultipartUploadResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` + + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` UploadID string `xml:"UploadId"` } // completeMultipartUploadResult container for completed multipart // upload response. -type completeMultipartUploadResult struct { - Location string - Bucket string - Key string - ETag string +type CompleteMultipartUploadResult struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` + + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` } // CompletePart sub container lists individual part numbers and their @@ -192,12 +196,12 @@ type CompletePart struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"` // Part number identifies the part. - PartNumber int - ETag string + PartNumber int `xml:"PartNumber"` + ETag string `xml:"ETag"` } // completeMultipartUpload container for completing multipart upload. -type completeMultipartUpload struct { +type CompleteMultipartUpload struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"` Parts []CompletePart `xml:"Part"` } diff --git a/vendor/yunion.io/x/s3cli/core.go b/vendor/yunion.io/x/s3cli/core.go deleted file mode 100644 index e31b29de1c..0000000000 --- a/vendor/yunion.io/x/s3cli/core.go +++ /dev/null @@ -1,153 +0,0 @@ -/* - * MinIO Go Library for Amazon S3 Compatible Cloud Storage - * Copyright 2015-2017 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package s3cli - -import ( - "context" - "io" - "strings" - - "github.com/minio/minio-go/v6/pkg/encrypt" -) - -// Core - Inherits Client and adds new methods to expose the low level S3 APIs. -type Core struct { - *Client -} - -// NewCore - Returns new initialized a Core client, this CoreClient should be -// only used under special conditions such as need to access lower primitives -// and being able to use them to write your own wrappers. -func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool, debug bool) (*Core, error) { - var s3Client Core - client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure, debug) - if err != nil { - return nil, err - } - s3Client.Client = client - return &s3Client, nil -} - -// ListObjects - List all the objects at a prefix, optionally with marker and delimiter -// you can further filter the results. -func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { - return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys) -} - -// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses -// continuationToken instead of marker to support iteration over the results. -func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) { - return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter) -} - -// CopyObject - copies an object from source object to destination object on server side. -func (c Core) CopyObject(sourceBucket, sourceObject, destBucket, destObject string, metadata map[string]string) (ObjectInfo, error) { - return c.copyObjectDo(context.Background(), sourceBucket, sourceObject, destBucket, destObject, metadata) -} - -// CopyObjectPart - creates a part in a multipart upload by copying (a -// part of) an existing object. -func (c Core) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, - partID int, startOffset, length int64, metadata map[string]string) (p CompletePart, err error) { - - return c.copyObjectPartDo(context.Background(), srcBucket, srcObject, destBucket, destObject, uploadID, - partID, startOffset, length, metadata) -} - -// PutObject - Upload object. Uploads using single PUT call. -func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Base64, sha256Hex string, metadata map[string]string, sse encrypt.ServerSide) (ObjectInfo, error) { - opts := PutObjectOptions{} - m := make(map[string]string) - for k, v := range metadata { - if strings.ToLower(k) == "content-encoding" { - opts.ContentEncoding = v - } else if strings.ToLower(k) == "content-disposition" { - opts.ContentDisposition = v - } else if strings.ToLower(k) == "content-language" { - opts.ContentLanguage = v - } else if strings.ToLower(k) == "content-type" { - opts.ContentType = v - } else if strings.ToLower(k) == "cache-control" { - opts.CacheControl = v - } else if strings.EqualFold(k, amzWebsiteRedirectLocation) { - opts.WebsiteRedirectLocation = v - } else { - m[k] = metadata[k] - } - } - opts.UserMetadata = m - opts.ServerSideEncryption = sse - return c.putObjectDo(context.Background(), bucket, object, data, md5Base64, sha256Hex, size, opts) -} - -// NewMultipartUpload - Initiates new multipart upload and returns the new uploadID. -func (c Core) NewMultipartUpload(bucket, object string, opts PutObjectOptions) (uploadID string, err error) { - result, err := c.initiateMultipartUpload(context.Background(), bucket, object, opts) - return result.UploadID, err -} - -// ListMultipartUploads - List incomplete uploads. -func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { - return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) -} - -// PutObjectPart - Upload an object part. -func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, data io.Reader, size int64, md5Base64, sha256Hex string, sse encrypt.ServerSide) (ObjectPart, error) { - return c.uploadPart(context.Background(), bucket, object, uploadID, data, partID, md5Base64, sha256Hex, size, sse) -} - -// ListObjectParts - List uploaded parts of an incomplete upload.x -func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { - return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) -} - -// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. -func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) (string, error) { - res, err := c.completeMultipartUpload(context.Background(), bucket, object, uploadID, completeMultipartUpload{ - Parts: parts, - }) - return res.ETag, err -} - -// AbortMultipartUpload - Abort an incomplete upload. -func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { - return c.abortMultipartUpload(context.Background(), bucket, object, uploadID) -} - -// GetBucketPolicy - fetches bucket access policy for a given bucket. -func (c Core) GetBucketPolicy(bucket string) (string, error) { - return c.getBucketPolicy(bucket) -} - -// PutBucketPolicy - applies a new bucket access policy for a given bucket. -func (c Core) PutBucketPolicy(bucket, bucketPolicy string) error { - return c.putBucketPolicy(bucket, bucketPolicy) -} - -// GetObject is a lower level API implemented to support reading -// partial objects and also downloading objects with special conditions -// matching etag, modtime etc. -func (c Core) GetObject(bucketName, objectName string, opts GetObjectOptions) (io.ReadCloser, ObjectInfo, error) { - return c.getObject(context.Background(), bucketName, objectName, opts) -} - -// StatObject is a lower level API implemented to support special -// conditions matching etag, modtime on a request. -func (c Core) StatObject(bucketName, objectName string, opts StatObjectOptions) (ObjectInfo, error) { - return c.statObject(context.Background(), bucketName, objectName, opts) -} From a324776bbace5705eb6ee50cb13c2c0807af85b8 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Mon, 12 Aug 2019 11:24:54 +0800 Subject: [PATCH 3/3] fix: image.download changes output parameters --- cmd/climc/shell/images.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/climc/shell/images.go b/cmd/climc/shell/images.go index 9ebbdbba6b..c331643912 100644 --- a/cmd/climc/shell/images.go +++ b/cmd/climc/shell/images.go @@ -381,7 +381,7 @@ func init() { } else { sink = os.Stdout } - meta, src, err := modules.Images.Download(s, imgId, args.Format, args.Torrent) + meta, src, size, err := modules.Images.Download(s, imgId, args.Format, args.Torrent) if err != nil { return err } @@ -391,6 +391,7 @@ func init() { } if len(args.Output) > 0 { printObject(meta) + fmt.Println("Image size: ", size) } return nil })