From 9ba6bef99463e070483bcce6f1364aad696601b3 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Sun, 22 Dec 2024 10:21:23 +0800 Subject: [PATCH] fix: s3cli upgrade 20241222 (#21874) Co-authored-by: Qiu Jian --- go.mod | 4 +-- go.sum | 8 ++--- vendor/modules.txt | 4 +-- .../pkg/multicloud/objectstore/buckets.go | 4 ++- .../pkg/multicloud/objectstore/shell.go | 29 +++++++++++++++++++ vendor/yunion.io/x/s3cli/api-s3-datatypes.go | 12 ++++---- 6 files changed, 46 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 39880b7650..a09a23da46 100644 --- a/go.mod +++ b/go.mod @@ -93,13 +93,13 @@ require ( k8s.io/cri-api v0.22.17 k8s.io/klog/v2 v2.20.0 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241218101206-5e3f6758dc2c + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241221173250-5f4ed89000f5 yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 yunion.io/x/pkg v1.10.2 - yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e + yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c ) diff --git a/go.sum b/go.sum index fa89ae9c75..105a2cfd6a 100644 --- a/go.sum +++ b/go.sum @@ -1376,8 +1376,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241218101206-5e3f6758dc2c h1:h5qhNEV9VsJ6VkbC/lmV+kZW2ZsMT+DRDwb+gGIpr+w= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241218101206-5e3f6758dc2c/go.mod h1:rj/pb3DitJlQaQD8UW1oxx/KD+PzDZqoywzqRJaFE9A= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241221173250-5f4ed89000f5 h1:Nrrybroza4SiDpHUVlShsrSUhwg9R74AhK2W2jNno8w= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241221173250-5f4ed89000f5/go.mod h1:KQ/jWx7bZlmjCE711KEWuvHW/dzpdr/UTlBjjutkj0Y= yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8= yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= @@ -1393,8 +1393,8 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v1.10.2 h1:oZhedvlvDsebIWcOvrOMJQ31SxzjxxqUXUBYseBjh7w= yunion.io/x/pkg v1.10.2/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= -yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= -yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= +yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4= +yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1 h1:HWPqY1I5JSmM6Sks6FyK9hnq/MjL7FDghM6M8DXHob0= yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ= diff --git a/vendor/modules.txt b/vendor/modules.txt index d8b31d8be5..1af6d43f63 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1785,7 +1785,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241218101206-5e3f6758dc2c +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241221173250-5f4ed89000f5 ## explicit; go 1.21 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1934,7 +1934,7 @@ yunion.io/x/pkg/util/vmdkutils yunion.io/x/pkg/util/wait yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils -# yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e +# yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 ## explicit; go 1.12 yunion.io/x/s3cli # yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1 diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/buckets.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/buckets.go index fc8175cb9d..c0b9872fd4 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/buckets.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/buckets.go @@ -22,6 +22,7 @@ import ( "strings" "time" + "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" "yunion.io/x/s3cli" @@ -243,7 +244,8 @@ func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.Reade } err = obj.SetAcl(cannedAcl) if err != nil && errors.Cause(err) != cloudprovider.ErrNotImplemented { - return errors.Wrap(err, "obj.SetAcl") + log.Errorf("PubObject SetAcl fail %s", err) + // return errors.Wrap(err, "obj.SetAcl") } return nil } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/shell.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/shell.go index c6e12f63be..f0aeacad3c 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/shell.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/shell.go @@ -838,6 +838,35 @@ func S3Shell() { return nil }) + type BucketObjectBatchDownloadOptions struct { + BUCKET string `help:"name of bucket"` + PREFIX string `help:"Prefix of object"` + Output string `help:"target output directory, default to current directory"` + } + shellutils.R(&BucketObjectBatchDownloadOptions{}, "batch-download", "Download objects recursively", func(cli cloudprovider.ICloudRegion, args *BucketObjectBatchDownloadOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + marker := "" + const maxCount = 1000 + for { + results, err := bucket.ListObjects(args.PREFIX, marker, "", maxCount) + if err != nil { + return errors.Wrapf(err, "ListObjects prefix %s marker %s", args.PREFIX, marker) + } + for _, obj := range results.Objects { + fmt.Println(obj.GetKey()) + } + if !results.IsTruncated { + break + } else { + marker = results.NextMarker + } + } + return nil + }) + type BucketObjectDownloadOptions struct { BUCKET string `help:"name of bucket"` KEY string `help:"Key of object"` diff --git a/vendor/yunion.io/x/s3cli/api-s3-datatypes.go b/vendor/yunion.io/x/s3cli/api-s3-datatypes.go index b9a7e7afd6..4ddf88f3ec 100644 --- a/vendor/yunion.io/x/s3cli/api-s3-datatypes.go +++ b/vendor/yunion.io/x/s3cli/api-s3-datatypes.go @@ -187,7 +187,7 @@ type ListObjectPartsResult struct { // initiateMultipartUploadResult container for InitiateMultiPartUpload // response. type InitiateMultipartUploadResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` + XMLName xml.Name `xml:"InitiateMultipartUploadResult" json:"-"` Bucket string `xml:"Bucket"` Key string `xml:"Key"` @@ -197,7 +197,7 @@ type InitiateMultipartUploadResult struct { // completeMultipartUploadResult container for completed multipart // upload response. type CompleteMultipartUploadResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` + XMLName xml.Name `xml:"CompleteMultipartUploadResult" json:"-"` Location string `xml:"Location"` Bucket string `xml:"Bucket"` @@ -208,7 +208,7 @@ type CompleteMultipartUploadResult struct { // CompletePart sub container lists individual part numbers and their // md5sum, part of completeMultipartUpload. type CompletePart struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"` + XMLName xml.Name `xml:"Part" json:"-"` // Part number identifies the part. PartNumber int `xml:"PartNumber"` @@ -217,13 +217,13 @@ type CompletePart struct { // completeMultipartUpload container for completing multipart upload. type CompleteMultipartUpload struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"` + XMLName xml.Name `xml:"CompleteMultipartUpload" json:"-"` Parts []CompletePart `xml:"Part"` } // createBucketConfiguration container for bucket configuration. type CreateBucketConfiguration struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"` + XMLName xml.Name `xml:"CreateBucketConfiguration" json:"-"` Location string `xml:"LocationConstraint"` } @@ -267,7 +267,7 @@ type deleteMultiObjectsResult struct { } type VersioningConfiguration struct { - XMLName xml.Name `xmlns:"http://s3.amazonaws.com/doc/2006-03-01/" xml:"VersioningConfiguration"` + XMLName xml.Name `xml:"VersioningConfiguration"` Status string `xml:"Status,omitempty"` MfaDelete string `xml:"MfaDelete,omitempty"` }