From 93b8bac5bba57472a15794ff9f2a35053ceed7f9 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Tue, 23 Jul 2019 01:30:09 +0800 Subject: [PATCH] s3gateway round 2 --- cmd/climc/shell/buckets.go | 86 ++++ go.mod | 9 +- go.sum | 18 +- pkg/apis/compute/bucket.go | 3 + pkg/appsrv/appsrv.go | 3 +- pkg/appsrv/handlerinfo.go | 43 +- pkg/cloudcommon/db/interface.go | 3 + pkg/cloudcommon/db/modelbase.go | 17 +- pkg/cloudprovider/objectstore.go | 131 ++++- pkg/cloudprovider/resources.go | 2 +- pkg/compute/models/buckets.go | 224 +++++++- pkg/compute/models/guest_actions.go | 11 +- pkg/compute/models/natdtable.go | 1 + pkg/compute/models/natgateways.go | 1 + pkg/compute/models/natstable.go | 1 + pkg/compute/models/snapshotpolicy.go | 1 + pkg/image/models/images.go | 2 + pkg/mcclient/modules/mod_buckets.go | 50 +- pkg/multicloud/no_storage_region.go | 2 +- pkg/multicloud/objectstore/buckets.go | 74 +++ pkg/multicloud/objectstore/object.go | 15 + pkg/multicloud/objectstore/objectstore.go | 9 +- pkg/multicloud/objectstore/shell.go | 190 +++++++ pkg/multicloud/objectstore/shell/bucket.go | 60 +-- pkg/util/aliyun/bucket.go | 124 +++++ pkg/util/aliyun/objects.go | 15 + pkg/util/aliyun/region.go | 60 ++- pkg/util/aliyun/shell/bucket.go | 7 + pkg/util/aws/bucket.go | 133 +++++ pkg/util/aws/region.go | 4 +- pkg/util/aws/s3object.go | 13 + pkg/util/aws/shell/bucket.go | 7 + pkg/util/azure/blobobject.go | 13 + pkg/util/azure/classic_disk.go | 3 +- pkg/util/azure/region.go | 4 +- pkg/util/azure/shell/bucket.go | 50 +- pkg/util/azure/shell/storageaccount.go | 2 +- pkg/util/azure/storageaccount.go | 300 ++++++++++- pkg/util/httputils/httputils.go | 8 +- pkg/util/huawei/bucket.go | 128 +++++ pkg/util/huawei/object.go | 13 + pkg/util/huawei/region.go | 25 +- pkg/util/huawei/shell/bucket.go | 9 + pkg/util/huawei/shell/obs.go | 2 +- pkg/util/openstack/region.go | 2 +- pkg/util/qcloud/bucket.go | 122 ++++- pkg/util/qcloud/object.go | 13 + pkg/util/qcloud/qcloud.go | 106 ---- pkg/util/qcloud/region.go | 152 +++++- pkg/util/qcloud/shell/cos.go | 131 +---- pkg/util/qcloud/storagecache.go | 42 +- pkg/util/ucloud/region.go | 18 +- pkg/util/ucloud/shell/ufile.go | 38 +- pkg/util/ucloud/storagecache.go | 30 +- pkg/util/ucloud/ufile.go | 93 ++-- .../github.com/google/go-querystring/LICENSE | 27 + .../google/go-querystring/query/encode.go | 320 ++++++++++++ .../mozillazg/go-httpheader/.bumpversion.cfg | 7 + .../mozillazg/go-httpheader/.gitignore | 27 + .../mozillazg/go-httpheader/.travis.yml | 25 + .../mozillazg/go-httpheader/CHANGELOG.md | 15 + .../mozillazg/go-httpheader/LICENSE | 21 + .../mozillazg/go-httpheader/Makefile | 15 + .../mozillazg/go-httpheader/README.md | 63 +++ .../mozillazg/go-httpheader/encode.go | 290 +++++++++++ .../github.com/mozillazg/go-httpheader/go.mod | 1 + .../nelsonken/cos-go-sdk-v5/cos/auth.go | 128 ----- .../nelsonken/cos-go-sdk-v5/cos/bucket.go | 315 ------------ .../nelsonken/cos-go-sdk-v5/cos/client.go | 164 ------ .../nelsonken/cos-go-sdk-v5/cos/conf.go | 31 -- .../nelsonken/cos-go-sdk-v5/cos/conn.go | 141 ----- .../nelsonken/cos-go-sdk-v5/cos/option.go | 10 - .../nelsonken/cos-go-sdk-v5/cos/requests.go | 102 ---- .../nelsonken/cos-go-sdk-v5/cos/response.go | 156 ------ .../tencentyun/cos-go-sdk-v5/.bumpversion.cfg | 7 + .../tencentyun/cos-go-sdk-v5/.gitignore | 29 ++ .../tencentyun/cos-go-sdk-v5/.travis.yml | 29 ++ .../tencentyun/cos-go-sdk-v5/LICENSE | 21 + .../tencentyun/cos-go-sdk-v5/Makefile | 22 + .../tencentyun/cos-go-sdk-v5/README.md | 95 ++++ .../tencentyun/cos-go-sdk-v5/auth.go | 305 +++++++++++ .../tencentyun/cos-go-sdk-v5/bucket.go | 104 ++++ .../tencentyun/cos-go-sdk-v5/bucket_acl.go | 62 +++ .../tencentyun/cos-go-sdk-v5/bucket_cors.go | 71 +++ .../cos-go-sdk-v5/bucket_inventory.go | 134 +++++ .../cos-go-sdk-v5/bucket_lifecycle.go | 92 ++++ .../cos-go-sdk-v5/bucket_location.go | 28 + .../cos-go-sdk-v5/bucket_logging.go | 53 ++ .../tencentyun/cos-go-sdk-v5/bucket_part.go | 57 +++ .../cos-go-sdk-v5/bucket_replication.go | 73 +++ .../cos-go-sdk-v5/bucket_tagging.go | 69 +++ .../cos-go-sdk-v5/bucket_version.go | 45 ++ .../tencentyun/cos-go-sdk-v5/cos.go | 346 +++++++++++++ .../tencentyun/cos-go-sdk-v5/debug/http.go | 70 +++ .../tencentyun/cos-go-sdk-v5/doc.go | 3 + .../tencentyun/cos-go-sdk-v5/error.go | 49 ++ .../tencentyun/cos-go-sdk-v5/helper.go | 85 +++ .../tencentyun/cos-go-sdk-v5/object.go | 483 ++++++++++++++++++ .../tencentyun/cos-go-sdk-v5/object_acl.go | 63 +++ .../tencentyun/cos-go-sdk-v5/object_part.go | 191 +++++++ .../tencentyun/cos-go-sdk-v5/service.go | 35 ++ vendor/modules.txt | 37 +- vendor/yunion.io/x/structarg/structarg.go | 2 +- 103 files changed, 5509 insertions(+), 1632 deletions(-) create mode 100644 pkg/multicloud/objectstore/object.go create mode 100644 pkg/multicloud/objectstore/shell.go create mode 100644 pkg/util/aliyun/objects.go create mode 100644 pkg/util/aliyun/shell/bucket.go create mode 100644 pkg/util/aws/s3object.go create mode 100644 pkg/util/aws/shell/bucket.go create mode 100644 pkg/util/azure/blobobject.go create mode 100644 pkg/util/huawei/object.go create mode 100644 pkg/util/huawei/shell/bucket.go create mode 100644 pkg/util/qcloud/object.go create mode 100644 vendor/github.com/google/go-querystring/LICENSE create mode 100644 vendor/github.com/google/go-querystring/query/encode.go create mode 100644 vendor/github.com/mozillazg/go-httpheader/.bumpversion.cfg create mode 100644 vendor/github.com/mozillazg/go-httpheader/.gitignore create mode 100644 vendor/github.com/mozillazg/go-httpheader/.travis.yml create mode 100644 vendor/github.com/mozillazg/go-httpheader/CHANGELOG.md create mode 100644 vendor/github.com/mozillazg/go-httpheader/LICENSE create mode 100644 vendor/github.com/mozillazg/go-httpheader/Makefile create mode 100644 vendor/github.com/mozillazg/go-httpheader/README.md create mode 100644 vendor/github.com/mozillazg/go-httpheader/encode.go create mode 100644 vendor/github.com/mozillazg/go-httpheader/go.mod delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/auth.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/bucket.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/client.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conf.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conn.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/option.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/requests.go delete mode 100644 vendor/github.com/nelsonken/cos-go-sdk-v5/cos/response.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/.bumpversion.cfg create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/.gitignore create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/.travis.yml create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/LICENSE create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/Makefile create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/README.md create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/auth.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_acl.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_cors.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_inventory.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_location.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_logging.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_part.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_replication.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_tagging.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_version.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/debug/http.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/doc.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/error.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/object.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/object_acl.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go create mode 100644 vendor/github.com/tencentyun/cos-go-sdk-v5/service.go diff --git a/cmd/climc/shell/buckets.go b/cmd/climc/shell/buckets.go index d30b12e72c..59099ac5fe 100644 --- a/cmd/climc/shell/buckets.go +++ b/cmd/climc/shell/buckets.go @@ -3,6 +3,8 @@ package shell import ( "yunion.io/x/jsonutils" + "io" + "os" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" @@ -84,4 +86,88 @@ func init() { printObject(result) return nil }) + + type BucketListObjectsOptions struct { + ID string `help:"ID or name of bucket" json:"-"` + Prefix string `help:"List objects with prefix"` + Recursive bool `help:"List objects recursively"` + } + R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error { + params, err := options.StructToParams(args) + if err != nil { + return err + } + result, err := modules.Buckets.GetSpecific(s, args.ID, "objects", params) + if err != nil { + return err + } + + arrays, _ := result.(*jsonutils.JSONArray).GetArray() + listResult := modules.ListResult{Data: arrays} + printList(&listResult, []string{}) + return nil + }) + + type BucketDeleteObjectsOptions struct { + ID string `help:"ID or name of bucket" json:"-"` + KEYS []string `help:"List of objects to delete"` + } + R(&BucketDeleteObjectsOptions{}, "bucket-object-delete", "Delete objects in a bucket", func(s *mcclient.ClientSession, args *BucketDeleteObjectsOptions) error { + params, err := options.StructToParams(args) + if err != nil { + return err + } + result, err := modules.Buckets.PerformAction(s, args.ID, "delete", params) + if err != nil { + return err + } + printObject(result) + return nil + }) + + type BucketUploadObjectsOptions struct { + ID string `help:"ID or name of bucket" json:"-"` + KEY string `help:"Key of object to upload"` + Path string `help:"Path to file to upload"` + + ContentType string `help:"Content type"` + StorageClass string `help:"storage CLass"` + } + R(&BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *BucketUploadObjectsOptions) error { + var body io.Reader + if len(args.Path) > 0 { + file, err := os.Open(args.Path) + if err != nil { + return err + } + defer file.Close() + body = file + } else { + body = os.Stdin + } + err := modules.Buckets.Upload(s, args.ID, args.KEY, body, args.ContentType, args.StorageClass) + if err != nil { + return err + } + return nil + }) + + type BucketPresignObjectsOptions struct { + ID string `help:"ID or name of bucket" json:"-"` + KEY string `help:"Key of object to upload"` + Method string `help:"Request method" choices:"GET|PUT|DELETE"` + ExpireSeconds int `help:"expire in seconds" default:"60"` + } + R(&BucketPresignObjectsOptions{}, "bucket-object-tempurl", "Get temporal URL for an object in a bucket", func(s *mcclient.ClientSession, args *BucketPresignObjectsOptions) error { + params, err := options.StructToParams(args) + if err != nil { + return err + } + result, err := modules.Buckets.PerformAction(s, args.ID, "temp-url", params) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/go.mod b/go.mod index a03f7321c1..5e05726e35 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/protobuf v1.3.1 github.com/google/btree v1.0.0 // indirect + github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/google/gopacket v1.1.17 github.com/google/uuid v1.1.0 // indirect @@ -102,8 +103,8 @@ require ( github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/moul/http2curl v1.0.0 + github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/mozillazg/go-pinyin v0.15.0 - github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect github.com/opentracing/opentracing-go v1.0.2 // indirect github.com/openzipkin/zipkin-go-opentracing v0.3.4 // indirect @@ -123,6 +124,7 @@ require ( github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.3.0 github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e + github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f github.com/tinylib/msgp v1.1.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c @@ -144,7 +146,10 @@ require ( google.golang.org/genproto v0.0.0-20181218023534-67d6565462c5 // indirect google.golang.org/grpc v1.19.0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect + gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v8 v8.18.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.44.0 // indirect gopkg.in/ldap.v3 v3.0.3 gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20181004124137-fd83cbc87e76 @@ -157,5 +162,5 @@ require ( yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30 yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda - yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe + yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d ) diff --git a/go.sum b/go.sum index c2c7621ebb..4f82d9b67a 100644 --- a/go.sum +++ b/go.sum @@ -179,6 +179,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY= @@ -288,12 +290,12 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/mozillazg/go-pinyin v0.15.0 h1:sSwlnsogK/WMzcf0HnjgxyAI4GU6LFqwXnhr77q1Z80= github.com/mozillazg/go-pinyin v0.15.0/go.mod h1:bO+dztNW6O2lSJdYLha7LO3bujXzjjU3UvKb2IGANfg= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80 h1:1tGm26e9ktdEIa7LHD7xf7oMUCE0V2wP4URkUAvEttA= -github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80/go.mod h1:UZaoQ2hntRH4P8MwrKOxcVXvNgg1N4atxfa7NP+1wWg= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= @@ -362,6 +364,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M= github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e h1:CtKVGXKh2bfmepZ/YogAjvL/CrSy9NGZET500K7arf4= github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= +github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f h1:TzE7Cs9HhTyfot4WIoMnbD1rWfD4Jkwy2M3Zs66CaRE= +github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f/go.mod h1:/4BhymH1yO6ljUGQgcKsd7L3W+pdKRxoRiOuoZPLnGg= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -453,8 +457,14 @@ gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUy gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0= +gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ldap.v3 v3.0.3 h1:YKRHW/2sIl05JsCtx/5ZuUueFuJyoj/6+DGXe3wp6ro= gopkg.in/ldap.v3 v3.0.3/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -488,5 +498,5 @@ yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30 h1:6CkrwtX4xeYFqqpdWtQPAVqEnD yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda h1:i+/3Hh+kVmPZM1P+j3wfViEzb0XlttWyNvDSYA5DDDU= yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= -yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe h1:Wr2EXv72ynwtW+x0VCqWwpsfUBWecaZWnLMgaP3UiTo= -yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= +yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d h1:00kGV39weRaYPldUUh5mllj4aHcGMOZZx4m3CotRESw= +yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/compute/bucket.go b/pkg/apis/compute/bucket.go index 2855fd3b3a..44ed26491b 100644 --- a/pkg/apis/compute/bucket.go +++ b/pkg/apis/compute/bucket.go @@ -9,4 +9,7 @@ const ( BUCKET_STATUS_DELETING = "deleting" BUCKET_STATUS_DELETED = "deleted" BUCKET_STATUS_DELETE_FAIL = "delete_fail" + + BUCKET_UPLOAD_OBJECT_KEY_HEADER = "X-Yunion-Bucket-Upload-Key" + BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER = "X-Yunion-Bucket-Upload-Storageclass" ) diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index ee2f16bf02..78fa92263a 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -250,7 +250,8 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri to = app.processTimeout } var ( - ctx context.Context = app.context + ctx = app.context + cancel context.CancelFunc = nil ) if to > 0 { diff --git a/pkg/appsrv/handlerinfo.go b/pkg/appsrv/handlerinfo.go index dde6d05581..14c5fa6926 100644 --- a/pkg/appsrv/handlerinfo.go +++ b/pkg/appsrv/handlerinfo.go @@ -26,27 +26,38 @@ type handlerRequestCounter struct { duration float64 } +type TProcessTimeoutCallback func(*SHandlerInfo, *http.Request) time.Duration + type SHandlerInfo struct { - method string - path []string - name string - handler func(context.Context, http.ResponseWriter, *http.Request) - metadata map[string]interface{} - tags map[string]string - counter2XX handlerRequestCounter - counter4XX handlerRequestCounter - counter5XX handlerRequestCounter - processTimeout time.Duration - workerMan *SWorkerManager - skipLog bool + method string + path []string + name string + handler func(context.Context, http.ResponseWriter, *http.Request) + metadata map[string]interface{} + tags map[string]string + counter2XX handlerRequestCounter + counter4XX handlerRequestCounter + counter5XX handlerRequestCounter + workerMan *SWorkerManager + skipLog bool + + processTimeout time.Duration + processTimeoutCallback TProcessTimeoutCallback } func (this *SHandlerInfo) FetchProcessTimeout(r *http.Request) time.Duration { - if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 { - return time.Hour * 2 - } else { - return this.processTimeout + var tm time.Duration + if this.processTimeoutCallback != nil { + tm = this.processTimeoutCallback(this, r) } + if tm < this.processTimeout { + tm = this.processTimeout + } + return tm +} + +func (this *SHandlerInfo) SetProcessTimeoutCallback(callback TProcessTimeoutCallback) { + this.processTimeoutCallback = callback } func (this *SHandlerInfo) GetName(params map[string]string) string { diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 3f8e767ede..91d33d0690 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/sqlchemy" + "time" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/object" @@ -90,6 +91,8 @@ type IModelManager interface { InitializeData() error CustomizeHandlerInfo(info *appsrv.SHandlerInfo) + SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration + FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) FetchUpdateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) IsCustomizedGetDetailsBody() bool diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 6182934897..4396b60c4a 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -22,6 +22,8 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/sqlchemy" + "log" + "time" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/object" "yunion.io/x/onecloud/pkg/httperrors" @@ -53,7 +55,11 @@ func NewModelBaseManager(model interface{}, tableName string, keyword string, ke } func (manager *SModelBaseManager) GetIModelManager() IModelManager { - return manager.GetVirtualObject().(IModelManager) + virt := manager.GetVirtualObject() + if virt == nil { + log.Fatalf("%s.GetIModelManager got nil!", manager.keywordPlural) + } + return virt.(IModelManager) } func (manager *SModelBaseManager) SetAlias(alias string, aliasPlural string) { @@ -203,7 +209,14 @@ func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query } func (manager *SModelBaseManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) { - // do nothing + info.SetProcessTimeoutCallback(manager.GetIModelManager().SetHandlerProcessTimeout) +} + +func (manager *SModelBaseManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration { + if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 { + return time.Hour * 2 + } + return -time.Second } func (manager *SModelBaseManager) FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) { diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index 209bc9d708..cfc1e8a0cf 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -1,8 +1,11 @@ package cloudprovider import ( + "context" + "io" "time" + "strings" "yunion.io/x/pkg/errors" ) @@ -11,6 +14,22 @@ type SBucketAccessUrl struct { Description string } +type SBaseCloudObject struct { + Key string + SizeBytes int64 + StorageClass string + ETag string + LastModified time.Time + ContentType string +} + +type SListObjectResult struct { + Objects []ICloudObject + NextMarker string + CommonPrefixes []ICloudObject + IsTruncated bool +} + type ICloudBucket interface { IVirtualResource @@ -22,17 +41,125 @@ type ICloudBucket interface { GetCreateAt() time.Time GetStorageClass() string GetAccessUrls() []SBucketAccessUrl + + ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error) + GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error) + PutObject(ctx context.Context, key string, input io.ReadSeeker, contType string, storageClass string) error + DeleteObject(ctx context.Context, keys string) error + GetTempUrl(method string, key string, expire time.Duration) (string, error) + // ObjectExist(key string) (bool, error) } -func GetIBucketByName(region ICloudRegion, name string) (ICloudBucket, error) { +type ICloudObject interface { + GetIBucket() ICloudBucket + + GetKey() string + GetSizeBytes() int64 + GetLastModified() time.Time + GetStorageClass() string + GetETag() string + GetContentType() string +} + +func ICloudObject2BaseCloudObject(obj ICloudObject) SBaseCloudObject { + return SBaseCloudObject{ + Key: obj.GetKey(), + SizeBytes: obj.GetSizeBytes(), + StorageClass: obj.GetStorageClass(), + ETag: obj.GetETag(), + LastModified: obj.GetLastModified(), + ContentType: obj.GetContentType(), + } +} + +func (o *SBaseCloudObject) GetKey() string { + return o.Key +} + +func (o *SBaseCloudObject) GetSizeBytes() int64 { + return o.SizeBytes +} + +func (o *SBaseCloudObject) GetLastModified() time.Time { + return o.LastModified +} + +func (o *SBaseCloudObject) GetStorageClass() string { + return o.StorageClass +} + +func (o *SBaseCloudObject) GetETag() string { + return o.ETag +} + +func (o *SBaseCloudObject) GetContentType() string { + return o.ContentType +} + +func GetIBucketById(region ICloudRegion, name string) (ICloudBucket, error) { buckets, err := region.GetIBuckets() if err != nil { return nil, errors.Wrap(err, "region.GetIBuckets") } for i := range buckets { - if buckets[i].GetName() == name { + if buckets[i].GetGlobalId() == name { return buckets[i], nil } } return nil, ErrNotFound } + +func GetIObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) { + delimiter := "/" + if isRecursive { + delimiter = "" + } + ret := make([]ICloudObject, 0) + // Save marker for next request. + var marker string + for { + // Get list of objects a maximum of 1000 per request. + result, err := bucket.ListObjects(objectPrefix, marker, delimiter, 1000) + if err != nil { + return nil, errors.Wrap(err, "bucket.ListObjects") + } + + // Send all objects + if len(result.Objects) > 0 { + ret = append(ret, result.Objects...) + marker = result.Objects[len(result.Objects)-1].GetKey() + } + + // Send all common prefixes if any. + // NOTE: prefixes are only present if the request is delimited. + if len(result.CommonPrefixes) > 0 { + ret = append(ret, result.CommonPrefixes...) + } + + // If next marker present, save it for next request. + if result.NextMarker != "" { + marker = result.NextMarker + } + + // Listing ends result is not truncated, break the loop + if !result.IsTruncated { + break + } + } + return ret, nil +} + +func Makedir(ctx context.Context, bucket ICloudBucket, key string) error { + segs := make([]string, 0) + for _, seg := range strings.Split(key, "/") { + if len(seg) > 0 { + segs = append(segs, seg) + } + } + path := strings.Join(segs, "/") + "/" + err := bucket.PutObject(ctx, path, strings.NewReader(""), "", "") + if err != nil { + return errors.Wrap(err, "PutObject") + } + return nil +} diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 3eb5b87cb6..5cccd1efe3 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -114,7 +114,7 @@ type ICloudRegion interface { CreateIBucket(name string, storageClassStr string, acl string) error DeleteIBucket(name string) error IBucketExist(name string) (bool, error) - GetIBucketByName(name string) (ICloudBucket, error) + GetIBucketById(name string) (ICloudBucket, error) GetProvider() string } diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 12d751fec0..d38b521976 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -3,7 +3,11 @@ package models import ( "context" "database/sql" - "regexp" + "net/http" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -11,7 +15,9 @@ import ( "yunion.io/x/pkg/util/compare" "yunion.io/x/sqlchemy" + "io" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" @@ -19,6 +25,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" ) type SBucketManager struct { @@ -52,6 +59,14 @@ type SBucket struct { Acl string `width:"36" charset:"ascii" nullable:"false" list:"user"` } +func (manager *SBucketManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration { + if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/upload") && r.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER) != "" { + log.Debugf("upload object, set process timeout to 2 hour!!!") + return 2 * time.Hour + } + return manager.SVirtualResourceBaseManager.SetHandlerProcessTimeout(info, r) +} + func (manager *SBucketManager) fetchBuckets(provider *SCloudprovider, region *SCloudregion) ([]SBucket, error) { q := manager.Query() if provider != nil { @@ -267,10 +282,16 @@ func (bucket *SBucket) GetIRegion() (cloudprovider.ICloudRegion, error) { return provider.GetIRegionById(region.GetExternalId()) } -var BUCKET_NAME_REG = regexp.MustCompile(`^[a-zA-Z0-9-]+$`) +func (bucket *SBucket) GetIBucket() (cloudprovider.ICloudBucket, error) { + iregion, err := bucket.GetIRegion() + if err != nil { + return nil, errors.Wrap(err, "bucket.GetIRegion") + } + return iregion.GetIBucketById(bucket.ExternalId) +} -func isValidBucketName(name string) bool { - return BUCKET_NAME_REG.MatchString(name) +func isValidBucketName(name string) error { + return s3utils.CheckValidBucketNameStrict(name) } func (manager *SBucketManager) ValidateCreateData( @@ -293,8 +314,9 @@ func (manager *SBucketManager) ValidateCreateData( if len(nameStr) == 0 { return nil, httperrors.NewInputParameterError("missing name") } - if !isValidBucketName(nameStr) { - return nil, httperrors.NewInputParameterError("invalid name, only alphabets, digits and hyphen(-) allowed") + err := isValidBucketName(nameStr) + if err != nil { + return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err) } return manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data) } @@ -323,8 +345,9 @@ func (bucket *SBucket) ValidateUpdateData( ) (*jsonutils.JSONDict, error) { nameStr, _ := data.GetString("name") if len(nameStr) > 0 { - if !isValidBucketName(nameStr) { - return nil, httperrors.NewInputParameterError("invalid name, only alphabets, digits and hyphen(-) allowed") + err := isValidBucketName(nameStr) + if err != nil { + return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err) } } return bucket.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data) @@ -343,7 +366,7 @@ func (bucket *SBucket) RemoteCreate(ctx context.Context, userCred mcclient.Token if err != nil { return errors.Wrap(err, "db.SetExternalId") } - extBucket, err := iregion.GetIBucketByName(bucket.Name) + extBucket, err := iregion.GetIBucketById(bucket.Name) if err != nil { return errors.Wrap(err, "iregion.GetIBucketByName") } @@ -401,3 +424,186 @@ func (manager *SBucketManager) ListItemFilter(ctx context.Context, q *sqlchemy.S return q, nil } + +func (bucket *SBucket) AllowGetDetailsObjects( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, +) bool { + return bucket.IsOwner(userCred) +} + +func (bucket *SBucket) GetDetailsObjects( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + iBucket, err := bucket.GetIBucket() + if err != nil { + return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err) + } + prefix, _ := query.GetString("prefix") + isRecursive := jsonutils.QueryBoolean(query, "recursive", false) + objects, err := iBucket.GetIObjects(prefix, isRecursive) + if err != nil { + return nil, httperrors.NewInternalServerError("fail to get objects: %s", err) + } + retArray := jsonutils.NewArray() + for i := range objects { + retArray.Add(jsonutils.Marshal(cloudprovider.ICloudObject2BaseCloudObject(objects[i]))) + } + return retArray, nil +} + +func (bucket *SBucket) AllowPerformTempUrl(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) bool { + return bucket.IsOwner(userCred) +} + +func (bucket *SBucket) PerformTempUrl( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + method, _ := data.GetString("method") + key, _ := data.GetString("key") + expire, _ := data.Int("expire_seconds") + + if len(method) == 0 { + method = "GET" + } + if len(key) == 0 { + return nil, httperrors.NewInputParameterError("missing key") + } + if expire == 0 { + expire = 60 // default 60 seconds + } + + iBucket, err := bucket.GetIBucket() + if err != nil { + return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err) + } + tmpUrl, err := iBucket.GetTempUrl(method, key, time.Duration(expire)*time.Second) + if err != nil { + return nil, httperrors.NewInternalServerError("fail to generate temp url: %s", err) + } + ret := jsonutils.NewDict() + ret.Add(jsonutils.NewString(tmpUrl), "url") + return ret, nil +} + +func (bucket *SBucket) AllowPerformMakedir(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) bool { + return bucket.IsOwner(userCred) +} + +func (bucket *SBucket) PerformMakedir( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + key, _ := data.GetString("key") + if key[len(key)-1] != '/' { + return nil, httperrors.NewInputParameterError("directory must ends with /") + } + err := s3utils.CheckValidObjectName(key) + if err != nil { + return nil, httperrors.NewInputParameterError("invalid key: %s", err) + } + + iBucket, err := bucket.GetIBucket() + if err != nil { + return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err) + } + + err = cloudprovider.Makedir(ctx, iBucket, key) + if err != nil { + return nil, httperrors.NewInternalServerError("fail to mkdir: %s", err) + } + + return nil, nil +} + +func (bucket *SBucket) AllowPerformDelete(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) bool { + return bucket.IsOwner(userCred) +} + +func (bucket *SBucket) PerformDelete( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + keys, _ := data.Get("keys") + if keys == nil { + return nil, httperrors.NewInputParameterError("missing keys") + } + keyStrs := keys.(*jsonutils.JSONArray).GetStringArray() + if len(keyStrs) == 0 { + return nil, httperrors.NewInputParameterError("empty keys") + } + + iBucket, err := bucket.GetIBucket() + if err != nil { + return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err) + } + ok := jsonutils.NewDict() + results := modules.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) { + err := iBucket.DeleteObject(ctx, key) + if err != nil { + return nil, err + } else { + return ok, nil + } + }) + return modules.SubmitResults2JSON(results), nil +} + +func (bucket *SBucket) AllowPerformUpload(ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) bool { + return bucket.IsOwner(userCred) +} + +func (bucket *SBucket) PerformUpload( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + appParams := appsrv.AppContextGetParams(ctx) + + key := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER) + err := s3utils.CheckValidObjectName(key) + if err != nil { + return nil, httperrors.NewInputParameterError("invalid object key: %s", err) + } + + iBucket, err := bucket.GetIBucket() + if err != nil { + return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err) + } + + contType := appParams.Request.Header.Get("Content-Type") + storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER) + err = iBucket.PutObject(ctx, key, appParams.Request.Body.(io.ReadSeeker), contType, storageClass) + if err != nil { + return nil, httperrors.NewInternalServerError("put object error %s", err) + } + + return nil, nil +} diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index fa2068d167..48dde0a64b 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -82,12 +82,17 @@ func (self *SGuest) GetDetailsVnc(ctx context.Context, userCred mcclient.TokenCr func (self *SGuest) AllowPerformMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, - data jsonutils.JSONObject) bool { + data jsonutils.JSONObject, +) bool { return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "monitor") } -func (self *SGuest) PerformMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, - data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func (self *SGuest) PerformMonitor( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_BLOCK_STREAM}) { cmd, err := data.GetString("command") if err != nil { diff --git a/pkg/compute/models/natdtable.go b/pkg/compute/models/natdtable.go index 916ff6a2fe..a16518eff2 100644 --- a/pkg/compute/models/natdtable.go +++ b/pkg/compute/models/natdtable.go @@ -46,6 +46,7 @@ func init() { "natdtables", ), } + NatDTableManager.SetVirtualObject(NatDTableManager) } type SNatDEntry struct { diff --git a/pkg/compute/models/natgateways.go b/pkg/compute/models/natgateways.go index bb14d1d66e..8062e7da50 100644 --- a/pkg/compute/models/natgateways.go +++ b/pkg/compute/models/natgateways.go @@ -46,6 +46,7 @@ func init() { "natgateways", ), } + NatGatewayManager.SetVirtualObject(NatGatewayManager) } type SNatGateway struct { diff --git a/pkg/compute/models/natstable.go b/pkg/compute/models/natstable.go index 4ce445213b..31e3970fd5 100644 --- a/pkg/compute/models/natstable.go +++ b/pkg/compute/models/natstable.go @@ -46,6 +46,7 @@ func init() { "natstables", ), } + NatSTableManager.SetVirtualObject(NatSTableManager) } type SNatSEntry struct { diff --git a/pkg/compute/models/snapshotpolicy.go b/pkg/compute/models/snapshotpolicy.go index b33e449d9e..786c4a9116 100644 --- a/pkg/compute/models/snapshotpolicy.go +++ b/pkg/compute/models/snapshotpolicy.go @@ -62,6 +62,7 @@ func init() { "snapshotpolicies", ), } + SnapshotPolicyManager.SetVirtualObject(SnapshotPolicyManager) } func (manager *SSnapshotPolicyManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go index 3a2879ddbb..7ba528daf5 100644 --- a/pkg/image/models/images.go +++ b/pkg/image/models/images.go @@ -122,6 +122,8 @@ type SImage struct { } func (manager *SImageManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) { + manager.SVirtualResourceBaseManager.CustomizeHandlerInfo(info) + switch info.GetName(nil) { case "get_details", "create", "update": info.SetProcessTimeout(time.Minute * 120).SetWorkerManager(imgStreamingWorkerMan) diff --git a/pkg/mcclient/modules/mod_buckets.go b/pkg/mcclient/modules/mod_buckets.go index efe7fd6470..bc3284dd6b 100644 --- a/pkg/mcclient/modules/mod_buckets.go +++ b/pkg/mcclient/modules/mod_buckets.go @@ -14,17 +14,53 @@ package modules +import ( + "fmt" + "io" + "net/http" + + "github.com/pkg/errors" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" +) + +type SBucketManager struct { + ResourceManager +} + +func (manager *SBucketManager) Upload(s *mcclient.ClientSession, bucketId string, key string, body io.Reader, contType string, storageClass string) error { + method := httputils.POST + path := fmt.Sprintf("/%s/%s/upload", manager.URLPath(), bucketId) + headers := http.Header{} + headers.Set(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER, key) + if len(contType) > 0 { + headers.Set("Content-Type", contType) + } + if len(storageClass) > 0 { + headers.Set(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER, storageClass) + } + + _, err := manager.rawRequest(s, method, path, headers, body) + if err != nil { + return errors.Wrap(err, "rawRequest") + } + return nil +} + var ( - Buckets ResourceManager + Buckets SBucketManager ) func init() { - Buckets = NewComputeManager("bucket", "buckets", - []string{"ID", "Name", "Storage_Class", - "Status", "location", "acl", - "region", - }, - []string{}) + Buckets = SBucketManager{ + NewComputeManager("bucket", "buckets", + []string{"ID", "Name", "Storage_Class", + "Status", "location", "acl", + "region", "manager_id", + }, + []string{}), + } registerCompute(&Buckets) } diff --git a/pkg/multicloud/no_storage_region.go b/pkg/multicloud/no_storage_region.go index 0865f02cc3..cee40885f3 100644 --- a/pkg/multicloud/no_storage_region.go +++ b/pkg/multicloud/no_storage_region.go @@ -22,7 +22,7 @@ func (cli *SNoObjectStorageRegion) IBucketExist(name string) (bool, error) { return false, cloudprovider.ErrNotSupported } -func (cli *SNoObjectStorageRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { +func (cli *SNoObjectStorageRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { return nil, cloudprovider.ErrNotSupported } diff --git a/pkg/multicloud/objectstore/buckets.go b/pkg/multicloud/objectstore/buckets.go index 6411138d17..c91ffe11dd 100644 --- a/pkg/multicloud/objectstore/buckets.go +++ b/pkg/multicloud/objectstore/buckets.go @@ -1,10 +1,16 @@ package objectstore import ( + "context" "fmt" + "io" "path" "time" + "github.com/minio/minio-go" + + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -58,3 +64,71 @@ func (bucket *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { }, } } + +func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + isRecursive := true + if delimiter == "/" { + isRecursive = false + } + result := cloudprovider.SListObjectResult{} + var err error + result.Objects, err = bucket.GetIObjects(prefix, isRecursive) + return result, err +} + +func (bucket *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + doneCh := make(chan struct{}) + defer close(doneCh) + + ret := make([]cloudprovider.ICloudObject, 0) + objectCh := bucket.client.client.ListObjects(bucket.Name, prefix, isRecursive, doneCh) + for object := range objectCh { + if object.Err != nil { + return nil, errors.Wrap(object.Err, "ListObjects") + } + obj := &SObject{ + bucket: bucket, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: object.StorageClass, + Key: object.Key, + SizeBytes: object.Size, + ETag: object.ETag, + LastModified: object.LastModified, + ContentType: object.ContentType, + }, + } + ret = append(ret, obj) + } + return ret, nil +} + +func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.ReadSeeker, contType string, storageClass string) error { + opts := minio.PutObjectOptions{} + if len(contType) > 0 { + opts.ContentType = contType + } + if len(storageClass) > 0 { + opts.StorageClass = storageClass + } + _, err := bucket.client.client.PutObjectWithContext(ctx, bucket.Name, key, input, -1, opts) + return err +} + +func (bucket *SBucket) DeleteObject(ctx context.Context, key string) error { + err := bucket.client.client.RemoveObject(bucket.Name, key) + if err != nil { + return errors.Wrap(err, "RemoveObject") + } + return nil +} + +func (bucket *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + if method != "GET" && method != "PUT" && method != "DELETE" { + return "", errors.Error("unsupported method") + } + url, err := bucket.client.client.Presign(method, bucket.Name, key, expire, nil) + if err != nil { + return "", errors.Wrap(err, "Presign") + } + return url.String(), nil +} diff --git a/pkg/multicloud/objectstore/object.go b/pkg/multicloud/objectstore/object.go new file mode 100644 index 0000000000..61eb78f264 --- /dev/null +++ b/pkg/multicloud/objectstore/object.go @@ -0,0 +1,15 @@ +package objectstore + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} diff --git a/pkg/multicloud/objectstore/objectstore.go b/pkg/multicloud/objectstore/objectstore.go index 22f4415aec..59e0bad569 100644 --- a/pkg/multicloud/objectstore/objectstore.go +++ b/pkg/multicloud/objectstore/objectstore.go @@ -9,10 +9,12 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/secrules" + "time" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/object" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SObjectStoreClient struct { @@ -54,6 +56,9 @@ func NewObjectStoreClient(providerId string, providerName string, endpoint strin return nil, errors.Wrap(err, "minio.New") } + tr := httputils.GetTransport(true, time.Second*5) + cli.SetCustomTransport(tr) + client.client = cli return &client, nil @@ -319,6 +324,6 @@ func (cli *SObjectStoreClient) IBucketExist(name string) (bool, error) { return exist, nil } -func (cli *SObjectStoreClient) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { - return cloudprovider.GetIBucketByName(cli, name) +func (cli *SObjectStoreClient) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + return cloudprovider.GetIBucketById(cli, name) } diff --git a/pkg/multicloud/objectstore/shell.go b/pkg/multicloud/objectstore/shell.go new file mode 100644 index 0000000000..9486454984 --- /dev/null +++ b/pkg/multicloud/objectstore/shell.go @@ -0,0 +1,190 @@ +package objectstore + +import ( + "context" + "fmt" + "io" + "os" + "time" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/printutils" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func S3Shell() { + type BucketListOptions struct { + } + shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli cloudprovider.ICloudRegion, args *BucketListOptions) error { + buckets, err := cli.GetIBuckets() + if err != nil { + return err + } + printutils.PrintGetterList(buckets, nil) + return nil + }) + + type BucketCreateOptions struct { + NAME string `help:"name of bucket to create"` + } + shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli cloudprovider.ICloudRegion, args *BucketCreateOptions) error { + err := cli.CreateIBucket(args.NAME, "", "") + if err != nil { + return err + } + return nil + }) + + type BucketDeleteOptions struct { + NAME string `help:"name of bucket to delete"` + } + shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteOptions) error { + err := cli.DeleteIBucket(args.NAME) + if err != nil { + return err + } + return nil + }) + + type BucketObjectsOptions struct { + BUCKET string `help:"name of bucket to list objects"` + Prefix string `help:"prefix"` + Marker string `help:"marker"` + Demiliter string `help:"delimiter"` + Max int `help:"Max count"` + } + shellutils.R(&BucketObjectsOptions{}, "bucket-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketObjectsOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + result, err := bucket.ListObjects(args.Prefix, args.Marker, args.Demiliter, args.Max) + if err != nil { + return err + } + if result.IsTruncated { + fmt.Println("NextMarker: %s IsTruncated: %v", result.NextMarker, result.IsTruncated) + } + fmt.Println("Common prefixes:") + printutils.PrintGetterList(result.CommonPrefixes, []string{"key", "size_bytes"}) + fmt.Println("Objects:") + printutils.PrintGetterList(result.Objects, []string{"key", "size_bytes"}) + return nil + }) + + type BucketListObjectsOptions struct { + BUCKET string `help:"name of bucket to list objects"` + Prefix string `help:"prefix"` + } + shellutils.R(&BucketListObjectsOptions{}, "bucket-list-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + objects, err := bucket.GetIObjects(args.Prefix, true) + if err != nil { + return err + } + printutils.PrintGetterList(objects, []string{"key", "size_bytes"}) + return nil + }) + + shellutils.R(&BucketListObjectsOptions{}, "bucket-dir-object", "List objects in a bucket like directory", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + objects, err := bucket.GetIObjects(args.Prefix, false) + if err != nil { + return err + } + printutils.PrintGetterList(objects, []string{"key", "size_bytes"}) + return nil + }) + + type BucketMakrdirOptions struct { + BUCKET string `help:"name of bucket to put object"` + DIR string `help:"dir to make"` + } + shellutils.R(&BucketMakrdirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketMakrdirOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + err = cloudprovider.Makedir(context.Background(), bucket, args.DIR) + if err != nil { + return err + } + fmt.Printf("Mkdir success\n") + return nil + }) + + type BucketPutObjectOptions struct { + BUCKET string `help:"name of bucket to put object"` + KEY string `help:"key of object"` + Path string `help:"Path of file to upload"` + + ContentType string `help:"content-type"` + StorageClass string `help:"storage class"` + } + shellutils.R(&BucketPutObjectOptions{}, "put-object", "Put object into a bucket", func(cli cloudprovider.ICloudRegion, args *BucketPutObjectOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + var input io.ReadSeeker + if len(args.Path) > 0 { + file, err := os.Open(args.Path) + if err != nil { + return err + } + defer file.Close() + + input = file + } else { + input = os.Stdout + } + err = bucket.PutObject(context.Background(), args.KEY, input, args.ContentType, args.StorageClass) + if err != nil { + return err + } + fmt.Printf("Upload success\n") + return nil + }) + + type BucketDeleteObjectOptions struct { + BUCKET string `help:"name of bucket to put object"` + KEY string `help:"key of object"` + } + shellutils.R(&BucketDeleteObjectOptions{}, "delete-object", "Delete object from a bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteObjectOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + err = bucket.DeleteObject(context.Background(), args.KEY) + if err != nil { + return err + } + fmt.Printf("Delete success\n") + return nil + }) + + type BucketTempUrlOption struct { + BUCKET string `help:"name of bucket to put object"` + METHOD string `help:"http method" choices:"GET|PUT|DELETE"` + KEY string `help:"key of object"` + Duration int `help:"duration in seconds" default:"60"` + } + shellutils.R(&BucketTempUrlOption{}, "temp-url", "generate temp url", func(cli cloudprovider.ICloudRegion, args *BucketTempUrlOption) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + urlStr, err := bucket.GetTempUrl(args.METHOD, args.KEY, time.Duration(args.Duration)*time.Second) + if err != nil { + return err + } + fmt.Println(urlStr) + return nil + }) +} diff --git a/pkg/multicloud/objectstore/shell/bucket.go b/pkg/multicloud/objectstore/shell/bucket.go index f8c19fceb9..df5a4c18fa 100644 --- a/pkg/multicloud/objectstore/shell/bucket.go +++ b/pkg/multicloud/objectstore/shell/bucket.go @@ -1,63 +1,7 @@ package shell -import ( - "fmt" - "yunion.io/x/onecloud/pkg/multicloud/objectstore" - "yunion.io/x/onecloud/pkg/util/shellutils" -) +import "yunion.io/x/onecloud/pkg/multicloud/objectstore" func init() { - type BucketListOptions struct { - } - shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli *objectstore.SObjectStoreClient, args *BucketListOptions) error { - buckets, err := cli.GetIBuckets() - if err != nil { - return err - } - printList(buckets, nil) - return nil - }) - - type BucketCreateOptions struct { - NAME string `help:"name of bucket to create"` - } - shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *objectstore.SObjectStoreClient, args *BucketCreateOptions) error { - err := cli.CreateIBucket(args.NAME, "", "") - if err != nil { - return err - } - return nil - }) - - type BucketDeleteOptions struct { - NAME string `help:"name of bucket to delete"` - } - shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli *objectstore.SObjectStoreClient, args *BucketDeleteOptions) error { - err := cli.DeleteIBucket(args.NAME) - if err != nil { - return err - } - return nil - }) - - type BucketPolicyOptions struct { - NAME string `help:"name of bucket to get policy"` - } - shellutils.R(&BucketPolicyOptions{}, "bucket-policy", "Get bucket policy", func(cli *objectstore.SObjectStoreClient, args *BucketPolicyOptions) error { - policy, err := cli.GetIBucketPolicy(args.NAME) - if err != nil { - return err - } - fmt.Println(policy) - return nil - }) - - shellutils.R(&BucketPolicyOptions{}, "bucket-lifecycle", "Get bucket lifecycle", func(cli *objectstore.SObjectStoreClient, args *BucketPolicyOptions) error { - lifecycle, err := cli.GetIBucketLiftcycle(args.NAME) - if err != nil { - return err - } - fmt.Println(lifecycle) - return nil - }) + objectstore.S3Shell() } diff --git a/pkg/util/aliyun/bucket.go b/pkg/util/aliyun/bucket.go index 83fb960a78..00862debad 100644 --- a/pkg/util/aliyun/bucket.go +++ b/pkg/util/aliyun/bucket.go @@ -1,9 +1,15 @@ package aliyun import ( + "context" "fmt" + "io" "time" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -61,3 +67,121 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { }, } } + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + osscli, err := b.region.GetOssClient() + if err != nil { + return result, errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return result, errors.Wrap(err, "Bucket") + } + opts := make([]oss.Option, 0) + if len(prefix) > 0 { + opts = append(opts, oss.Prefix(prefix)) + } + if len(delimiter) > 0 { + opts = append(opts, oss.Delimiter(delimiter)) + } + if len(marker) > 0 { + opts = append(opts, oss.Marker(marker)) + } + if maxCount > 0 { + opts = append(opts, oss.MaxKeys(maxCount)) + } + oResult, err := bucket.ListObjects(opts...) + if err != nil { + return result, errors.Wrap(err, "ListObjects") + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + for _, object := range oResult.Objects { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: object.StorageClass, + Key: object.Key, + SizeBytes: object.Size, + ETag: object.ETag, + LastModified: object.LastModified, + ContentType: object.Type, + }, + } + result.Objects = append(result.Objects, obj) + } + if oResult.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(oResult.CommonPrefixes)) + for i, commPrefix := range oResult.CommonPrefixes { + result.CommonPrefixes[i] = &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: commPrefix}, + } + } + } + result.IsTruncated = oResult.IsTruncated + result.NextMarker = oResult.NextMarker + return result, nil +} + +func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + return cloudprovider.GetIObjects(b, prefix, isRecursive) +} + +func (b *SBucket) PutObject(ctx context.Context, key string, input io.ReadSeeker, contType string, storageClassStr string) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return errors.Wrap(err, "Bucket") + } + opts := make([]oss.Option, 0) + if len(contType) > 0 { + opts = append(opts, oss.ContentType(contType)) + } + if len(storageClassStr) > 0 { + storageClass, err := str2StorageClass(storageClassStr) + if err != nil { + return errors.Wrap(err, "str2StorageClass") + } + opts = append(opts, oss.ObjectStorageClass(storageClass)) + } + return bucket.PutObject(key, input, opts...) +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return errors.Wrap(err, "Bucket") + } + err = bucket.DeleteObject(key) + if err != nil { + return errors.Wrap(err, "DeleteObject") + } + return nil +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + if method != "GET" && method != "PUT" && method != "DELETE" { + return "", errors.Error("unsupported method") + } + osscli, err := b.region.GetOssClient() + if err != nil { + return "", errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return "", errors.Wrap(err, "Bucket") + } + urlStr, err := bucket.SignURL(key, oss.HTTPMethod(method), int64(expire/time.Second)) + if err != nil { + return "", errors.Wrap(err, "SignURL") + } + return urlStr, nil +} diff --git a/pkg/util/aliyun/objects.go b/pkg/util/aliyun/objects.go new file mode 100644 index 0000000000..173201ae04 --- /dev/null +++ b/pkg/util/aliyun/objects.go @@ -0,0 +1,15 @@ +package aliyun + +import ( + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} diff --git a/pkg/util/aliyun/region.go b/pkg/util/aliyun/region.go index 72c8a82567..00bcc2bd2e 100644 --- a/pkg/util/aliyun/region.go +++ b/pkg/util/aliyun/region.go @@ -953,6 +953,34 @@ func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { return ret, nil } +func str2StorageClass(storageClassStr string) (oss.StorageClassType, error) { + storageClass := oss.StorageStandard + if strings.EqualFold(storageClassStr, string(oss.StorageStandard)) { + // + } else if strings.EqualFold(storageClassStr, string(oss.StorageIA)) { + storageClass = oss.StorageIA + } else if strings.EqualFold(storageClassStr, string(oss.StorageArchive)) { + storageClass = oss.StorageArchive + } else { + return storageClass, errors.Error("not supported storageClass") + } + return storageClass, nil +} + +func str2Acl(aclStr string) (oss.ACLType, error) { + acl := oss.ACLPrivate + if strings.EqualFold(aclStr, string(oss.ACLPrivate)) { + // private, default + } else if strings.EqualFold(aclStr, string(oss.ACLPublicRead)) { + acl = oss.ACLPublicRead + } else if strings.EqualFold(aclStr, string(oss.ACLPublicReadWrite)) { + acl = oss.ACLPublicReadWrite + } else { + return acl, errors.Error("not supported acl") + } + return acl, nil +} + func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { osscli, err := region.GetOssClient() if err != nil { @@ -960,32 +988,18 @@ func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr } opts := make([]oss.Option, 0) if len(storageClassStr) > 0 { - storageClass := oss.StorageStandard - if strings.EqualFold(storageClassStr, string(oss.StorageStandard)) { - // - } else if strings.EqualFold(storageClassStr, string(oss.StorageIA)) { - storageClass = oss.StorageIA - } else if strings.EqualFold(storageClassStr, string(oss.StorageArchive)) { - storageClass = oss.StorageArchive - } else { - return errors.Error("not supported storageClass") + storageClass, err := str2StorageClass(storageClassStr) + if err != nil { + return err } - opt := oss.StorageClass(storageClass) - opts = append(opts, opt) + opts = append(opts, oss.StorageClass(storageClass)) } if len(aclStr) > 0 { - acl := oss.ACLPrivate - if strings.EqualFold(aclStr, string(oss.ACLPrivate)) { - // private, default - } else if strings.EqualFold(aclStr, string(oss.ACLPublicRead)) { - acl = oss.ACLPublicRead - } else if strings.EqualFold(aclStr, string(oss.ACLPublicReadWrite)) { - acl = oss.ACLPublicReadWrite - } else { - return errors.Error("not supported acl") + acl, err := str2Acl(aclStr) + if err != nil { + return err } - opt := oss.ACL(acl) - opts = append(opts, opt) + opts = append(opts, oss.ACL(acl)) } err = osscli.CreateBucket(name, opts...) if err != nil { @@ -1031,7 +1045,7 @@ func (region *SRegion) IBucketExist(name string) (bool, error) { return exist, nil } -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { osscli, err := region.GetOssClient() if err != nil { return nil, errors.Wrap(err, "region.GetOssClient") diff --git a/pkg/util/aliyun/shell/bucket.go b/pkg/util/aliyun/shell/bucket.go new file mode 100644 index 0000000000..df5a4c18fa --- /dev/null +++ b/pkg/util/aliyun/shell/bucket.go @@ -0,0 +1,7 @@ +package shell + +import "yunion.io/x/onecloud/pkg/multicloud/objectstore" + +func init() { + objectstore.S3Shell() +} diff --git a/pkg/util/aws/bucket.go b/pkg/util/aws/bucket.go index 107119acba..612abe630a 100644 --- a/pkg/util/aws/bucket.go +++ b/pkg/util/aws/bucket.go @@ -1,9 +1,16 @@ package aws import ( + "context" "fmt" + "io" "time" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/s3" + + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -59,3 +66,129 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { }, } } + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + s3cli, err := b.region.GetS3Client() + if err != nil { + return result, errors.Wrap(err, "GetS3Client") + } + input := &s3.ListObjectsInput{} + input.SetBucket(b.Name) + if len(prefix) > 0 { + input.SetPrefix(prefix) + } + if len(marker) > 0 { + input.SetMarker(marker) + } + if len(delimiter) > 0 { + input.SetDelimiter(delimiter) + } + if maxCount > 0 { + input.SetMaxKeys(int64(maxCount)) + } + oResult, err := s3cli.ListObjects(input) + if err != nil { + return result, errors.Wrap(err, "ListObjects") + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + for _, object := range oResult.Contents { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: *object.StorageClass, + Key: *object.Key, + SizeBytes: *object.Size, + ETag: *object.ETag, + LastModified: *object.LastModified, + ContentType: "", + }, + } + result.Objects = append(result.Objects, obj) + } + if oResult.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(oResult.CommonPrefixes)) + for i, commPrefix := range oResult.CommonPrefixes { + result.CommonPrefixes[i] = &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: *commPrefix.Prefix}, + } + } + } + if oResult.IsTruncated != nil { + result.IsTruncated = *oResult.IsTruncated + } + if oResult.NextMarker != nil { + result.NextMarker = *oResult.NextMarker + } + return result, nil +} + +func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + return cloudprovider.GetIObjects(b, prefix, isRecursive) +} + +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + input := &s3.PutObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + input.SetBody(reader) + if len(storageClassStr) > 0 { + input.SetStorageClass(storageClassStr) + } + if len(contType) > 0 { + input.SetContentType(contType) + } + _, err = s3cli.PutObjectWithContext(ctx, input) + if err != nil { + return errors.Wrap(err, "PutObjectWithContext") + } + return nil +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + input := &s3.DeleteObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + _, err = s3cli.DeleteObjectWithContext(ctx, input) + if err != nil { + return errors.Wrap(err, "DeleteObject") + } + return nil +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return "", errors.Wrap(err, "GetS3Client") + } + var request *request.Request + switch method { + case "GET": + input := &s3.GetObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + request, _ = s3cli.GetObjectRequest(input) + case "PUT": + input := &s3.PutObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + request, _ = s3cli.PutObjectRequest(input) + case "DELETE": + input := &s3.DeleteObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + request, _ = s3cli.DeleteObjectRequest(input) + default: + return "", errors.Error("unsupported method") + } + return request.Presign(expire) +} diff --git a/pkg/util/aws/region.go b/pkg/util/aws/region.go index 43b86125fe..bb78f66e49 100644 --- a/pkg/util/aws/region.go +++ b/pkg/util/aws/region.go @@ -610,8 +610,8 @@ func (region *SRegion) IBucketExist(name string) (bool, error) { return true, nil } -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { - return cloudprovider.GetIBucketByName(region, name) +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + return cloudprovider.GetIBucketById(region, name) } func (region *SRegion) getBaseEndpoint() string { diff --git a/pkg/util/aws/s3object.go b/pkg/util/aws/s3object.go new file mode 100644 index 0000000000..7bf181e7e0 --- /dev/null +++ b/pkg/util/aws/s3object.go @@ -0,0 +1,13 @@ +package aws + +import "yunion.io/x/onecloud/pkg/cloudprovider" + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} diff --git a/pkg/util/aws/shell/bucket.go b/pkg/util/aws/shell/bucket.go new file mode 100644 index 0000000000..df5a4c18fa --- /dev/null +++ b/pkg/util/aws/shell/bucket.go @@ -0,0 +1,7 @@ +package shell + +import "yunion.io/x/onecloud/pkg/multicloud/objectstore" + +func init() { + objectstore.S3Shell() +} diff --git a/pkg/util/azure/blobobject.go b/pkg/util/azure/blobobject.go new file mode 100644 index 0000000000..509314d08e --- /dev/null +++ b/pkg/util/azure/blobobject.go @@ -0,0 +1,13 @@ +package azure + +import "yunion.io/x/onecloud/pkg/cloudprovider" + +type SObject struct { + container *SContainer + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.container.storageaccount +} diff --git a/pkg/util/azure/classic_disk.go b/pkg/util/azure/classic_disk.go index d150df9f79..c51cc97b7f 100644 --- a/pkg/util/azure/classic_disk.go +++ b/pkg/util/azure/classic_disk.go @@ -22,6 +22,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" + "github.com/Azure/azure-sdk-for-go/storage" billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -67,7 +68,7 @@ func (self *SRegion) GetStorageAccountDisksWithSnapshots(storageaccount SStorage } for _, container := range containers { if container.Name == "vhds" { - files, err := container.ListFiles() + files, err := container.ListAllFiles(&storage.IncludeBlobDataset{Snapshots: true, Metadata: true}) if err != nil { log.Errorf("List storage %s container %s files error: %v", storageaccount.Name, container.Name, err) return nil, nil, err diff --git a/pkg/util/azure/region.go b/pkg/util/azure/region.go index 45bfc648a2..1350f6ddb5 100644 --- a/pkg/util/azure/region.go +++ b/pkg/util/azure/region.go @@ -651,6 +651,6 @@ func (region *SRegion) IBucketExist(name string) (bool, error) { return region.checkStorageAccountNameExist(name) } -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { - return cloudprovider.GetIBucketByName(region, name) +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + return cloudprovider.GetIBucketById(region, name) } diff --git a/pkg/util/azure/shell/bucket.go b/pkg/util/azure/shell/bucket.go index ba7e570d52..ae85368d81 100644 --- a/pkg/util/azure/shell/bucket.go +++ b/pkg/util/azure/shell/bucket.go @@ -1,55 +1,9 @@ package shell import ( - "yunion.io/x/onecloud/pkg/util/azure" - "yunion.io/x/onecloud/pkg/util/printutils" - "yunion.io/x/onecloud/pkg/util/shellutils" + "yunion.io/x/onecloud/pkg/multicloud/objectstore" ) func init() { - type BucketListOptions struct { - } - shellutils.R(&BucketListOptions{}, "bucket-list", "List buckets", func(cli *azure.SRegion, args *BucketListOptions) error { - buckets, err := cli.GetIBuckets() - if err != nil { - return err - } - printutils.PrintGetterList(buckets, nil) - return nil - }) - - type BucketCreateOptions struct { - BUCKET string `help:"bucket name"` - STORAGECLASS string `help:"storage class"` - } - shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli *azure.SRegion, args *BucketCreateOptions) error { - err := cli.CreateIBucket(args.BUCKET, args.STORAGECLASS, "") - if err != nil { - return err - } - return nil - }) - - type BucketDeleteOptions struct { - BUCKET string `help:"bucket name"` - } - shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete a bucket", func(cli *azure.SRegion, args *BucketDeleteOptions) error { - err := cli.DeleteIBucket(args.BUCKET) - if err != nil { - return err - } - return nil - }) - - type BucketShowOptions struct { - BUCKET string `help:"bucket name"` - } - shellutils.R(&BucketShowOptions{}, "bucket-show", "Show a bucket", func(cli *azure.SRegion, args *BucketShowOptions) error { - bucket, err := cli.GetIBucketByName(args.BUCKET) - if err != nil { - return err - } - printObject(bucket) - return nil - }) + objectstore.S3Shell() } diff --git a/pkg/util/azure/shell/storageaccount.go b/pkg/util/azure/shell/storageaccount.go index ca1a3477f7..de6d79f668 100644 --- a/pkg/util/azure/shell/storageaccount.go +++ b/pkg/util/azure/shell/storageaccount.go @@ -98,7 +98,7 @@ func init() { if err != nil { return err } - blobs, err := container.ListFiles() + blobs, err := container.ListAllFiles(nil) if err != nil { return err } diff --git a/pkg/util/azure/storageaccount.go b/pkg/util/azure/storageaccount.go index 9c49d5977c..dfbd361496 100644 --- a/pkg/util/azure/storageaccount.go +++ b/pkg/util/azure/storageaccount.go @@ -15,7 +15,9 @@ package azure import ( + "context" "fmt" + "io" "math/rand" "path" "strings" @@ -27,9 +29,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/pkg/utils" ) type SContainer struct { @@ -75,7 +77,8 @@ type AccountProperties struct { } type SStorageAccount struct { - region *SRegion + region *SRegion + accountKey string Sku SSku `json:"sku,omitempty"` Kind string `json:"kind,omitempty"` @@ -416,18 +419,52 @@ func (self *SStorageAccount) GetContainer(name string) (*SContainer, error) { return nil, cloudprovider.ErrNotFound } -func (self *SContainer) ListFiles() ([]storage.Blob, error) { +func (self *SContainer) ListAllFiles(include *storage.IncludeBlobDataset) ([]storage.Blob, error) { + blobs := make([]storage.Blob, 0) + var marker string + for { + result, err := self.ListFiles("", marker, "", 5000, include) + if err != nil { + return nil, errors.Wrap(err, "ListFiles") + } + if len(result.Blobs) > 0 { + blobs = append(blobs, result.Blobs...) + } + if len(result.NextMarker) == 0 { + break + } else { + marker = result.NextMarker + } + } + return blobs, nil +} + +func (self *SContainer) ListFiles(prefix string, marker string, delimiter string, maxCount int, include *storage.IncludeBlobDataset) (storage.BlobListResponse, error) { + var result storage.BlobListResponse storageaccount := self.storageaccount client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env) if err != nil { - return nil, err + return result, err } blobService := client.GetBlobService() - result, err := blobService.GetContainerReference(self.Name).ListBlobs(storage.ListBlobsParameters{Include: &storage.IncludeBlobDataset{Snapshots: true, Metadata: true}}) - if err != nil { - return nil, err + params := storage.ListBlobsParameters{Include: include} + if len(prefix) > 0 { + params.Prefix = prefix } - return result.Blobs, nil + if len(marker) > 0 { + params.Marker = marker + } + if len(delimiter) > 0 { + params.Delimiter = delimiter + } + if maxCount > 0 { + params.MaxResults = uint(maxCount) + } + result, err = blobService.GetContainerReference(self.Name).ListBlobs(params) + if err != nil { + return result, err + } + return result, nil } func (self *SContainer) getClient() (storage.Client, error) { @@ -471,6 +508,51 @@ func (self *SContainer) CopySnapshot(snapshotId, fileName string) (*storage.Blob return blobRef, blobRef.GetProperties(&storage.GetBlobPropertiesOptions{}) } +func (self *SContainer) UploadStream(key string, reader io.ReadSeeker, contType string) error { + storageaccount := self.storageaccount + client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env) + if err != nil { + return errors.Wrap(err, "NewBasicClientOnSovereignCloud") + } + blobService := client.GetBlobService() + containerRef := blobService.GetContainerReference(self.Name) + blobRef := containerRef.GetBlobReference(key) + blobRef.Properties.BlobType = storage.BlobTypeBlock + blobRef.Properties.ContentType = contType + return blobRef.CreateBlockBlobFromReader(reader, &storage.PutBlobOptions{}) +} + +func (self *SContainer) SignUrl(method string, key string, expire time.Duration) (string, error) { + storageaccount := self.storageaccount + client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env) + if err != nil { + return "", errors.Wrap(err, "NewBasicClientOnSovereignCloud") + } + blobService := client.GetBlobService() + containerRef := blobService.GetContainerReference(self.Name) + sas := storage.ContainerSASOptions{} + sas.Start = time.Now() + sas.Expiry = sas.Start.Add(expire) + sas.UseHTTPS = true + sas.Identifier = key + switch method { + case "GET": + sas.Read = true + case "PUT": + sas.Read = true + sas.Add = true + sas.Create = true + sas.Write = true + case "DELETE": + sas.Read = true + sas.Write = true + sas.Delete = true + default: + return "", errors.Error("unsupport method") + } + return containerRef.GetSASURI(sas) +} + func (self *SContainer) UploadFile(filePath string) (string, error) { storageaccount := self.storageaccount client, err := storage.NewBasicClientOnSovereignCloud(storageaccount.Name, storageaccount.accountKey, storageaccount.region.client.env) @@ -525,11 +607,19 @@ func (self *SContainer) UploadFile(filePath string) (string, error) { } func (self *SStorageAccount) UploadFile(containerName string, filePath string) (string, error) { + container, err := self.getOrCreateContainer(containerName, true) + if err != nil { + return "", errors.Wrap(err, "getOrCreateContainer") + } + return container.UploadFile(filePath) +} + +func (self *SStorageAccount) getOrCreateContainer(containerName string, create bool) (*SContainer, error) { containers, err := self.GetContainers() if err != nil { - return "", err + return nil, errors.Wrap(err, "GetContainers") } - container := &SContainer{} + var container *SContainer find := false for i := 0; i < len(containers); i++ { if containers[i].Name == containerName { @@ -539,12 +629,23 @@ func (self *SStorageAccount) UploadFile(containerName string, filePath string) ( } } if !find { + if !create { + return nil, cloudprovider.ErrNotFound + } container, err = self.CreateContainer(containerName) if err != nil { - return "", err + return nil, errors.Wrap(err, "CreateContainer") } } - return container.UploadFile(filePath) + return container, nil +} + +func (self *SStorageAccount) UploadStream(containerName string, key string, reader io.ReadSeeker, contType string) error { + container, err := self.getOrCreateContainer(containerName, true) + if err != nil { + return errors.Wrap(err, "getOrCreateContainer") + } + return container.UploadStream(key, reader, contType) } func (b *SStorageAccount) GetProjectId() string { @@ -624,3 +725,178 @@ func (b *SStorageAccount) GetAccessUrls() []cloudprovider.SBucketAccessUrl { } return primary } + +func (b *SStorageAccount) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + return cloudprovider.GetIObjects(b, prefix, isRecursive) +} + +func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + containers, err := b.GetContainers() + if err != nil { + return result, errors.Wrap(err, "GetContainers") + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0) + for i := 0; i < len(containers); i += 1 { + container := containers[i] + matchLen := len(container.Name) + if matchLen > len(prefix) { + matchLen = len(prefix) + } + var subMarker string + if len(marker) < len(container.Name) { + if marker > container.Name { + continue + } + subMarker = "" + } else { + containerMarker := marker[:len(container.Name)] + if containerMarker > container.Name { + continue + } + subMarker = marker[len(container.Name)+1:] + } + if marker > container.Name { + continue + } + if maxCount <= 0 { + break + } + // container name matches prefix + if matchLen == 0 || container.Name[:matchLen] == prefix[:matchLen] { + if delimiter == "/" && (len(prefix) == 0 || prefix == container.Name+delimiter) { + // populate CommonPrefixes + o := &SObject{ + container: &container, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: container.Name + "/", + }, + } + result.CommonPrefixes = append(result.CommonPrefixes, o) + maxCount -= 1 + } else if len(prefix) <= len(container.Name)+1 { + // returns contain names only + o := &SObject{ + container: &container, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: container.Name + "/", + }, + } + result.Objects = append(result.Objects, o) + maxCount -= 1 + } + if delimiter == "" || len(prefix) >= len(container.Name)+1 { + subPrefix := "" + if len(prefix) >= len(container.Name) { + subPrefix = prefix[len(container.Name)+1:] + } + oResult, err := container.ListFiles(subPrefix, subMarker, delimiter, maxCount, nil) + if err != nil { + return result, errors.Wrap(err, "ListFiles") + } + for i := range oResult.Blobs { + blob := oResult.Blobs[i] + o := &SObject{ + container: &container, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: container.Name + "/" + blob.Name, + SizeBytes: blob.Properties.ContentLength, + StorageClass: "", + ETag: blob.Properties.Etag, + LastModified: time.Time(blob.Properties.LastModified), + ContentType: blob.Properties.ContentType, + }, + } + result.Objects = append(result.Objects, o) + maxCount -= 1 + if maxCount == 0 { + break + } + result.NextMarker = blob.Name + } + for i := range oResult.BlobPrefixes { + o := &SObject{ + container: &container, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: container.Name + "/" + oResult.BlobPrefixes[i], + }, + } + result.CommonPrefixes = append(result.CommonPrefixes, o) + maxCount -= 1 + } + if len(oResult.NextMarker) > 0 { + result.NextMarker = container.Name + "/" + oResult.NextMarker + result.IsTruncated = true + break + } + } + } + } + return result, nil +} + +func splitKey(key string) (string, string, error) { + slashPos := strings.IndexByte(key, '/') + if slashPos <= 0 { + return "", "", errors.Error("cannot put object to root") + } + containerName := key[:slashPos] + key = key[slashPos+1:] + if len(key) == 0 { + return "", "", errors.Error("empty blob path") + } + return containerName, key, nil +} + +func (b *SStorageAccount) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error { + containerName, blob, err := splitKey(key) + if err != nil { + return errors.Wrap(err, "splitKey") + } + err = b.UploadStream(containerName, blob, reader, contType) + if err != nil { + return errors.Wrap(err, "UploadStream") + } + return nil +} + +func (b *SStorageAccount) DeleteObject(ctx context.Context, key string) error { + containerName, blob, err := splitKey(key) + if err != nil { + return errors.Wrap(err, "splitKey") + } + client, err := storage.NewBasicClientOnSovereignCloud(b.Name, b.accountKey, b.region.client.env) + if err != nil { + return errors.Wrap(err, "storage.NewBasicClientOnSovereignCloud") + } + blobService := client.GetBlobService() + containerRef := blobService.GetContainerReference(containerName) + if len(blob) > 0 { + // delete object + blobRef := containerRef.GetBlobReference(blob) + _, err = blobRef.DeleteIfExists(nil) + if err != nil { + return errors.Wrap(err, "blobRef.DeleteIfExists") + } + } else { + // delete container + _, err = containerRef.DeleteIfExists(nil) + if err != nil { + return errors.Wrap(err, "containerRef.DeleteIfExists") + } + } + return nil +} + +func (b *SStorageAccount) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + containerName, blob, err := splitKey(key) + if err != nil { + return "", errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, false) + if err != nil { + return "", errors.Wrap(err, "getOrCreateContainer") + } + return container.SignUrl(method, blob, expire) +} diff --git a/pkg/util/httputils/httputils.go b/pkg/util/httputils/httputils.go index e722924237..d233d5fe95 100644 --- a/pkg/util/httputils/httputils.go +++ b/pkg/util/httputils/httputils.go @@ -127,8 +127,8 @@ func GetAddrPort(urlStr string) (string, int, error) { } } -func GetClient(insecure bool, timeout time.Duration) *http.Client { - tr := &http.Transport{ +func GetTransport(insecure bool, timeout time.Duration) *http.Transport { + return &http.Transport{ DialContext: (&net.Dialer{ Timeout: 5 * time.Second, }).DialContext, @@ -137,6 +137,10 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client { ExpectContinueTimeout: 1 * time.Second, TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, } +} + +func GetClient(insecure bool, timeout time.Duration) *http.Client { + tr := GetTransport(insecure, timeout) return &http.Client{ Transport: tr, Timeout: timeout, diff --git a/pkg/util/huawei/bucket.go b/pkg/util/huawei/bucket.go index 5d7a4b2002..9f5850658e 100644 --- a/pkg/util/huawei/bucket.go +++ b/pkg/util/huawei/bucket.go @@ -4,7 +4,12 @@ import ( "fmt" "time" + "yunion.io/x/pkg/errors" + + "context" + "io" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/util/huawei/obs" ) type SBucket struct { @@ -73,3 +78,126 @@ func (b *SBucket) GetSizeByte() int64 { func (b *SBucket) GetObjectNumber() int { return b.ObjectNumber } + +func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + return cloudprovider.GetIObjects(b, prefix, isRecursive) +} + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + obscli, err := b.region.getOBSClient() + if err != nil { + return result, errors.Wrap(err, "GetOBSClient") + } + input := &obs.ListObjectsInput{} + input.Bucket = b.Name + if len(prefix) > 0 { + input.Prefix = prefix + } + if len(marker) > 0 { + input.Marker = marker + } + if len(delimiter) > 0 { + input.Delimiter = delimiter + } + if maxCount > 0 { + input.MaxKeys = maxCount + } + oResult, err := obscli.ListObjects(input) + if err != nil { + return result, errors.Wrap(err, "ListObjects") + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + for _, object := range oResult.Contents { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: string(object.StorageClass), + Key: object.Key, + SizeBytes: object.Size, + ETag: object.ETag, + LastModified: object.LastModified, + ContentType: "", + }, + } + result.Objects = append(result.Objects, obj) + } + if oResult.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0) + for _, commonPrefix := range oResult.CommonPrefixes { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: commonPrefix, + }, + } + result.CommonPrefixes = append(result.CommonPrefixes, obj) + } + } + result.IsTruncated = oResult.IsTruncated + result.NextMarker = oResult.NextMarker + return result, nil +} + +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.PutObjectInput{} + input.Bucket = b.Name + input.Key = key + input.Body = reader + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return err + } + } + if len(contType) > 0 { + input.ContentType = contType + } + _, err = obscli.PutObject(input) + if err != nil { + return errors.Wrap(err, "PutObject") + } + return nil +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.DeleteObjectInput{} + input.Bucket = b.Name + input.Key = key + _, err = obscli.DeleteObject(input) + if err != nil { + return errors.Wrap(err, "DeleteObject") + } + return nil +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + input := obs.CreateSignedUrlInput{} + input.Bucket = b.Name + input.Key = key + input.Expires = int(expire / time.Second) + switch method { + case "GET": + input.Method = obs.HttpMethodGet + case "PUT": + input.Method = obs.HttpMethodPut + case "DELETE": + input.Method = obs.HttpMethodDelete + default: + return "", errors.Error("unsupported method") + } + output, err := obscli.CreateSignedUrl(&input) + return output.SignedUrl, nil +} diff --git a/pkg/util/huawei/object.go b/pkg/util/huawei/object.go new file mode 100644 index 0000000000..6bd0bb6c46 --- /dev/null +++ b/pkg/util/huawei/object.go @@ -0,0 +1,13 @@ +package huawei + +import "yunion.io/x/onecloud/pkg/cloudprovider" + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} diff --git a/pkg/util/huawei/region.go b/pkg/util/huawei/region.go index c0a04985ec..b6fdd67340 100644 --- a/pkg/util/huawei/region.go +++ b/pkg/util/huawei/region.go @@ -705,6 +705,18 @@ func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { return ret, nil } +func str2StorageClass(storageClassStr string) (obs.StorageClassType, error) { + if strings.EqualFold(storageClassStr, string(obs.StorageClassStandard)) { + return obs.StorageClassStandard, nil + } else if strings.EqualFold(storageClassStr, string(obs.StorageClassWarm)) { + return obs.StorageClassWarm, nil + } else if strings.EqualFold(storageClassStr, string(obs.StorageClassCold)) { + return obs.StorageClassCold, nil + } else { + return obs.StorageClassStandard, errors.Error("unsupported storageClass") + } +} + func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { obsClient, err := region.getOBSClient() if err != nil { @@ -725,14 +737,9 @@ func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr } } if len(storageClassStr) > 0 { - if strings.EqualFold(storageClassStr, string(obs.StorageClassStandard)) { - input.StorageClass = obs.StorageClassStandard - } else if strings.EqualFold(storageClassStr, string(obs.StorageClassWarm)) { - input.StorageClass = obs.StorageClassWarm - } else if strings.EqualFold(storageClassStr, string(obs.StorageClassCold)) { - input.StorageClass = obs.StorageClassCold - } else { - return errors.Error("unsupported storageClass") + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return err } } _, err = obsClient.CreateBucket(input) @@ -769,7 +776,7 @@ func (region *SRegion) IBucketExist(name string) (bool, error) { return true, nil } -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { obsClient, err := region.getOBSClient() if err != nil { return nil, errors.Wrap(err, "region.getOBSClient") diff --git a/pkg/util/huawei/shell/bucket.go b/pkg/util/huawei/shell/bucket.go new file mode 100644 index 0000000000..ae85368d81 --- /dev/null +++ b/pkg/util/huawei/shell/bucket.go @@ -0,0 +1,9 @@ +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/objectstore" +) + +func init() { + objectstore.S3Shell() +} diff --git a/pkg/util/huawei/shell/obs.go b/pkg/util/huawei/shell/obs.go index 07b87633b3..79bc38c8e1 100644 --- a/pkg/util/huawei/shell/obs.go +++ b/pkg/util/huawei/shell/obs.go @@ -30,7 +30,7 @@ func init() { BUCKET string `help:"bucket name to show"` } shellutils.R(&ObsBucketShowOptions{}, "obs-show", "Show bucket detail", func(cli *huawei.SRegion, args *ObsBucketShowOptions) error { - bucket, err := cli.GetIBucketByName(args.BUCKET) + bucket, err := cli.GetIBucketById(args.BUCKET) if err != nil { return err } diff --git a/pkg/util/openstack/region.go b/pkg/util/openstack/region.go index 313afdf711..be9bd57d3c 100644 --- a/pkg/util/openstack/region.go +++ b/pkg/util/openstack/region.go @@ -499,6 +499,6 @@ func (region *SRegion) IBucketExist(name string) (bool, error) { return false, cloudprovider.ErrNotImplemented } -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { return nil, cloudprovider.ErrNotImplemented } diff --git a/pkg/util/qcloud/bucket.go b/pkg/util/qcloud/bucket.go index 9743f78fb2..584bf73fb8 100644 --- a/pkg/util/qcloud/bucket.go +++ b/pkg/util/qcloud/bucket.go @@ -1,9 +1,16 @@ package qcloud import ( + "context" "fmt" + "io" "time" + "github.com/tencentyun/cos-go-sdk-v5" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -49,10 +56,14 @@ func (b *SBucket) GetAcl() string { return b.Acl } +func (b *SBucket) getBucketUrl() string { + return fmt.Sprintf("https://%s.%s", b.FullName, b.region.getCosEndpoint()) +} + func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { return []cloudprovider.SBucketAccessUrl{ { - Url: fmt.Sprintf("https://%s.%s", b.FullName, b.region.getCosEndpoint()), + Url: b.getBucketUrl(), Description: "bucket domain", }, { @@ -61,3 +72,112 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { }, } } + +func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + return cloudprovider.GetIObjects(b, prefix, isRecursive) +} + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + coscli, err := b.region.GetCosClient(b) + if err != nil { + return result, errors.Wrap(err, "GetCosClient") + } + opts := &cos.BucketGetOptions{} + if len(prefix) > 0 { + opts.Prefix = prefix + } + if len(marker) > 0 { + opts.Marker = marker + } + if len(delimiter) > 0 { + opts.Delimiter = delimiter + } + if maxCount > 0 { + opts.MaxKeys = maxCount + } + oResult, _, err := coscli.Bucket.Get(context.Background(), opts) + if err != nil { + return result, errors.Wrap(err, "coscli.Bucket.Get") + } + result.Objects = make([]cloudprovider.ICloudObject, 0) + for _, object := range oResult.Contents { + lastModified, _ := timeutils.ParseTimeStr(object.LastModified) + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: string(object.StorageClass), + Key: object.Key, + SizeBytes: int64(object.Size), + ETag: object.ETag, + LastModified: lastModified, + ContentType: "", + }, + } + result.Objects = append(result.Objects, obj) + } + if oResult.CommonPrefixes != nil { + result.CommonPrefixes = make([]cloudprovider.ICloudObject, 0) + for _, commPrefix := range oResult.CommonPrefixes { + obj := &SObject{ + bucket: b, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: commPrefix, + }, + } + result.CommonPrefixes = append(result.CommonPrefixes, obj) + } + } + result.IsTruncated = oResult.IsTruncated + result.NextMarker = oResult.NextMarker + return result, nil +} + +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + opts := &cos.ObjectPutOptions{} + if len(contType) > 0 { + opts.ContentType = contType + } + if len(storageClassStr) > 0 { + opts.XCosStorageClass = storageClassStr + } + _, err = coscli.Object.Put(ctx, key, reader, opts) + if err != nil { + return errors.Wrap(err, "coscli.Object.Put") + } + return nil +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + _, err = coscli.Object.Delete(ctx, key) + if err != nil { + return errors.Wrap(err, "coscli.Object.Delete") + } + return nil +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + if method != "GET" && method != "PUT" && method != "DELETE" { + return "", errors.Error("unsupported method") + } + coscli, err := b.region.GetCosClient(b) + if err != nil { + return "", errors.Wrap(err, "GetCosClient") + } + url, err := coscli.Object.GetPresignedURL(context.Background(), method, key, + b.region.client.SecretKey, + b.region.client.SecretID, + expire, nil) + if err != nil { + return "", errors.Wrap(err, "coscli.Object.GetPresignedURL") + } + return url.String(), nil +} diff --git a/pkg/util/qcloud/object.go b/pkg/util/qcloud/object.go new file mode 100644 index 0000000000..b9e9218c5c --- /dev/null +++ b/pkg/util/qcloud/object.go @@ -0,0 +1,13 @@ +package qcloud + +import "yunion.io/x/onecloud/pkg/cloudprovider" + +type SObject struct { + bucket *SBucket + + cloudprovider.SBaseCloudObject +} + +func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { + return o.bucket +} diff --git a/pkg/util/qcloud/qcloud.go b/pkg/util/qcloud/qcloud.go index 6cdf11145a..a35ff54e8c 100644 --- a/pkg/util/qcloud/qcloud.go +++ b/pkg/util/qcloud/qcloud.go @@ -20,9 +20,6 @@ import ( "strings" "time" - "yunion.io/x/pkg/utils" - - coslib "github.com/nelsonken/cos-go-sdk-v5/cos" "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" sdkerrors "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors" tchttp "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http" @@ -30,12 +27,9 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/pkg/util/timeutils" ) const ( @@ -587,103 +581,3 @@ func (client *SQcloudClient) GetIProjects() ([]cloudprovider.ICloudProject, erro } return iprojects, nil } - -func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { - ctx := appctx.Background - cos, err := region.GetCosClient() - if err != nil { - return nil, errors.Wrap(err, "GetCosClient") - } - result, err := cos.GetBucketList(ctx) - if err != nil { - return nil, errors.Wrap(err, "GetBucketList") - } - ret := make([]cloudprovider.ICloudBucket, 0) - for i := range result.Buckets.Bucket { - bInfo := result.Buckets.Bucket[i] - // ignore buckets not belong to this region - if bInfo.Location != region.GetId() { - continue - } - createAt, _ := timeutils.ParseTimeStr(bInfo.CreateDate) - name := bInfo.Name - // name = name[:len(name)-len(result.Owner.ID)-1] - name = name[:strings.LastIndexByte(name, '-')] - b := SBucket{ - region: region, - - Name: name, - FullName: bInfo.Name, - Location: bInfo.Location, - CreateDate: createAt, - } - ret = append(ret, &b) - } - return ret, nil -} - -func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { - ctx := appctx.Background - cos, err := region.GetCosClient() - if err != nil { - return errors.Wrap(err, "GetCosClient") - } - acl := &coslib.AccessControl{} - if len(aclStr) > 0 { - if utils.IsInStringArray(aclStr, []string{ - "private", "public-read", "public-read-write", "authenticated-read", - }) { - acl.ACL = aclStr - } else { - return errors.Error("invalid acl") - } - } - err = cos.CreateBucket(ctx, name, acl) - if err != nil { - return errors.Wrap(err, "oss.CreateBucket") - } - return nil -} - -func cosHttpCode(err error) int { - if httpErr, ok := err.(coslib.HTTPError); ok { - return httpErr.Code - } - if httpErr, ok := err.(*coslib.HTTPError); ok { - return httpErr.Code - } - return -1 -} - -func (region *SRegion) DeleteIBucket(name string) error { - ctx := appctx.Background - cos, err := region.GetCosClient() - if err != nil { - return errors.Wrap(err, "GetCosClient") - } - err = cos.DeleteBucket(ctx, name) - if err != nil { - if cosHttpCode(err) == 404 { - return nil - } - return errors.Wrap(err, "DeleteBucket") - } - return nil -} - -func (region *SRegion) IBucketExist(name string) (bool, error) { - ctx := appctx.Background - cos, err := region.GetCosClient() - if err != nil { - return false, errors.Wrap(err, "GetCosClient") - } - err = cos.BucketExists(ctx, name) - if err != nil { - return false, errors.Wrap(err, "BucketExists") - } - return true, nil -} - -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { - return cloudprovider.GetIBucketByName(region, name) -} diff --git a/pkg/util/qcloud/region.go b/pkg/util/qcloud/region.go index 7433d7d201..13aea25b99 100644 --- a/pkg/util/qcloud/region.go +++ b/pkg/util/qcloud/region.go @@ -15,15 +15,22 @@ package qcloud import ( + "context" "fmt" + "net/http" + "net/url" + "strings" "time" - "github.com/nelsonken/cos-go-sdk-v5/cos" + "github.com/tencentyun/cos-go-sdk-v5" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/timeutils" "yunion.io/x/pkg/utils" + "github.com/tencentyun/cos-go-sdk-v5/debug" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" @@ -32,8 +39,7 @@ import ( type SRegion struct { multicloud.SRegion - client *SQcloudClient - cosClient *cos.Client + client *SQcloudClient izones []cloudprovider.ICloudZone ivpcs []cloudprovider.ICloudVpc @@ -266,16 +272,30 @@ func (self *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudpro return self.GetIVpcById(vpcId) } -func (self *SRegion) GetCosClient() (*cos.Client, error) { - if self.cosClient == nil { - self.cosClient = cos.New(&cos.Option{ - AppID: self.client.AppID, - SecretID: self.client.SecretID, - SecretKey: self.client.SecretKey, - Region: self.Region, - }) +func (self *SRegion) GetCosClient(bucket *SBucket) (*cos.Client, error) { + var baseUrl *cos.BaseURL + if bucket != nil { + u, _ := url.Parse(bucket.getBucketUrl()) + baseUrl = &cos.BaseURL{ + BucketURL: u, + } } - return self.cosClient, nil + cosClient := cos.NewClient( + baseUrl, + &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: self.client.SecretID, + SecretKey: self.client.SecretKey, + Transport: &debug.DebugRequestTransport{ + RequestHeader: self.client.Debug, + RequestBody: self.client.Debug, + ResponseHeader: self.client.Debug, + ResponseBody: self.client.Debug, + }, + }, + }, + ) + return cosClient, nil } func (self *SRegion) GetClient() *SQcloudClient { @@ -820,3 +840,111 @@ func (self *SRegion) QueryAccountBalance() (*SAccountBalance, error) { func (self *SRegion) getCosEndpoint() string { return fmt.Sprintf("cos.%s.myqcloud.com", self.GetId()) } + +func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { + coscli, err := region.GetCosClient(nil) + if err != nil { + return nil, errors.Wrap(err, "GetCosClient") + } + s, _, err := coscli.Service.Get(context.Background()) + if err != nil { + return nil, errors.Wrap(err, "coscli.Service.Get") + } + ret := make([]cloudprovider.ICloudBucket, 0) + for i := range s.Buckets { + bInfo := s.Buckets[i] + // ignore buckets not belong to this region + if bInfo.Region != region.GetId() { + continue + } + createAt, _ := timeutils.ParseTimeStr(bInfo.CreationDate) + name := bInfo.Name + // name = name[:len(name)-len(result.Owner.ID)-1] + name = name[:strings.LastIndexByte(name, '-')] + b := SBucket{ + region: region, + + Name: name, + FullName: bInfo.Name, + Location: bInfo.Region, + CreateDate: createAt, + } + ret = append(ret, &b) + } + return ret, nil +} + +func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { + bucket := &SBucket{ + region: region, + Name: name, + FullName: fmt.Sprintf("%s-%s", name, region.client.AppID), + } + coscli, err := region.GetCosClient(bucket) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + opts := &cos.BucketPutOptions{} + if len(aclStr) > 0 { + if utils.IsInStringArray(aclStr, []string{ + "private", "public-read", "public-read-write", "authenticated-read", + }) { + opts.XCosACL = aclStr + } else { + return errors.Error("invalid acl") + } + } + _, err = coscli.Bucket.Put(context.Background(), opts) + if err != nil { + return errors.Wrap(err, "coscli.Bucket.Put") + } + return nil +} + +func cosHttpCode(err error) int { + if httpErr, ok := err.(*cos.ErrorResponse); ok { + return httpErr.Response.StatusCode + } + return -1 +} + +func (region *SRegion) DeleteIBucket(name string) error { + bucket := &SBucket{ + region: region, + Name: name, + FullName: fmt.Sprintf("%s-%s", name, region.client.AppID), + } + coscli, err := region.GetCosClient(bucket) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + _, err = coscli.Bucket.Delete(context.Background()) + if err != nil { + if cosHttpCode(err) == 404 { + return nil + } + return errors.Wrap(err, "DeleteBucket") + } + return nil +} + +func (region *SRegion) IBucketExist(name string) (bool, error) { + bucket := &SBucket{ + region: region, + Name: name, + FullName: fmt.Sprintf("%s-%s", name, region.client.AppID), + } + coscli, err := region.GetCosClient(bucket) + if err != nil { + return false, errors.Wrap(err, "GetCosClient") + } + _, err = coscli.Bucket.Head(context.Background()) + if err != nil { + return false, errors.Wrap(err, "BucketExists") + } + return true, nil +} + +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + return cloudprovider.GetIBucketById(region, name) +} diff --git a/pkg/util/qcloud/shell/cos.go b/pkg/util/qcloud/shell/cos.go index eaf489cf00..9482e9f16d 100644 --- a/pkg/util/qcloud/shell/cos.go +++ b/pkg/util/qcloud/shell/cos.go @@ -15,136 +15,9 @@ package shell import ( - "context" - "os" - - coslib "github.com/nelsonken/cos-go-sdk-v5/cos" - - "yunion.io/x/onecloud/pkg/util/printutils" - "yunion.io/x/onecloud/pkg/util/qcloud" - "yunion.io/x/onecloud/pkg/util/shellutils" + "yunion.io/x/onecloud/pkg/multicloud/objectstore" ) func init() { - type CosListOptions struct { - } - shellutils.R(&CosListOptions{}, "cos-list", "List COS buckets", func(cli *qcloud.SRegion, args *CosListOptions) error { - buckets, err := cli.GetIBuckets() - if err != nil { - return err - } - printList(buckets, 0, 0, 0, nil) - return nil - }) - - shellutils.R(&CosListOptions{}, "bucket-list", "List COS buckets", func(cli *qcloud.SRegion, args *CosListOptions) error { - buckets, err := cli.GetIBuckets() - if err != nil { - return err - } - printutils.PrintGetterList(buckets, nil) - return nil - }) - - type CosCreateBucketOptions struct { - BUCKET string `help:"name of bucket to create"` - Acl string `help:"Acl"` - } - shellutils.R(&CosCreateBucketOptions{}, "cos-create-bucket", "Create a COS bucket", func(cli *qcloud.SRegion, args *CosCreateBucketOptions) error { - err := cli.CreateIBucket(args.BUCKET, "", args.Acl) - if err != nil { - return err - } - return nil - }) - - type CosDeleteBucketOptions struct { - BUCKET string `help:"name of bucket to delete"` - } - shellutils.R(&CosDeleteBucketOptions{}, "cos-delete-bucket", "Delete a COS bucket", func(cli *qcloud.SRegion, args *CosDeleteBucketOptions) error { - err := cli.DeleteIBucket(args.BUCKET) - if err != nil { - return err - } - return nil - }) - - type CosListBucketOptions struct { - BUCKET string `help:"bucket name"` - } - - shellutils.R(&CosListBucketOptions{}, "cos-bucket-list", "List content of a OSS bucket", func(cli *qcloud.SRegion, args *CosListBucketOptions) error { - cos, err := cli.GetCosClient() - if err != nil { - return err - } - result, err := cos.ListBucketContents(context.Background(), args.BUCKET, &coslib.QueryCondition{}) - if err != nil { - return err - } - printList(result.Contents, len(result.Contents), 0, len(result.Contents), nil) - return nil - }) - - shellutils.R(&CosListBucketOptions{}, "cos-bucket-create", "Create a OSS bucket", func(cli *qcloud.SRegion, args *CosListBucketOptions) error { - cos, err := cli.GetCosClient() - if err != nil { - return err - } - return cos.CreateBucket(context.Background(), args.BUCKET, &coslib.AccessControl{}) - }) - - type CosUploadOptions struct { - BUCKET string `help:"bucket name"` - KEY string `help:"Object key"` - FILE string `help:"Local file path"` - Acl string `help:"Object ACL" choices:"private|public-read|public-read-write"` - } - shellutils.R(&CosUploadOptions{}, "cos-upload", "Upload a file to a Cos bucket", func(cli *qcloud.SRegion, args *CosUploadOptions) error { - cos, err := cli.GetCosClient() - if err != nil { - return err - } - return cos.Bucket(args.BUCKET).UploadObjectBySlice(context.Background(), args.KEY, args.FILE, 3, nil) - }) - - type CosDownloadOptions struct { - BUCKET string `help:"bucket name"` - NAME string `help:"File name"` - } - shellutils.R(&CosDownloadOptions{}, "cos-download", "Download a file", func(cli *qcloud.SRegion, args *CosDownloadOptions) error { - cos, err := cli.GetCosClient() - if err != nil { - return err - } - //file - return cos.Bucket(args.BUCKET).DownloadObject(context.Background(), args.NAME, os.Stdout) - //return cos.Bucket(args.BUCKET).UploadObjectBySlice(context.Background(), args.KEY, args.FILE, 3, nil) - }) - - type CosObjectAclOptions struct { - BUCKET string `help:"bucket name"` - KEY string `help:"object key"` - ACL string `help:"ACL" choices:"private|public-read|public-read-write"` - } - shellutils.R(&CosObjectAclOptions{}, "cos-set-acl", "Set acl for a object", func(cli *qcloud.SRegion, args *CosObjectAclOptions) error { - cos, err := cli.GetCosClient() - if err != nil { - return err - } - return cos.SetBucketACL(context.Background(), args.KEY, &coslib.AccessControl{ACL: args.ACL}) - }) - - type CosDeleteOptions struct { - BUCKET string `help:"bucket name"` - KEY string `help:"Object key"` - } - - shellutils.R(&CosDeleteOptions{}, "cos-delete", "Delete a file from a Cos bucket", func(cli *qcloud.SRegion, args *CosDeleteOptions) error { - cos, err := cli.GetCosClient() - if err != nil { - return err - } - return cos.Bucket(args.BUCKET).DeleteObject(context.Background(), args.KEY) - }) + objectstore.S3Shell() } diff --git a/pkg/util/qcloud/storagecache.go b/pkg/util/qcloud/storagecache.go index 962f57a101..57de5605d3 100644 --- a/pkg/util/qcloud/storagecache.go +++ b/pkg/util/qcloud/storagecache.go @@ -18,15 +18,13 @@ import ( "context" "fmt" "io" - "os" "strings" "time" - coslib "github.com/nelsonken/cos-go-sdk-v5/cos" - "yunion.io/x/jsonutils" "yunion.io/x/log" + "github.com/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/options" @@ -165,33 +163,16 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To // first upload image to oss s := auth.GetAdminSession(ctx, options.Options.Region, "") - meta, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) + _, reader, err := modules.Images.Download(s, image.ImageId, string(qemuimg.VMDK), false) if err != nil { return "", err } - tmpFile := fmt.Sprintf("%s/%s", options.Options.TempPath, image.ImageId) - defer os.Remove(tmpFile) - f, err := os.Create(tmpFile) - if err != nil { - return "", err - } - defer f.Close() - if _, err := io.Copy(f, reader); err != nil { - return "", err - } - - log.Infof("meta data %s", meta) - cos, err := self.region.GetCosClient() - if err != nil { - log.Errorf("GetOssClient err %s", err) - return "", err - } - bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.GetId())) - err = cos.BucketExists(context.Background(), bucketName) - if err != nil { + bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s-%s", self.region.GetId(), image.ImageId)) + exists, _ := self.region.IBucketExist(bucketName) + if !exists { log.Debugf("Bucket %s not exists, to create ...", bucketName) - err := cos.CreateBucket(context.Background(), bucketName, &coslib.AccessControl{ACL: "public-read"}) + err := self.region.CreateIBucket(bucketName, "", "public-read") if err != nil { log.Errorf("Create bucket error %s", err) return "", err @@ -199,14 +180,19 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To } else { log.Debugf("Bucket %s exists", bucketName) } + defer self.region.DeleteIBucket(bucketName) log.Debugf("To upload image to bucket %s ...", bucketName) - err = cos.Bucket(bucketName).UploadObjectBySlice(context.Background(), image.ImageId, tmpFile, 3, map[string]string{}) + bucket, err := self.region.GetIBucketById(bucketName) + if err != nil { + return "", errors.Wrap(err, "GetIBucketByName") + } + err = bucket.PutObject(context.Background(), image.ImageId, reader.(io.ReadSeeker), "", "") if err != nil { log.Errorf("UploadObject error %s %s", image.ImageId, err) - return "", err + return "", errors.Wrap(err, "bucket.PutObject") } - defer cos.Bucket(bucketName).DeleteObject(context.Background(), image.ImageId) + defer bucket.DeleteObject(context.Background(), image.ImageId) // 腾讯云镜像名称需要小于20个字符 imageBaseName := image.ImageId[:10] diff --git a/pkg/util/ucloud/region.go b/pkg/util/ucloud/region.go index 137b842a18..b813a770fe 100644 --- a/pkg/util/ucloud/region.go +++ b/pkg/util/ucloud/region.go @@ -636,6 +636,7 @@ func (self *SRegion) GetClient() *SUcloudClient { return self.client } +// https://docs.ucloud.cn/api/ufile-api/describe_bucket func (region *SRegion) listBuckets(name string, offset int, limit int) ([]SBucket, error) { params := NewUcloudParams() if len(name) > 0 { @@ -645,7 +646,8 @@ func (region *SRegion) listBuckets(name string, offset int, limit int) ([]SBucke params.Set("Offset", offset) } buckets := make([]SBucket, 0) - err := region.DoAction("DescribeBucket", params, &buckets) + // request without RegionId + err := region.client.DoAction("DescribeBucket", params, &buckets) if err != nil { return nil, errors.Wrap(err, "DoAction DescribeBucket") } @@ -673,6 +675,7 @@ func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { ret := make([]cloudprovider.ICloudBucket, len(buckets)) for i := range buckets { buckets[i].region = region + buckets[i].projectId = region.client.projectId ret[i] = &buckets[i] } return ret, nil @@ -707,15 +710,6 @@ func (region *SRegion) IBucketExist(name string) (bool, error) { return true, nil } -func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { - parts, err := region.listBuckets(name, 0, 1) - if err != nil { - return nil, errors.Wrap(err, "region.listBuckets") - } - if len(parts) == 0 { - return nil, cloudprovider.ErrNotFound - } - bucket := parts[0] - bucket.region = region - return &bucket, nil +func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + return cloudprovider.GetIBucketById(region, name) } diff --git a/pkg/util/ucloud/shell/ufile.go b/pkg/util/ucloud/shell/ufile.go index 6045d06a22..ae85368d81 100644 --- a/pkg/util/ucloud/shell/ufile.go +++ b/pkg/util/ucloud/shell/ufile.go @@ -1,43 +1,9 @@ package shell import ( - "yunion.io/x/onecloud/pkg/util/printutils" - "yunion.io/x/onecloud/pkg/util/shellutils" - "yunion.io/x/onecloud/pkg/util/ucloud" + "yunion.io/x/onecloud/pkg/multicloud/objectstore" ) func init() { - type UFileBucketListOptions struct { - } - shellutils.R(&UFileBucketListOptions{}, "bucket-list", "List buckets", func(cli *ucloud.SRegion, args *UFileBucketListOptions) error { - buckets, err := cli.GetIBuckets() - if err != nil { - return err - } - printutils.PrintGetterList(buckets, nil) - return nil - }) - - type UFileBucketCreateOptions struct { - BUCKET string `help:"Name of bucket"` - Acl string `help:"Acl" choices:"private|public"` - } - shellutils.R(&UFileBucketCreateOptions{}, "bucket-create", "create a bucket", func(cli *ucloud.SRegion, args *UFileBucketCreateOptions) error { - err := cli.CreateIBucket(args.BUCKET, "", args.Acl) - if err != nil { - return err - } - return nil - }) - - type UFileBucketDeleteOptions struct { - BUCKET string `help:"Name of bucket"` - } - shellutils.R(&UFileBucketDeleteOptions{}, "bucket-delete", "delete a bucket", func(cli *ucloud.SRegion, args *UFileBucketDeleteOptions) error { - err := cli.DeleteIBucket(args.BUCKET) - if err != nil { - return err - } - return nil - }) + objectstore.S3Shell() } diff --git a/pkg/util/ucloud/storagecache.go b/pkg/util/ucloud/storagecache.go index 7720a5395f..a4f3a4d80f 100644 --- a/pkg/util/ucloud/storagecache.go +++ b/pkg/util/ucloud/storagecache.go @@ -143,10 +143,14 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To bucketName := GetBucketName(self.region.GetId(), image.ImageId) // create bucket - if _, err := self.region.GetBucketDomain(bucketName); err != nil { + exist, err := self.region.IBucketExist(bucketName) + if err != nil { + return "", errors.Wrap(err, "self.region.IBucketExist") + } + if !exist { err = self.region.CreateBucket(bucketName, "private") if err != nil { - return "", err + return "", errors.Wrap(err, "CreateBucket") } } defer func() { @@ -165,7 +169,7 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To log.Debugf("Images meta data %s", meta) minDiskMB, _ := meta.Int("min_disk") minDiskGB := int64(math.Ceil(float64(minDiskMB) / 1024)) - // 在使用OBS桶的外部镜像文件制作镜像时生效且为必选字段。取值为40～1024GB。 + if minDiskGB < 40 { minDiskGB = 40 } else if minDiskGB > 1024 { @@ -175,13 +179,16 @@ func (self *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.To md5, _ := meta.GetString("checksum") diskFormat, _ := meta.GetString("disk_format") // upload to ucloud + bucket, err := self.region.GetIBucketById(bucketName) + if err != nil { + return "", errors.Wrap(err, "GetIBucketByName") + } file := SFile{ - region: self.region, - BucketName: bucketName, - File: reader, - FileSize: size, - FileName: image.ImageId, - FileMD5: md5, + bucket: bucket.(*SBucket), + File: reader, + FileSize: size, + FileName: image.ImageId, + FileMD5: md5, } err = file.Upload() @@ -245,8 +252,7 @@ func (self *SRegion) createIImage(snapshotId, imageName, imageDesc string) (stri return "", cloudprovider.ErrNotSupported } -// https://docs.ucloud.cn/api/ufile-api/describe_bucket -func (self *SRegion) GetBucketDomain(name string) (string, error) { +/*func (self *SRegion) GetBucketDomain(name string) (string, error) { params := NewUcloudParams() params.Set("BucketName", name) @@ -261,7 +267,7 @@ func (self *SRegion) GetBucketDomain(name string) (string, error) { } else { return "", fmt.Errorf("GetBucketDomain failed. %v", res) } -} +}*/ // https://docs.ucloud.cn/api/ufile-api/create_bucket func (self *SRegion) CreateBucket(name, bucketType string) error { diff --git a/pkg/util/ucloud/ufile.go b/pkg/util/ucloud/ufile.go index 3179211498..82711bda62 100644 --- a/pkg/util/ucloud/ufile.go +++ b/pkg/util/ucloud/ufile.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/log" + "context" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/util/httputils" ) @@ -34,6 +35,8 @@ import ( type SBucket struct { region *SRegion + projectId string + Domain Domain `json:"Domain"` BucketID string `json:"BucketId"` Region string `json:"Region"` @@ -55,13 +58,13 @@ type Domain struct { } type SFile struct { - region *SRegion + bucket *SBucket - BucketName string - File io.Reader - FileSize int64 - FileName string - FileMD5 string + // BucketName string + File io.Reader + FileSize int64 + FileName string + FileMD5 string } func (self *SFile) signHeader(httpMethod string) string { @@ -76,24 +79,25 @@ func (self *SFile) signHeader(httpMethod string) string { data += md5 + "\n" data += contentType + "\n" data += "\n" - data += "/" + self.BucketName + "/" + self.FileName + data += "/" + self.bucket.BucketName + "/" + self.FileName - h := hmac.New(sha1.New, []byte(self.region.client.accessKeySecret)) + h := hmac.New(sha1.New, []byte(self.bucket.region.client.accessKeySecret)) h.Write([]byte(data)) return base64.StdEncoding.EncodeToString(h.Sum(nil)) } func (self *SFile) auth(httpMethod string) string { - return "UCloud" + " " + self.region.client.accessKeyId + ":" + self.signHeader(httpMethod) + return "UCloud" + " " + self.bucket.region.client.accessKeyId + ":" + self.signHeader(httpMethod) } func (self *SFile) GetHost() string { - host, err := self.region.GetBucketDomain(self.BucketName) + return self.bucket.Domain.Src[0] + /*host, err := self.bucket.region.GetBucketDomain(self.BucketName) if err != nil { log.Errorf("SFile GetHost %s", err) return "" } - return host + return host*/ } func (self *SFile) GetUrl() string { @@ -105,13 +109,13 @@ func (self *SFile) FetchFileUrl() string { expired := strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10) // sign data := "GET\n\n\n" + expired + "\n" - data += "/" + self.BucketName + "/" + self.FileName - h := hmac.New(sha1.New, []byte(self.region.client.accessKeySecret)) + data += "/" + self.bucket.BucketName + "/" + self.FileName + h := hmac.New(sha1.New, []byte(self.bucket.region.client.accessKeySecret)) h.Write([]byte(data)) sign := base64.StdEncoding.EncodeToString(h.Sum(nil)) urlEncoder := url.Values{} - urlEncoder.Add("UCloudPublicKey", self.region.client.accessKeyId) + urlEncoder.Add("UCloudPublicKey", self.bucket.region.client.accessKeyId) urlEncoder.Add("Signature", sign) urlEncoder.Add("Expires", expired) querys := urlEncoder.Encode() @@ -135,7 +139,7 @@ func (self *SFile) Delete() error { } func (self *SFile) request(req *http.Request) error { - res, err := http.DefaultClient.Do(req) + res, err := httputils.GetDefaultClient().Do(req) if err != nil { return err } @@ -150,11 +154,11 @@ func (self *SFile) request(req *http.Request) error { } func (b *SBucket) GetProjectId() string { - return "" + return b.projectId } func (b *SBucket) GetGlobalId() string { - return b.BucketName + return b.BucketID } func (b *SBucket) GetName() string { @@ -181,20 +185,51 @@ func (b *SBucket) GetAcl() string { return b.Type } +func (b *SBucket) getSrcUrl() string { + if len(b.Domain.Src) > 0 { + return b.Domain.Src[0] + } + return "" +} + func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { ret := make([]cloudprovider.SBucketAccessUrl, 0) - regionId := b.region.GetId() - // hack, remove trailing digits - for len(regionId) > 0 { - lastDigit := regionId[len(regionId)-1] - if lastDigit >= '0' && lastDigit <= '9' { - regionId = regionId[:len(regionId)-1] - } else { - break - } + for i, u := range b.Domain.Src { + ret = append(ret, cloudprovider.SBucketAccessUrl{ + Url: u, + Description: fmt.Sprintf("src%d", i), + }) + } + for i, u := range b.Domain.CDN { + ret = append(ret, cloudprovider.SBucketAccessUrl{ + Url: u, + Description: fmt.Sprintf("cdn%d", i), + }) } - ret = append(ret, cloudprovider.SBucketAccessUrl{ - Url: fmt.Sprintf("https://%s.%s.ufileos.com", b.BucketName, regionId), - }) return ret } + +func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { + return cloudprovider.GetIObjects(b, prefix, isRecursive) +} + +func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { + result := cloudprovider.SListObjectResult{} + return result, cloudprovider.ErrNotSupported +} + +func (b *SBucket) PutObject(ctx context.Context, key string, reader io.ReadSeeker, contType string, storageClassStr string) error { + return cloudprovider.ErrNotSupported +} + +func (b *SBucket) DeleteObject(ctx context.Context, key string) error { + file := SFile{ + bucket: b, + FileName: key, + } + return file.Delete() +} + +func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { + return "", cloudprovider.ErrNotSupported +} diff --git a/vendor/github.com/google/go-querystring/LICENSE b/vendor/github.com/google/go-querystring/LICENSE new file mode 100644 index 0000000000..ae121a1e46 --- /dev/null +++ b/vendor/github.com/google/go-querystring/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 Google. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/google/go-querystring/query/encode.go b/vendor/github.com/google/go-querystring/query/encode.go new file mode 100644 index 0000000000..37080b19b5 --- /dev/null +++ b/vendor/github.com/google/go-querystring/query/encode.go @@ -0,0 +1,320 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package query implements encoding of structs into URL query parameters. +// +// As a simple example: +// +// type Options struct { +// Query string `url:"q"` +// ShowAll bool `url:"all"` +// Page int `url:"page"` +// } +// +// opt := Options{ "foo", true, 2 } +// v, _ := query.Values(opt) +// fmt.Print(v.Encode()) // will output: "q=foo&all=true&page=2" +// +// The exact mapping between Go values and url.Values is described in the +// documentation for the Values() function. +package query + +import ( + "bytes" + "fmt" + "net/url" + "reflect" + "strconv" + "strings" + "time" +) + +var timeType = reflect.TypeOf(time.Time{}) + +var encoderType = reflect.TypeOf(new(Encoder)).Elem() + +// Encoder is an interface implemented by any type that wishes to encode +// itself into URL values in a non-standard way. +type Encoder interface { + EncodeValues(key string, v *url.Values) error +} + +// Values returns the url.Values encoding of v. +// +// Values expects to be passed a struct, and traverses it recursively using the +// following encoding rules. +// +// Each exported struct field is encoded as a URL parameter unless +// +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option +// +// The empty values are false, 0, any nil pointer or interface value, any array +// slice, map, or string of length zero, and any time.Time that returns true +// for IsZero(). +// +// The URL parameter name defaults to the struct field name but can be +// specified in the struct field's tag value. The "url" key in the struct +// field's tag value is the key name, followed by an optional comma and +// options. For example: +// +// // Field is ignored by this package. +// Field int `url:"-"` +// +// // Field appears as URL parameter "myName". +// Field int `url:"myName"` +// +// // Field appears as URL parameter "myName" and the field is omitted if +// // its value is empty +// Field int `url:"myName,omitempty"` +// +// // Field appears as URL parameter "Field" (the default), but the field +// // is skipped if empty. Note the leading comma. +// Field int `url:",omitempty"` +// +// For encoding individual field values, the following type-dependent rules +// apply: +// +// Boolean values default to encoding as the strings "true" or "false". +// Including the "int" option signals that the field should be encoded as the +// strings "1" or "0". +// +// time.Time values default to encoding as RFC3339 timestamps. Including the +// "unix" option signals that the field should be encoded as a Unix time (see +// time.Unix()) +// +// Slice and Array values default to encoding as multiple URL values of the +// same name. Including the "comma" option signals that the field should be +// encoded as a single comma-delimited value. Including the "space" option +// similarly encodes the value as a single space-delimited string. Including +// the "semicolon" option will encode the value as a semicolon-delimited string. +// Including the "brackets" option signals that the multiple URL values should +// have "[]" appended to the value name. "numbered" will append a number to +// the end of each incidence of the value name, example: +// name0=value0&name1=value1, etc. +// +// Anonymous struct fields are usually encoded as if their inner exported +// fields were fields in the outer struct, subject to the standard Go +// visibility rules. An anonymous struct field with a name given in its URL +// tag is treated as having that name, rather than being anonymous. +// +// Non-nil pointer values are encoded as the value pointed to. +// +// Nested structs are encoded including parent fields in value names for +// scoping. e.g: +// +// "user[name]=acme&user[addr][postcode]=1234&user[addr][city]=SFO" +// +// All other values are encoded using their default string representation. +// +// Multiple fields that encode to the same URL parameter name will be included +// as multiple URL values of the same name. +func Values(v interface{}) (url.Values, error) { + values := make(url.Values) + val := reflect.ValueOf(v) + for val.Kind() == reflect.Ptr { + if val.IsNil() { + return values, nil + } + val = val.Elem() + } + + if v == nil { + return values, nil + } + + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("query: Values() expects struct input. Got %v", val.Kind()) + } + + err := reflectValue(values, val, "") + return values, err +} + +// reflectValue populates the values parameter from the struct fields in val. +// Embedded structs are followed recursively (using the rules defined in the +// Values function documentation) breadth-first. +func reflectValue(values url.Values, val reflect.Value, scope string) error { + var embedded []reflect.Value + + typ := val.Type() + for i := 0; i < typ.NumField(); i++ { + sf := typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + + sv := val.Field(i) + tag := sf.Tag.Get("url") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if name == "" { + if sf.Anonymous && sv.Kind() == reflect.Struct { + // save embedded struct for later processing + embedded = append(embedded, sv) + continue + } + + name = sf.Name + } + + if scope != "" { + name = scope + "[" + name + "]" + } + + if opts.Contains("omitempty") && isEmptyValue(sv) { + continue + } + + if sv.Type().Implements(encoderType) { + if !reflect.Indirect(sv).IsValid() { + sv = reflect.New(sv.Type().Elem()) + } + + m := sv.Interface().(Encoder) + if err := m.EncodeValues(name, &values); err != nil { + return err + } + continue + } + + if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array { + var del byte + if opts.Contains("comma") { + del = ',' + } else if opts.Contains("space") { + del = ' ' + } else if opts.Contains("semicolon") { + del = ';' + } else if opts.Contains("brackets") { + name = name + "[]" + } + + if del != 0 { + s := new(bytes.Buffer) + first := true + for i := 0; i < sv.Len(); i++ { + if first { + first = false + } else { + s.WriteByte(del) + } + s.WriteString(valueString(sv.Index(i), opts)) + } + values.Add(name, s.String()) + } else { + for i := 0; i < sv.Len(); i++ { + k := name + if opts.Contains("numbered") { + k = fmt.Sprintf("%s%d", name, i) + } + values.Add(k, valueString(sv.Index(i), opts)) + } + } + continue + } + + for sv.Kind() == reflect.Ptr { + if sv.IsNil() { + break + } + sv = sv.Elem() + } + + if sv.Type() == timeType { + values.Add(name, valueString(sv, opts)) + continue + } + + if sv.Kind() == reflect.Struct { + reflectValue(values, sv, name) + continue + } + + values.Add(name, valueString(sv, opts)) + } + + for _, f := range embedded { + if err := reflectValue(values, f, scope); err != nil { + return err + } + } + + return nil +} + +// valueString returns the string representation of a value. +func valueString(v reflect.Value, opts tagOptions) string { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return "" + } + v = v.Elem() + } + + if v.Kind() == reflect.Bool && opts.Contains("int") { + if v.Bool() { + return "1" + } + return "0" + } + + if v.Type() == timeType { + t := v.Interface().(time.Time) + if opts.Contains("unix") { + return strconv.FormatInt(t.Unix(), 10) + } + return t.Format(time.RFC3339) + } + + return fmt.Sprint(v.Interface()) +} + +// isEmptyValue checks if a value should be considered empty for the purposes +// of omitting fields with the "omitempty" option. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + + if v.Type() == timeType { + return v.Interface().(time.Time).IsZero() + } + + return false +} + +// tagOptions is the string following a comma in a struct field's "url" tag, or +// the empty string. It does not include the leading comma. +type tagOptions []string + +// parseTag splits a struct field's url tag into its name and comma-separated +// options. +func parseTag(tag string) (string, tagOptions) { + s := strings.Split(tag, ",") + return s[0], s[1:] +} + +// Contains checks whether the tagOptions contains the specified option. +func (o tagOptions) Contains(option string) bool { + for _, s := range o { + if s == option { + return true + } + } + return false +} diff --git a/vendor/github.com/mozillazg/go-httpheader/.bumpversion.cfg b/vendor/github.com/mozillazg/go-httpheader/.bumpversion.cfg new file mode 100644 index 0000000000..18dad225a6 --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/.bumpversion.cfg @@ -0,0 +1,7 @@ +[bumpversion] +commit = True +tag = True +current_version = 0.2.1 + +[bumpversion:file:encode.go] + diff --git a/vendor/github.com/mozillazg/go-httpheader/.gitignore b/vendor/github.com/mozillazg/go-httpheader/.gitignore new file mode 100644 index 0000000000..098e254bc8 --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/.gitignore @@ -0,0 +1,27 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +dist/ +cover.html +cover.out diff --git a/vendor/github.com/mozillazg/go-httpheader/.travis.yml b/vendor/github.com/mozillazg/go-httpheader/.travis.yml new file mode 100644 index 0000000000..fbb97928b5 --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/.travis.yml @@ -0,0 +1,25 @@ +language: go +go: + - 1.6 + - 1.7 + - 1.8 + - tip + +sudo: false + +before_install: + - go get github.com/mattn/goveralls + +install: + - go get + - go build + +script: + - make test + - $HOME/gopath/bin/goveralls -service=travis-ci -ignore=vendor/ + +matrix: + allow_failures: + - go: 1.6 + - go: 1.7 + - go: tip diff --git a/vendor/github.com/mozillazg/go-httpheader/CHANGELOG.md b/vendor/github.com/mozillazg/go-httpheader/CHANGELOG.md new file mode 100644 index 0000000000..7b643fd64d --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## 0.2.1 (2018-11-03) + +* add go.mod file to identify as a module + + +## 0.2.0 (2017-06-24) + +* support http.Header field. + + +## 0.1.0 (2017-06-10) + +* Initial Release diff --git a/vendor/github.com/mozillazg/go-httpheader/LICENSE b/vendor/github.com/mozillazg/go-httpheader/LICENSE new file mode 100644 index 0000000000..8ff7942e20 --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 mozillazg + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/mozillazg/go-httpheader/Makefile b/vendor/github.com/mozillazg/go-httpheader/Makefile new file mode 100644 index 0000000000..aadfe8298e --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/Makefile @@ -0,0 +1,15 @@ +help: + @echo "test run test" + @echo "lint run lint" + +.PHONY: test +test: + go test -v -cover -coverprofile cover.out + go tool cover -html=cover.out -o cover.html + +.PHONY: lint +lint: + gofmt -s -w . + goimports -w . + golint . + go vet diff --git a/vendor/github.com/mozillazg/go-httpheader/README.md b/vendor/github.com/mozillazg/go-httpheader/README.md new file mode 100644 index 0000000000..d73daf64ce --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/README.md @@ -0,0 +1,63 @@ +# go-httpheader + +go-httpheader is a Go library for encoding structs into Header fields. + +[![Build Status](https://img.shields.io/travis/mozillazg/go-httpheader/master.svg)](https://travis-ci.org/mozillazg/go-httpheader) +[![Coverage Status](https://img.shields.io/coveralls/mozillazg/go-httpheader/master.svg)](https://coveralls.io/r/mozillazg/go-httpheader?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/mozillazg/go-httpheader)](https://goreportcard.com/report/github.com/mozillazg/go-httpheader) +[![GoDoc](https://godoc.org/github.com/mozillazg/go-httpheader?status.svg)](https://godoc.org/github.com/mozillazg/go-httpheader) + +## install + +`go get -u github.com/mozillazg/go-httpheader` + + +## usage + +```go +package main + +import ( + "fmt" + "net/http" + + "github.com/mozillazg/go-httpheader" +) + +type Options struct { + hide string + ContentType string `header:"Content-Type"` + Length int + XArray []string `header:"X-Array"` + TestHide string `header:"-"` + IgnoreEmpty string `header:"X-Empty,omitempty"` + IgnoreEmptyN string `header:"X-Empty-N,omitempty"` + CustomHeader http.Header +} + +func main() { + opt := Options{ + hide: "hide", + ContentType: "application/json", + Length: 2, + XArray: []string{"test1", "test2"}, + TestHide: "hide", + IgnoreEmptyN: "n", + CustomHeader: http.Header{ + "X-Test-1": []string{"233"}, + "X-Test-2": []string{"666"}, + }, + } + h, _ := httpheader.Header(opt) + fmt.Printf("%#v", h) + // h: + // http.Header{ + // "X-Test-1": []string{"233"}, + // "X-Test-2": []string{"666"}, + // "Content-Type": []string{"application/json"}, + // "Length": []string{"2"}, + // "X-Array": []string{"test1", "test2"}, + // "X-Empty-N": []string{"n"}, + //} +} +``` diff --git a/vendor/github.com/mozillazg/go-httpheader/encode.go b/vendor/github.com/mozillazg/go-httpheader/encode.go new file mode 100644 index 0000000000..c122f2b882 --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/encode.go @@ -0,0 +1,290 @@ +// Package query implements encoding of structs into http.Header fields. +// +// As a simple example: +// +// type Options struct { +// ContentType string `header:"Content-Type"` +// Length int +// } +// +// opt := Options{"application/json", 2} +// h, _ := httpheader.Header(opt) +// fmt.Printf("%#v", h) +// // will output: +// // http.Header{"Content-Type":[]string{"application/json"},"Length":[]string{"2"}} +// +// The exact mapping between Go values and http.Header is described in the +// documentation for the Header() function. +package httpheader + +import ( + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "time" +) + +const tagName = "header" + +// Version ... +const Version = "0.2.1" + +var timeType = reflect.TypeOf(time.Time{}) +var headerType = reflect.TypeOf(http.Header{}) + +var encoderType = reflect.TypeOf(new(Encoder)).Elem() + +// Encoder is an interface implemented by any type that wishes to encode +// itself into Header fields in a non-standard way. +type Encoder interface { + EncodeHeader(key string, v *http.Header) error +} + +// Header returns the http.Header encoding of v. +// +// Header expects to be passed a struct, and traverses it recursively using the +// following encoding rules. +// +// Each exported struct field is encoded as a Header field unless +// +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option +// +// The empty values are false, 0, any nil pointer or interface value, any array +// slice, map, or string of length zero, and any time.Time that returns true +// for IsZero(). +// +// The Header field name defaults to the struct field name but can be +// specified in the struct field's tag value. The "header" key in the struct +// field's tag value is the key name, followed by an optional comma and +// options. For example: +// +// // Field is ignored by this package. +// Field int `header:"-"` +// +// // Field appears as Header field "X-Name". +// Field int `header:"X-Name"` +// +// // Field appears as Header field "X-Name" and the field is omitted if +// // its value is empty +// Field int `header:"X-Name,omitempty"` +// +// // Field appears as Header field "Field" (the default), but the field +// // is skipped if empty. Note the leading comma. +// Field int `header:",omitempty"` +// +// For encoding individual field values, the following type-dependent rules +// apply: +// +// Boolean values default to encoding as the strings "true" or "false". +// Including the "int" option signals that the field should be encoded as the +// strings "1" or "0". +// +// time.Time values default to encoding as RFC1123("Mon, 02 Jan 2006 15:04:05 GMT") +// timestamps. Including the "unix" option signals that the field should be +// encoded as a Unix time (see time.Unix()) +// +// Slice and Array values default to encoding as multiple Header values of the +// same name. example: +// X-Name: []string{"Tom", "Jim"}, etc. +// +// http.Header values will be used to extend the Header fields. +// +// Anonymous struct fields are usually encoded as if their inner exported +// fields were fields in the outer struct, subject to the standard Go +// visibility rules. An anonymous struct field with a name given in its Header +// tag is treated as having that name, rather than being anonymous. +// +// Non-nil pointer values are encoded as the value pointed to. +// +// All other values are encoded using their default string representation. +// +// Multiple fields that encode to the same Header filed name will be included +// as multiple Header values of the same name. +func Header(v interface{}) (http.Header, error) { + h := make(http.Header) + val := reflect.ValueOf(v) + for val.Kind() == reflect.Ptr { + if val.IsNil() { + return h, nil + } + val = val.Elem() + } + + if v == nil { + return h, nil + } + + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("httpheader: Header() expects struct input. Got %v", val.Kind()) + } + + err := reflectValue(h, val) + return h, err +} + +// reflectValue populates the header fields from the struct fields in val. +// Embedded structs are followed recursively (using the rules defined in the +// Values function documentation) breadth-first. +func reflectValue(header http.Header, val reflect.Value) error { + var embedded []reflect.Value + + typ := val.Type() + for i := 0; i < typ.NumField(); i++ { + sf := typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + + sv := val.Field(i) + tag := sf.Tag.Get(tagName) + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if name == "" { + if sf.Anonymous && sv.Kind() == reflect.Struct { + // save embedded struct for later processing + embedded = append(embedded, sv) + continue + } + + name = sf.Name + } + + if opts.Contains("omitempty") && isEmptyValue(sv) { + continue + } + + if sv.Type().Implements(encoderType) { + if !reflect.Indirect(sv).IsValid() { + sv = reflect.New(sv.Type().Elem()) + } + + m := sv.Interface().(Encoder) + if err := m.EncodeHeader(name, &header); err != nil { + return err + } + continue + } + + if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array { + for i := 0; i < sv.Len(); i++ { + k := name + header.Add(k, valueString(sv.Index(i), opts)) + } + continue + } + + for sv.Kind() == reflect.Ptr { + if sv.IsNil() { + break + } + sv = sv.Elem() + } + + if sv.Type() == timeType { + header.Add(name, valueString(sv, opts)) + continue + } + if sv.Type() == headerType { + h := sv.Interface().(http.Header) + for k, vs := range h { + for _, v := range vs { + header.Add(k, v) + } + } + continue + } + + if sv.Kind() == reflect.Struct { + reflectValue(header, sv) + continue + } + + header.Add(name, valueString(sv, opts)) + } + + for _, f := range embedded { + if err := reflectValue(header, f); err != nil { + return err + } + } + + return nil +} + +// valueString returns the string representation of a value. +func valueString(v reflect.Value, opts tagOptions) string { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return "" + } + v = v.Elem() + } + + if v.Kind() == reflect.Bool && opts.Contains("int") { + if v.Bool() { + return "1" + } + return "0" + } + + if v.Type() == timeType { + t := v.Interface().(time.Time) + if opts.Contains("unix") { + return strconv.FormatInt(t.Unix(), 10) + } + return t.Format(http.TimeFormat) + } + + return fmt.Sprint(v.Interface()) +} + +// isEmptyValue checks if a value should be considered empty for the purposes +// of omitting fields with the "omitempty" option. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + + if v.Type() == timeType { + return v.Interface().(time.Time).IsZero() + } + + return false +} + +// tagOptions is the string following a comma in a struct field's "header" tag, or +// the empty string. It does not include the leading comma. +type tagOptions []string + +// parseTag splits a struct field's header tag into its name and comma-separated +// options. +func parseTag(tag string) (string, tagOptions) { + s := strings.Split(tag, ",") + return s[0], s[1:] +} + +// Contains checks whether the tagOptions contains the specified option. +func (o tagOptions) Contains(option string) bool { + for _, s := range o { + if s == option { + return true + } + } + return false +} diff --git a/vendor/github.com/mozillazg/go-httpheader/go.mod b/vendor/github.com/mozillazg/go-httpheader/go.mod new file mode 100644 index 0000000000..27af234ebf --- /dev/null +++ b/vendor/github.com/mozillazg/go-httpheader/go.mod @@ -0,0 +1 @@ +module github.com/mozillazg/go-httpheader diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/auth.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/auth.go deleted file mode 100644 index c560caa2c6..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/auth.go +++ /dev/null @@ -1,128 +0,0 @@ -package cos - -import ( - "crypto/hmac" - "crypto/sha1" - "encoding/hex" - "fmt" - "net/http" - "sort" - "strconv" - "strings" - "time" -) - -func (conn *Conn) signHeader(req *http.Request, params map[string]interface{}, headers map[string]string) { - signTime := getSignTime() - signature := conn.getSignature(req, params, headers, signTime) - authStr := fmt.Sprintf("q-sign-algorithm=sha1&q-ak=%s&q-sign-time=%s&q-key-time=%s&q-header-list=%s&q-url-param-list=%s&q-signature=%s", - conn.conf.SecretID, signTime, signTime, getHeadKeys(headers), getParamKeys(params), signature) - - req.Header.Set("Authorization", authStr) -} - -func getSignTime() string { - now := time.Now() - expired := now.Add(time.Second * 1800) - return fmt.Sprintf("%d;%d", now.Unix(), expired.Unix()) -} - -func getHeadKeys(headers map[string]string) string { - if headers == nil || len(headers) == 0 { - return "" - } - - tmp := []string{} - for k := range headers { - tmp = append(tmp, strings.ToLower(k)) - } - sort.Strings(tmp) - - return strings.Join(tmp, ";") -} - -func getParamKeys(params map[string]interface{}) string { - if params == nil || len(params) == 0 { - return "" - } - - tmp := []string{} - for k := range params { - tmp = append(tmp, strings.ToLower(k)) - } - sort.Strings(tmp) - - return strings.Join(tmp, ";") -} - -func (conn *Conn) getSignature(req *http.Request, params map[string]interface{}, headers map[string]string, signTime string) string { - httpString := fmt.Sprintf("%s\n%s\n%s\n%s\n", strings.ToLower(req.Method), - req.URL.Path, getParamStr(params), getHeadStr(headers)) - - httpString = sha(httpString) - signKey := hmacSha(conn.conf.SecretKey, signTime) - signStr := fmt.Sprintf("sha1\n%s\n%s\n", signTime, httpString) - - return hmacSha(signKey, signStr) -} - -func interfaceToString(i interface{}) string { - switch x := i.(type) { - case string: - return x - case int: - return strconv.Itoa(x) - case int64: - return strconv.FormatInt(x, 10) - case uint64: - return strconv.FormatUint(x, 10) - default: - return "" - } -} - -func getParamStr(params map[string]interface{}) string { - if params == nil || len(params) == 0 { - return "" - } - - tmp := []string{} - for k, v := range params { - str := strings.ToLower(fmt.Sprintf("%s=%s", k, interfaceToString(v))) - tmp = append(tmp, str) - } - sort.Strings(tmp) - - return strings.Join(tmp, "&") -} - -func getHeadStr(headers map[string]string) string { - if headers == nil || len(headers) == 0 { - return "" - } - - tmp := []string{} - for k, v := range headers { - str := fmt.Sprintf("%s=%s", strings.ToLower(k), escape(v)) - tmp = append(tmp, str) - } - sort.Strings(tmp) - - return strings.Join(tmp, "&") -} - -func sha(s string) string { - sha := sha1.New() - sha.Write([]byte(s)) - b := sha.Sum(nil) - - return hex.EncodeToString(b) -} - -func hmacSha(k, s string) string { - enc := hmac.New(sha1.New, []byte(k)) - enc.Write([]byte(s)) - b := enc.Sum(nil) - - return hex.EncodeToString(b) -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/bucket.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/bucket.go deleted file mode 100644 index 3458f2fb30..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/bucket.go +++ /dev/null @@ -1,315 +0,0 @@ -package cos - -import ( - "bytes" - "context" - "crypto/md5" - "encoding/hex" - "encoding/xml" - "fmt" - "io" - "net/http" - "os" - "strings" -) - -// Bucket bucket -type Bucket struct { - Name string - conn *Conn -} - -// ObjectSlice object slice -type ObjectSlice struct { - UploadID string - Size int64 - Offset int64 - Number int - MD5 string - Dst string - Result bool -} - -// 获得云存储上文件信息 -func (b *Bucket) HeadObject(ctx context.Context, object string) error { - resq, err := b.conn.Do(ctx, http.MethodHead, b.Name, object, nil, nil, nil) - if err == nil { - defer resq.Body.Close() - } else { - for k, v := range resq.Header { - value := fmt.Sprintf("%s", v) - fmt.Printf("%-18s: %s\n", k, strings.Replace(strings.Replace(value, "[", "", -1), "]", "", -1)) - } - } - - return err -} - -func (b *Bucket) UploadObject(ctx context.Context, object string, content io.Reader, acl *AccessControl) error { - res, err := b.conn.Do(ctx, http.MethodPut, b.Name, object, nil, acl.GenHead(), content) - if err == nil { - defer res.Body.Close() - } - - return err -} - -func (b *Bucket) CopyObject(ctx context.Context, src, dst string, acl *AccessControl) error { - srcURL := fmt.Sprintf("%s-%s.cos.%s.%s/%s", b.Name, b.conn.conf.AppID, b.conn.conf.Region, b.conn.conf.Domain, dst) - header := map[string]string{ - "x-cos-source-url": srcURL, - } - - res, err := b.conn.Do(ctx, http.MethodPut, b.Name, dst, nil, header, nil) - if err == nil { - defer res.Body.Close() - } - - return err -} - -func (b *Bucket) DeleteObject(ctx context.Context, obj string) error { - res, err := b.conn.Do(ctx, http.MethodDelete, b.Name, obj, nil, nil, nil) - if err == nil { - defer res.Body.Close() - } - - return err -} - -func (b *Bucket) DownloadObject(ctx context.Context, object string, w io.Writer) error { - res, err := b.conn.Do(ctx, http.MethodGet, b.Name, object, nil, nil, nil) - if err != nil { - return err - } - - _, err = io.Copy(w, res.Body) - - return err -} - -// UploadObjectBySlice upload by slice -func (b *Bucket) UploadObjectBySlice(ctx context.Context, dst, src string, taskNum int, headers map[string]string) error { - if taskNum < 1 { - return ParamError{"taskNum 必须大于1"} - } - - uploadID, err := b.InitSliceUpload(ctx, dst, headers) - if err != nil { - return err - } - - fd, err := os.Open(src) - if err != nil { - return err - } - defer fd.Close() - - slices, err := b.PerformSliceUpload(ctx, dst, uploadID, fd, taskNum) - if err != nil { - return err - } - - err = b.CompleteSliceUpload(ctx, dst, uploadID, fd, slices) - - return err -} - -// InitSliceUpload init upload by slice -func (b *Bucket) InitSliceUpload(ctx context.Context, obj string, headers map[string]string) (string, error) { - param := map[string]interface{}{ - "uploads": "", - } - res, err := b.conn.Do(ctx, http.MethodPost, b.Name, obj, param, headers, nil) - if err != nil { - return "", err - } - defer res.Body.Close() - - imur := &InitiateMultipartUploadResult{} - err = XMLDecode(res.Body, imur) - if err != nil { - return "", err - } - - return imur.UploadID, nil -} - -// CompleteSliceUpload finish slice Upload -func (b *Bucket) CompleteSliceUpload(ctx context.Context, dst, uploadID string, fd *os.File, slice []*ObjectSlice) error { - cmu := &CompleteMultipartUpload{} - cmu.Part = []struct { - PartNumber int - ETag string - }{} - - for _, osl := range slice { - cmu.Part = append(cmu.Part, struct { - PartNumber int - ETag string - }{PartNumber: osl.Number, ETag: osl.MD5}) - } - - cmuXML, err := xml.Marshal(cmu) - if err != nil { - return err - } - param := map[string]interface{}{ - "uploadId": uploadID, - } - res, err := b.conn.Do(ctx, http.MethodPost, b.Name, dst, param, nil, bytes.NewReader(cmuXML)) - if err == nil { - defer res.Body.Close() - } - - return err -} - -// PerformSliceUpload perform slice upload -func (b *Bucket) PerformSliceUpload(ctx context.Context, dst, uploadID string, fd *os.File, taskNum int) ([]*ObjectSlice, error) { - oss, err := b.getFileSlices(fd, uploadID, dst) - if err != nil { - return nil, err - } - - jobNum := len(oss) - jobs := make(chan *ObjectSlice, jobNum) - result := make(chan *ObjectSlice, jobNum) - - for i := 0; i < taskNum; i++ { - go b.Worker(ctx, fd, jobs, result) - } - - for _, osl := range oss { - jobs <- osl - } - close(jobs) - - for i := 0; i < jobNum; i++ { - res := <-result - if !res.Result { - return nil, SliceError{fmt.Sprintf("part info : num:%d, md5:%s", res.Number, res.MD5)} - } - } - - return oss, nil -} - -// Worker woker for slice upload -func (b *Bucket) Worker(ctx context.Context, fd *os.File, jobs <-chan *ObjectSlice, result chan<- *ObjectSlice) { - for job := range jobs { - content, err := getFilePartContent(fd, job.Offset, job.Size) - if err != nil { - continue - } - - err = b.UploadSlice(ctx, job.UploadID, job.Dst, job.Number, job.MD5, content) - if err == nil { - job.Result = true - } else { - job.Result = false - } - - result <- job - } -} - -// UploadSlice upload one slice -func (b *Bucket) UploadSlice(ctx context.Context, uploadID, dst string, number int, etag string, content io.Reader) error { - param := map[string]interface{}{ - "PartNumber": number, - "uploadId": uploadID, - } - res, err := b.conn.Do(ctx, http.MethodPut, b.Name, dst, param, nil, content) - - if err != nil { - return FileError{"PUT数据错误:" + err.Error()} - } - defer res.Body.Close() - - if strings.Trim(res.Header.Get("Etag"), "\"") != etag { - return FileError{"cos-etag与文件MD5不匹配"} - } - - return err -} - -func (b *Bucket) getFileSlices(fd *os.File, uploadID, dst string) ([]*ObjectSlice, error) { - sliceSize := b.conn.conf.PartSize - fi, err := fd.Stat() - if err != nil { - return nil, err - } - - fileSize := fi.Size() - oss := []*ObjectSlice{} - var i int - var offset int64 - for fileSize > 0 { - var size int64 - if fileSize > sliceSize { - size = sliceSize - } else { - size = fileSize - } - i++ - md5, err := getFileMD5(fd, offset, size) - if err != nil { - return nil, err - } - - osl := &ObjectSlice{} - osl.Size = size - osl.Number = i - osl.Offset = offset - osl.UploadID = uploadID - osl.MD5 = md5 - osl.Dst = dst - oss = append(oss, osl) - - fileSize -= sliceSize - offset += sliceSize - } - - return oss, nil -} - -func getFileMD5(fd *os.File, offset, size int64) (string, error) { - buf := make([]byte, size) - _, err := fd.ReadAt(buf, offset) - if err != nil { - return "", err - } - - encoder := md5.New() - encoder.Write(buf) - b := encoder.Sum(nil) - - return hex.EncodeToString(b), nil -} - -func getFilePartContent(fd *os.File, offset, size int64) (io.Reader, error) { - buf := make([]byte, size) - _, err := fd.ReadAt(buf, offset) - if err != nil { - return nil, err - } - - return bytes.NewReader(buf), nil -} - -func (b *Bucket) AbortUpload(ctx context.Context, obj, uploadID string) error { - param := map[string]interface{}{ - "uploadId": uploadID, - } - _, err := b.conn.Do(ctx, http.MethodDelete, b.Name, obj, param, nil, nil) - - return err -} - -// ObjectExists object exists -func (b *Bucket) ObjectExists(ctx context.Context, obj string) error { - _, err := b.conn.Do(ctx, http.MethodHead, b.Name, obj, nil, nil, nil) - - return err -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/client.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/client.go deleted file mode 100644 index 18beb021b5..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/client.go +++ /dev/null @@ -1,164 +0,0 @@ -package cos - -import ( - "context" - "net/http" - "time" -) - -// Client 客户端， cos的句柄 -type Client struct { - conn *Conn -} - -// New cos包的入口 -func New(o *Option) *Client { - client := Client{} - conf := getDefaultConf() - conf.AppID = o.AppID - conf.SecretID = o.SecretID - conf.SecretKey = o.SecretKey - conf.Region = o.Region - - if o.Domain != "" { - conf.Domain = o.Domain - } - - conn := Conn{&http.Client{}, conf} - client.conn = &conn - - return &client -} - -// GetTimeoutCtx 获取一个带超时的context -func GetTimeoutCtx(timeout time.Duration) context.Context { - ctx, _ := context.WithTimeout(context.Background(), timeout) - - return ctx -} - -// Bucket get bucket -func (c *Client) Bucket(name string) *Bucket { - return &Bucket{name, c.conn} -} - -// GetBucketList 获取bucketlist -func (c *Client) GetBucketList(ctx context.Context) (*ListAllMyBucketsResult, error) { - req, err := http.NewRequest(http.MethodGet, "http://service.cos.myqcloud.com/", nil) - if err != nil { - return nil, err - } - - req = req.WithContext(ctx) - c.conn.signHeader(req, nil, nil) - res, err := c.conn.c.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - res, err = checkHTTPErr(res) - if err != nil { - return nil, err - } - - labr := &ListAllMyBucketsResult{} - err = XMLDecode(res.Body, labr) - if err != nil { - return nil, err - } - - return labr, err -} - -// CreateBucket 建立bucket -func (c *Client) CreateBucket(ctx context.Context, name string, acl *AccessControl) error { - res, err := c.conn.Do(ctx, http.MethodPut, name, "", nil, acl.GenHead(), nil) - if err == nil { - defer res.Body.Close() - } - - return err -} - -// DeleteBucket delete a bucket -func (c *Client) DeleteBucket(ctx context.Context, name string) error { - _, err := c.conn.Do(ctx, http.MethodDelete, name, "", nil, nil, nil) - - return err -} - -// GetBucketACL get bucket's acl -func (c *Client) GetBucketACL(ctx context.Context, name string) (*AccessControlPolicy, error) { - params := map[string]interface{}{"acl": ""} - res, err := c.conn.Do(ctx, http.MethodGet, name, "", params, nil, nil) - - if err != nil { - return nil, err - } - defer res.Body.Close() - - aclp := &AccessControlPolicy{} - - err = XMLDecode(res.Body, aclp) - if err != nil { - return nil, err - } - - return aclp, nil -} - -// SetBucketACL set bucket's acl -func (c *Client) SetBucketACL(ctx context.Context, name string, acl *AccessControl) error { - params := map[string]interface{}{"acl": ""} - res, err := c.conn.Do(ctx, http.MethodPut, name, "", params, acl.GenHead(), nil) - if err == nil { - defer res.Body.Close() - } - - return err -} - -// BucketExists bucket exists? -func (c *Client) BucketExists(ctx context.Context, name string) error { - res, err := c.conn.Do(ctx, http.MethodHead, name, "", nil, nil, nil) - if err == nil { - defer res.Body.Close() - } - - return err -} - -// ListBucketContents list -func (c *Client) ListBucketContents(ctx context.Context, name string, qc *QueryCondition) (*ListBucketResult, error) { - resp, err := c.conn.Do(ctx, http.MethodGet, name, "", qc.GenParams(), nil, nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - lbr := &ListBucketResult{} - err = XMLDecode(resp.Body, lbr) - if err != nil { - return nil, err - } - - return lbr, nil -} - -// ListUploading list uploading task -func (c *Client) ListUploading(ctx context.Context, bucket string, lu *ListUploadParam) (*ListMultipartUploadsResult, error) { - res, err := c.conn.Do(ctx, http.MethodGet, bucket, "", lu.GenParams(), nil, nil) - if err != nil { - return nil, err - } - defer res.Body.Close() - - lmur := &ListMultipartUploadsResult{} - err = XMLDecode(res.Body, lmur) - if err != nil { - return nil, err - } - - return lmur, nil -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conf.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conf.go deleted file mode 100644 index 516c013164..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conf.go +++ /dev/null @@ -1,31 +0,0 @@ -package cos - -const ( - defaultPartSize = 80 * 1024 * 1024 - defaultRetryTimes = 3 - defaultUA = "cos-go-sdk-v5.2.9" - defaultDomain = "myqcloud.com" -) - -// Conf config struct -type Conf struct { - AppID string - SecretID string - SecretKey string - Region string - PartSize int64 - RetryTimes int - UA string - Domain string - Bucket string -} - -func getDefaultConf() *Conf { - conf := Conf{} - conf.PartSize = defaultPartSize - conf.RetryTimes = defaultRetryTimes - conf.UA = defaultUA - conf.Domain = defaultDomain - - return &conf -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conn.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conn.go deleted file mode 100644 index bf6380ac0e..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/conn.go +++ /dev/null @@ -1,141 +0,0 @@ -package cos - -import ( - "bytes" - "context" - "encoding/xml" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/url" - "strings" -) - -// Conn http 请求类 -type Conn struct { - c *http.Client - conf *Conf -} - -func (conn *Conn) Do(ctx context.Context, method, bucket, object string, params map[string]interface{}, headers map[string]string, body io.Reader) (*http.Response, error) { - queryStr := getQueryStr(params) - url := conn.buildURL(bucket, object, queryStr) - - switch body.(type) { - case *bytes.Buffer, *bytes.Reader, *strings.Reader: - default: - if body != nil { - b, err := ioutil.ReadAll(body) - if err != nil { - return nil, err - } - body = bytes.NewReader(b) - } - } - - req, err := http.NewRequest(method, url, body) - if err != nil { - return nil, err - } - - conn.signHeader(req, params, headers) - req.Header.Set("User-Agent", conn.conf.UA) - setHeader(req, headers) - - res, err := conn.c.Do(req) - - if err != nil { - return nil, err - } - - if res.StatusCode < 200 || res.StatusCode >= 300 { - defer res.Body.Close() - return checkHTTPErr(res) - } - - return res, nil -} - -func getQueryStr(params map[string]interface{}) string { - if params == nil || len(params) == 0 { - return "" - } - - buf := new(bytes.Buffer) - buf.WriteString("?") - for k, v := range params { - buf.WriteString(k) - vs := interfaceToString(v) - if vs == "" { - buf.WriteString("&") - continue - } - buf.WriteString("=") - buf.WriteString(vs) - buf.WriteString("&") - } - - return strings.Trim(buf.String(), "&") -} - -func (conn *Conn) buildURL(bucket, object, queryStr string) string { - domain := fmt.Sprintf("%s-%s.cos.%s.%s", bucket, conn.conf.AppID, conn.conf.Region, conn.conf.Domain) - url := fmt.Sprintf("http://%s/%s%s", domain, escape(object), queryStr) - - return url -} - -func escape(str string) string { - //go语言中将空格编码为+，需要改为%20 - return strings.Replace(url.QueryEscape(str), "+", "%20", -1) -} - -func setHeader(req *http.Request, headers map[string]string) { - if headers == nil { - return - } - - for k, v := range headers { - req.Header.Set(k, v) - } -} - -func checkHTTPErr(res *http.Response) (*http.Response, error) { - if res.StatusCode >= 200 && res.StatusCode < 300 { - return res, nil - } - - err := HTTPError{} - err.Code = res.StatusCode - if res.StatusCode >= 300 && res.StatusCode < 400 { - err.Message = "资源被重定向" - } - - if res.StatusCode >= 400 && res.StatusCode < 500 { - err.Message = "请求被拒绝" - } - - if res.StatusCode >= 500 { - err.Message = "cos服务器错误" - } - - if res.ContentLength > 0 { - resErr := &Error{} - e := XMLDecode(res.Body, resErr) - if e != nil { - return nil, err - } - err.Message += resErr.Message - } - - return res, err -} - -// XMLDecode xml解析方法 -func XMLDecode(r io.Reader, i interface{}) error { - jd := xml.NewDecoder(r) - err := jd.Decode(i) - - return err -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/option.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/option.go deleted file mode 100644 index d6bc8af99b..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/option.go +++ /dev/null @@ -1,10 +0,0 @@ -package cos - -type Option struct { - AppID string `mapstructure:"app_id" json:"app_id"` - SecretID string `mapstructure:"secret_id" json:"secret_id"` - SecretKey string `mapstructure:"secret_key" json:"secret_key"` - Region string `mapstructure:"region" json:"region"` - Domain string `mapstructure:"domain" json:"domain"` - Bucket string `mapstructure:"bucket" json:"bucket"` -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/requests.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/requests.go deleted file mode 100644 index e26c6757e9..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/requests.go +++ /dev/null @@ -1,102 +0,0 @@ -package cos - -// AccessControl privilige -type AccessControl struct { - ACL string - GrantRead string - GrantWrite string - FullControl string -} - -// GenHead 生成http head -func (acl *AccessControl) GenHead() map[string]string { - header := map[string]string{ - "x-cos-acl": acl.ACL, - "x-cos-grant-read": acl.GrantRead, - "x-cos-grant-write": acl.GrantWrite, - "x-cos-grant-full-control": acl.FullControl, - } - - for k, v := range header { - if v == "" { - delete(header, k) - } - } - - return header -} - -// QueryCondition query condition -type QueryCondition struct { - Prefix string - Delimiter string - EncodingType string - Marker string - MaxKeys int -} - -// GenParams generate params:map[string]interface{} -func (qc *QueryCondition) GenParams() map[string]interface{} { - params := map[string]interface{}{ - "prefix": qc.Prefix, - "delimiter": qc.Delimiter, - "encoding-type": qc.EncodingType, - "marker": qc.Marker, - "max-keys": qc.MaxKeys, - } - - for k, v := range params { - if v == "" { - delete(params, k) - } - if v == 0 { - delete(params, k) - } - } - - return params -} - -// ListUploadParam list upload param -type ListUploadParam struct { - Prefix string - Delimiter string - EncodingType string - MaxUploads int - KeyMarker string - UploadIDMarker string -} - -// GenParams generate params for request -func (lup *ListUploadParam) GenParams() map[string]interface{} { - params := map[string]interface{}{ - "prefix": lup.Prefix, - "delimiter": lup.Delimiter, - "encoding-type": lup.EncodingType, - "max-uploads": lup.MaxUploads, - "key-marker": lup.KeyMarker, - "upload-id-marker": lup.UploadIDMarker, - } - - for k, v := range params { - if v == "" { - delete(params, k) - } - - if v == 0 { - delete(params, k) - } - } - params["uploads"] = "" - - return params - -} - -// CompleteMultipartUpload compelete slice upload -type CompleteMultipartUpload struct { - Part []struct { - PartNumber int - ETag string - } -} diff --git a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/response.go b/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/response.go deleted file mode 100644 index 8c69ab30cd..0000000000 --- a/vendor/github.com/nelsonken/cos-go-sdk-v5/cos/response.go +++ /dev/null @@ -1,156 +0,0 @@ -package cos - -import ( - "fmt" -) - -// ListAllMyBucketsResult 获取bucket列表的结果 -type ListAllMyBucketsResult struct { - Owner struct { - ID string - DisplayName string - } - - Buckets struct { - Bucket []struct { - Name string - Location string - CreateDate string - } - } -} - -// Error 错误消息 -type Error struct { - Code string - Message string - Resource string - RequestID string `xml:"RequestId"` - TraceID string `xml:"TaceId"` -} - -// HTTPError http error struct -type HTTPError struct { - Code int - Message string -} - -// Error error interface -func (he HTTPError) Error() string { - return fmt.Sprintf("%d:%s", he.Code, he.Message) -} - -// AccessControlPolicy acl return -type AccessControlPolicy struct { - Owner struct { - ID string - DisplayName string - } - AccessControlList struct { - Grant []struct { - Grantee struct { - ID string - DisplayName string - } - Permission string - } - } -} - -// ListBucketResult list bucket contents result -type ListBucketResult struct { - Name string - EncodingType string `xml:"Encoding-Type"` - Prefix string - Marker string - MaxKeys int - IsTruncated bool - NextMarker string - Contents []struct { - Key string - LastModified string - ETag string - Size int64 - Owner struct { - ID string - } - StorageClass string - } - CommonPrefixes []struct { - Prefix string - } -} - -// ListMultipartUploadsResult list uploading task -type ListMultipartUploadsResult struct { - Bucket string - EncodingType string `xml:"Encoding-Type"` - KeyMarker string - UploadIDMarker string `xml:"UploadIdMarker"` - NextKeyMarker string - NextUploadIDMarker string `xml:"NextUploadIdMarker"` - MaxUploads int - IsTruncated bool - Prefix string - Delimiter string - Upload []struct { - Key string - UploadID string - StorageClass string - Initiator struct { - UIN string - } - Owner struct { - UID string - } - Initiated string - } - CommonPrefixes []struct { - Prefix string - } -} - -// InitiateMultipartUploadResult init slice upload -type InitiateMultipartUploadResult struct { - Bucket string - Key string - UploadID string `xml:"UploadId"` -} - -// CompleteMultipartUploadResult compeleted slice upload -type CompleteMultipartUploadResult struct { - Location string - Bucket string - Key string - ETag string -} - -// SliceError slice upload err -type SliceError struct { - Message string -} - -// Error implements error -func (se SliceError) Error() string { - return fmt.Sprintf("上传分片失败:%s", se.Message) -} - -// ParamError slice upload err -type ParamError struct { - Message string -} - -// Error implements error -func (pe ParamError) Error() string { - return fmt.Sprintf("参数错误:%s", pe.Message) -} - -// FileError slice upload err -type FileError struct { - Message string -} - -// Error implements error -func (fe FileError) Error() string { - return fmt.Sprintf("文件错误:%s", fe.Message) -} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/.bumpversion.cfg b/vendor/github.com/tencentyun/cos-go-sdk-v5/.bumpversion.cfg new file mode 100644 index 0000000000..d2bac6df90 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/.bumpversion.cfg @@ -0,0 +1,7 @@ +[bumpversion] +commit = True +tag = True +current_version = 0.7.0 + +[bumpversion:file:cos.go] + diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/.gitignore b/vendor/github.com/tencentyun/cos-go-sdk-v5/.gitignore new file mode 100644 index 0000000000..7da68739af --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/.gitignore @@ -0,0 +1,29 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +dist/ +cover.html +cover.out +covprofile +coverage.html \ No newline at end of file diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/.travis.yml b/vendor/github.com/tencentyun/cos-go-sdk-v5/.travis.yml new file mode 100644 index 0000000000..5202cd93ef --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/.travis.yml @@ -0,0 +1,29 @@ +language: go +go: +- '1.7' +- '1.8' +- '1.9' +- 1.10.x +- master +sudo: false +before_install: +- go get github.com/mattn/goveralls +- go get github.com/stretchr/testify +install: +- go get +- go build +- go build github.com/mattn/goveralls +script: +- make test +- make ci-test +- go test -coverprofile=cover.out github.com/toranger/cos-go-sdk-v5 +- "${TRAVIS_HOME}/gopath/bin/goveralls -service=travis-ci -coverprofile=cover.out" +matrix: + allow_failures: + - go: 1.7 + - go: master +env: + global: + - secure: XXB/cFVnJcAzhOZ2/zplwjhhhireQQGGRbNscPgQ0kpUQCyPZ6oIHvJMafuP4TVTJHEdMiaDxm0HNvgARuopXVaQNmK2UZj6xw40Ud7OT7ZUnw88xkQkXOI5GwG8oz9LqxIXUSItHegKXRLW0e1PoBdjZNv6lxGFAtuOcl9ekAg/q2lGIIQFefz6NK7gCmGYULKe+4J15VFldoYNM0JesxxxArTvtv8+k+U53oUwy9dex6z5oIA1zGIeKLcOD2xXgbjid/Ett3t0B2w3GfJWoM9rGV0eHgveOAUGe5tQkMKvl5LK1hj+93ZmU0MAG7x7t9jYKrFPqU/eDNJRMb4Ro6L7lIXVEKaBUkLx28PnwFQ5D043GBVtQGqYNcldZXIfbyYEHQZlD/BWFOt5YqTpGg+7Wm4NC3Yffqsurzk54juT7FftzVy0A8MFkqO+c5RHrOSUlm01pWXkGLHgZhUP5gEZEuUaoluSQTZksmAUJZ7F8DxwpE4SYBqfN27PZ87rWDNyOqNv1w1trzwx2IfdHHA+vfCZ7UM5e85gxFWUO2tJCUai2q21v3gBrcAgBOb6BwVzbWAorM2zY20f0l21XxOWMakA+r4JJA3s3EmcczcQeeL6pkFIAh+qKdFEPuyQTjH1mGpPzYFNbWtvPXijQo5PqyGrKL8W1t3ovwXMXoE= + - secure: bep0PPD/oYW5zY0QpeeC+WgFIya5DNRVmR92MO+e5BdFlSJPhstoG8bRh91EeftzC/Hyd3PUEIglPqTgZPxwysqW/81plsU95wV3qJi9gPi7+ZtYXH4xZTnaqgZsTr7jsKSVoKHSu7XqCtbSytW8YMN9wRWzG19/9hX2Z79Q6yNy5l9856Oyj1E2IXDjdZLPsWDhnZ8Vvk1wAVy2fc2esqKzHAZwm8n9vee2yR8vz7GXUszzpKvn4R43eNzdlFEHCmN0ANmxLJZmnYDpZHHfNf4slts+0S6I7awFXppuXUDaJPBRCia4XoFeSw+01IW1Vi0kAwvGLhxjJCWc4M/4ZU0byXDT11tDFvWa19NmnbYiizWiXNVecn1oNWYJqIKe7TTAMAtHSXAPmLX0rXuXKzwM09W6yrLFufCxyix9IOnenEbe9WwSdBbhmeLF3Wu/uVGkDog/FsXJM75sk956vV9UKh9zF4B9/NR8szJMF7shEs0Fbru5UUWheqg4AadPl3dhAWuj2+6NANa1LpH3JVD3II9dlXeMmMvsSwDvrYUaX/S8tf6JwZG0zCJK0TYp05rjxH+NIzWaMUTY7+HwYqqK3pOW3San0SlZiMq8N7GSnKUZ7WRQXYSB4gXHrg+mWyeVC7XnqiRtCwVi+LtPMu+YUbg7dwVi0vtKjYZYIUY= + - secure: Ob28vrOuHMKNKEtChkWbsaVv2SwLhcxXMnvGe4XN+y3mFvdhYnwpt6NdgThF8OCZ0761tvTRmvALfiZnO0uORjTtoHKkVPrnVIxlCcode0NVJZNHGn2fqjemdLKCnSeX7hm+9zeLpCnIvC+Sp3iZ3t2AH4AzgFx6nirWO3HwT5l9rNL9Q1CfwlOpNJJ36r9JTHwQnXmOfOmszUNoZ3rtiFXJ8dCi+BgY0lsiIRSiDkAH7KAPf86REM+ww81AaXG4/RuYx1Vj5zQCtZN7XEOViSXEbqqb8SrIFOccDu5FV12djg+4QS7FSjLVGrdIUcn4oI6pS24Et3oXf8xFx6JLYyGGhgZ2BsyJEx5vLQvkTWnMTrwZVRtCQ+g6lMUQpJhL2rBrmVBUqBFb5IH69O7corQm53n5qLM8IiosAQLfbOtML/1PyEpKCG2aOx1377Fx2yzxXW3ucP1PBqCzli0oCM2T52LfiNvZTzkIU6XJebBnzkZXepzOIFSur86kxgvQFElw9ro2X6XXPKU5S25xVaUSvaN1kmqLSkToJ9S1rmDYXnJR4aH0R2GcLw+EkMHFJJoAjnRHxrB4/1vOJbzmfS+qy6ShRhUMSD8gk4YJ6Y7o9h7oekuWOEn+XGhl29U9T5OApzHfoPEGZwLnpHxAiKJtQtv/TNhBIOFCjigsF7U= diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/LICENSE b/vendor/github.com/tencentyun/cos-go-sdk-v5/LICENSE new file mode 100644 index 0000000000..8ff7942e20 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 mozillazg + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/Makefile b/vendor/github.com/tencentyun/cos-go-sdk-v5/Makefile new file mode 100644 index 0000000000..e3b7aa3193 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/Makefile @@ -0,0 +1,22 @@ +help: + @echo "test run test" + @echo "lint run lint" + @echo "example run examples" + +.PHONY: test +test: + go test -v -cover -coverprofile cover.out + go tool cover -html=cover.out -o cover.html + +.PHONY: lint +lint: + gofmt -s -w . + goimports -w . + golint . + go vet + +.PHONY: example +example: + cd example && sh test.sh +ci-test: + cd costesting && go test -v diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md b/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md new file mode 100644 index 0000000000..0316a137ba --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md @@ -0,0 +1,95 @@ +# cos-go-sdk-v5 + +腾讯云对象存储服务 COS(Cloud Object Storage) Go SDK（API 版本：V5 版本的 XML API）。 + +## Install + +`go get -u github.com/tencentyun/cos-go-sdk-v5` + + +## Usage + +```go +package main + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "time" + + "github.com/tencentyun/cos-go-sdk-v5" +) + +func main() { + //将和修改为真实的信息 + //bucket的命名规则为{name}-{appid} ，此处填写的存储桶名称必须为此格式 + u, _ := url.Parse("https://.cos..myqcloud.com") + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + //设置超时时间 + Timeout: 100 * time.Second, + Transport: &cos.AuthorizationTransport{ + //如实填写账号和密钥，也可以设置为环境变量 + SecretID: os.Getenv("COS_SECRETID"), + SecretKey: os.Getenv("COS_SECRETKEY"), + }, + }) + + name := "test/hello.txt" + resp, err := c.Object.Get(context.Background(), name, nil) + if err != nil { + panic(err) + } + bs, _ := ioutil.ReadAll(resp.Body) + resp.Body.Close() + fmt.Printf("%s\n", string(bs)) +} +``` + +所有的 API 在 [example](./example/) 目录下都有对应的使用示例。 + +Service API: + +* [x] Get Service（使用示例：[service/get.go](./example/service/get.go)） + +Bucket API: + +* [x] Get Bucket（使用示例：[bucket/get.go](./example/bucket/get.go)） +* [x] Get Bucket ACL（使用示例：[bucket/getACL.go](./example/bucket/getACL.go)） +* [x] Get Bucket CORS（使用示例：[bucket/getCORS.go](./example/bucket/getCORS.go)） +* [x] Get Bucket Location（使用示例：[bucket/getLocation.go](./example/bucket/getLocation.go)） +* [x] Get Buket Lifecycle（使用示例：[bucket/getLifecycle.go](./example/bucket/getLifecycle.go)） +* [x] Get Bucket Tagging（使用示例：[bucket/getTagging.go](./example/bucket/getTagging.go)） +* [x] Put Bucket（使用示例：[bucket/put.go](./example/bucket/put.go)） +* [x] Put Bucket ACL（使用示例：[bucket/putACL.go](./example/bucket/putACL.go)） +* [x] Put Bucket CORS（使用示例：[bucket/putCORS.go](./example/bucket/putCORS.go)） +* [x] Put Bucket Lifecycle（使用示例：[bucket/putLifecycle.go](./example/bucket/putLifecycle.go)） +* [x] Put Bucket Tagging（使用示例：[bucket/putTagging.go](./example/bucket/putTagging.go)） +* [x] Delete Bucket（使用示例：[bucket/delete.go](./example/bucket/delete.go)） +* [x] Delete Bucket CORS（使用示例：[bucket/deleteCORS.go](./example/bucket/deleteCORS.go)） +* [x] Delete Bucket Lifecycle（使用示例：[bucket/deleteLifecycle.go](./example/bucket/deleteLifecycle.go)） +* [x] Delete Bucket Tagging（使用示例：[bucket/deleteTagging.go](./example/bucket/deleteTagging.go)） +* [x] Head Bucket（使用示例：[bucket/head.go](./example/bucket/head.go)） +* [x] List Multipart Uploads（使用示例：[bucket/listMultipartUploads.go](./example/bucket/listMultipartUploads.go)） + +Object API: + +* [x] Get Object（使用示例：[object/get.go](./example/object/get.go)） +* [x] Get Object ACL（使用示例：[object/getACL.go](./example/object/getACL.go)） +* [x] Put Object（使用示例：[object/put.go](./example/object/put.go)） +* [x] Put Object ACL（使用示例：[object/putACL.go](./example/object/putACL.go)） +* [x] Put Object Copy（使用示例：[object/copy.go](./example/object/copy.go)） +* [x] Delete Object（使用示例：[object/delete.go](./example/object/delete.go)） +* [x] Delete Multiple Object（使用示例：[object/deleteMultiple.go](./example/object/deleteMultiple.go)） +* [x] Head Object（使用示例：[object/head.go](./example/object/head.go)） +* [x] Options Object（使用示例：[object/options.go](./example/object/options.go)） +* [x] Initiate Multipart Upload（使用示例：[object/initiateMultipartUpload.go](./example/object/initiateMultipartUpload.go)） +* [x] Upload Part（使用示例：[object/uploadPart.go](./example/object/uploadPart.go)） +* [x] List Parts（使用示例：[object/listParts.go](./example/object/listParts.go)） +* [x] Complete Multipart Upload（使用示例：[object/completeMultipartUpload.go](./example/object/completeMultipartUpload.go)） +* [x] Abort Multipart Upload（使用示例：[object/abortMultipartUpload.go](./example/object/abortMultipartUpload.go)） +* [x] Mutipart Upload（使用示例：[object/MutiUpload.go](./example/object/MutiUpload.go)） diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/auth.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/auth.go new file mode 100644 index 0000000000..6b0cc554cf --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/auth.go @@ -0,0 +1,305 @@ +package cos + +import ( + "crypto/hmac" + "crypto/sha1" + "fmt" + "hash" + "net/http" + "net/url" + "sort" + "strings" + "sync" + "time" +) + +const sha1SignAlgorithm = "sha1" +const privateHeaderPrefix = "x-cos-" +const defaultAuthExpire = time.Hour + +// 需要校验的 Headers 列表 +var needSignHeaders = map[string]bool{ + "host": true, + "range": true, + "x-cos-acl": true, + "x-cos-grant-read": true, + "x-cos-grant-write": true, + "x-cos-grant-full-control": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "cache-control": true, + "content-disposition": true, + "content-encoding": true, + "content-type": true, + "content-length": true, + "content-md5": true, + "expect": true, + "expires": true, + "x-cos-content-sha1": true, + "x-cos-storage-class": true, + "if-modified-since": true, + "origin": true, + "access-control-request-method": true, + "access-control-request-headers": true, + "x-cos-object-type": true, +} + +func safeURLEncode(s string) string { + s = encodeURIComponent(s) + s = strings.Replace(s, "!", "%21", -1) + s = strings.Replace(s, "'", "%27", -1) + s = strings.Replace(s, "(", "%28", -1) + s = strings.Replace(s, ")", "%29", -1) + s = strings.Replace(s, "*", "%2A", -1) + return s +} + +type valuesSignMap map[string][]string + +func (vs valuesSignMap) Add(key, value string) { + key = strings.ToLower(key) + vs[key] = append(vs[key], value) +} + +func (vs valuesSignMap) Encode() string { + var keys []string + for k := range vs { + keys = append(keys, k) + } + sort.Strings(keys) + + var pairs []string + for _, k := range keys { + items := vs[k] + sort.Strings(items) + for _, val := range items { + pairs = append( + pairs, + fmt.Sprintf("%s=%s", safeURLEncode(k), safeURLEncode(val))) + } + } + return strings.Join(pairs, "&") +} + +// AuthTime 用于生成签名所需的 q-sign-time 和 q-key-time 相关参数 +type AuthTime struct { + SignStartTime time.Time + SignEndTime time.Time + KeyStartTime time.Time + KeyEndTime time.Time +} + +// NewAuthTime 生成 AuthTime 的便捷函数 +// +// expire: 从现在开始多久过期. +func NewAuthTime(expire time.Duration) *AuthTime { + signStartTime := time.Now() + keyStartTime := signStartTime + signEndTime := signStartTime.Add(expire) + keyEndTime := signEndTime + return &AuthTime{ + SignStartTime: signStartTime, + SignEndTime: signEndTime, + KeyStartTime: keyStartTime, + KeyEndTime: keyEndTime, + } +} + +// signString return q-sign-time string +func (a *AuthTime) signString() string { + return fmt.Sprintf("%d;%d", a.SignStartTime.Unix(), a.SignEndTime.Unix()) +} + +// keyString return q-key-time string +func (a *AuthTime) keyString() string { + return fmt.Sprintf("%d;%d", a.KeyStartTime.Unix(), a.KeyEndTime.Unix()) +} + +// newAuthorization 通过一系列步骤生成最终需要的 Authorization 字符串 +func newAuthorization(secretID, secretKey string, req *http.Request, authTime *AuthTime) string { + signTime := authTime.signString() + keyTime := authTime.keyString() + signKey := calSignKey(secretKey, keyTime) + + formatHeaders := *new(string) + signedHeaderList := *new([]string) + formatHeaders, signedHeaderList = genFormatHeaders(req.Header) + formatParameters, signedParameterList := genFormatParameters(req.URL.Query()) + formatString := genFormatString(req.Method, *req.URL, formatParameters, formatHeaders) + + stringToSign := calStringToSign(sha1SignAlgorithm, keyTime, formatString) + signature := calSignature(signKey, stringToSign) + + return genAuthorization( + secretID, signTime, keyTime, signature, signedHeaderList, + signedParameterList, + ) +} + +// AddAuthorizationHeader 给 req 增加签名信息 +func AddAuthorizationHeader(secretID, secretKey string, sessionToken string, req *http.Request, authTime *AuthTime) { + if secretID == "" { + return + } + + auth := newAuthorization(secretID, secretKey, req, + authTime, + ) + if len(sessionToken) > 0 { + req.Header.Set("x-cos-security-token", sessionToken) + } + req.Header.Set("Authorization", auth) +} + +// calSignKey 计算 SignKey +func calSignKey(secretKey, keyTime string) string { + digest := calHMACDigest(secretKey, keyTime, sha1SignAlgorithm) + return fmt.Sprintf("%x", digest) +} + +// calStringToSign 计算 StringToSign +func calStringToSign(signAlgorithm, signTime, formatString string) string { + h := sha1.New() + h.Write([]byte(formatString)) + return fmt.Sprintf("%s\n%s\n%x\n", signAlgorithm, signTime, h.Sum(nil)) +} + +// calSignature 计算 Signature +func calSignature(signKey, stringToSign string) string { + digest := calHMACDigest(signKey, stringToSign, sha1SignAlgorithm) + return fmt.Sprintf("%x", digest) +} + +// genAuthorization 生成 Authorization +func genAuthorization(secretID, signTime, keyTime, signature string, signedHeaderList, signedParameterList []string) string { + return strings.Join([]string{ + "q-sign-algorithm=" + sha1SignAlgorithm, + "q-ak=" + secretID, + "q-sign-time=" + signTime, + "q-key-time=" + keyTime, + "q-header-list=" + strings.Join(signedHeaderList, ";"), + "q-url-param-list=" + strings.Join(signedParameterList, ";"), + "q-signature=" + signature, + }, "&") +} + +// genFormatString 生成 FormatString +func genFormatString(method string, uri url.URL, formatParameters, formatHeaders string) string { + formatMethod := strings.ToLower(method) + formatURI := uri.Path + + return fmt.Sprintf("%s\n%s\n%s\n%s\n", formatMethod, formatURI, + formatParameters, formatHeaders, + ) +} + +// genFormatParameters 生成 FormatParameters 和 SignedParameterList +// instead of the url.Values{} +func genFormatParameters(parameters url.Values) (formatParameters string, signedParameterList []string) { + ps := valuesSignMap{} + for key, values := range parameters { + key = strings.ToLower(key) + for _, value := range values { + ps.Add(key, value) + signedParameterList = append(signedParameterList, key) + } + } + //formatParameters = strings.ToLower(ps.Encode()) + formatParameters = ps.Encode() + sort.Strings(signedParameterList) + return +} + +// genFormatHeaders 生成 FormatHeaders 和 SignedHeaderList +func genFormatHeaders(headers http.Header) (formatHeaders string, signedHeaderList []string) { + hs := valuesSignMap{} + for key, values := range headers { + key = strings.ToLower(key) + for _, value := range values { + if isSignHeader(key) { + hs.Add(key, value) + signedHeaderList = append(signedHeaderList, key) + } + } + } + formatHeaders = hs.Encode() + sort.Strings(signedHeaderList) + return +} + +// HMAC 签名 +func calHMACDigest(key, msg, signMethod string) []byte { + var hashFunc func() hash.Hash + switch signMethod { + case "sha1": + hashFunc = sha1.New + default: + hashFunc = sha1.New + } + h := hmac.New(hashFunc, []byte(key)) + h.Write([]byte(msg)) + return h.Sum(nil) +} + +func isSignHeader(key string) bool { + for k, v := range needSignHeaders { + if key == k && v { + return true + } + } + return strings.HasPrefix(key, privateHeaderPrefix) +} + +// AuthorizationTransport 给请求增加 Authorization header +type AuthorizationTransport struct { + SecretID string + SecretKey string + SessionToken string + rwLocker sync.RWMutex + // 签名多久过期 + Expire time.Duration + Transport http.RoundTripper +} + +// SetCredential update the SecretID(ak), SercretKey(sk), sessiontoken +func (t *AuthorizationTransport) SetCredential(ak, sk, token string) { + t.rwLocker.Lock() + defer t.rwLocker.Unlock() + t.SecretID = ak + t.SecretKey = sk + t.SessionToken = token +} + +// GetCredential get the ak, sk, token +func (t *AuthorizationTransport) GetCredential() (string, string, string) { + t.rwLocker.RLock() + defer t.rwLocker.RUnlock() + return t.SecretID, t.SecretKey, t.SessionToken +} + +// RoundTrip implements the RoundTripper interface. +func (t *AuthorizationTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = cloneRequest(req) // per RoundTrip contract + if t.Expire == time.Duration(0) { + t.Expire = defaultAuthExpire + } + + ak, sk, token := t.GetCredential() + // 增加 Authorization header + authTime := NewAuthTime(t.Expire) + AddAuthorizationHeader(ak, sk, token, req, authTime) + + resp, err := t.transport().RoundTrip(req) + return resp, err +} + +func (t *AuthorizationTransport) transport() http.RoundTripper { + if t.Transport != nil { + return t.Transport + } + return http.DefaultTransport +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket.go new file mode 100644 index 0000000000..2e3f92c2f6 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket.go @@ -0,0 +1,104 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// BucketService 相关 API +type BucketService service + +// BucketGetResult is the result of GetBucket +type BucketGetResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string + Prefix string `xml:"Prefix,omitempty"` + Marker string `xml:"Marker,omitempty"` + NextMarker string `xml:"NextMarker,omitempty"` + Delimiter string `xml:"Delimiter,omitempty"` + MaxKeys int + IsTruncated bool + Contents []Object `xml:"Contents,omitempty"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix,omitempty"` + EncodingType string `xml:"Encoding-Type,omitempty"` +} + +// BucketGetOptions is the option of GetBucket +type BucketGetOptions struct { + Prefix string `url:"prefix,omitempty"` + Delimiter string `url:"delimiter,omitempty"` + EncodingType string `url:"encoding-type,omitempty"` + Marker string `url:"marker,omitempty"` + MaxKeys int `url:"max-keys,omitempty"` +} + +// Get Bucket请求等同于 List Object请求，可以列出该Bucket下部分或者所有Object，发起该请求需要拥有Read权限。 +// +// https://www.qcloud.com/document/product/436/7734 +func (s *BucketService) Get(ctx context.Context, opt *BucketGetOptions) (*BucketGetResult, *Response, error) { + var res BucketGetResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/", + method: http.MethodGet, + optQuery: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// BucketPutOptions is same to the ACLHeaderOptions +type BucketPutOptions ACLHeaderOptions + +// Put Bucket请求可以在指定账号下创建一个Bucket。 +// +// https://www.qcloud.com/document/product/436/7738 +func (s *BucketService) Put(ctx context.Context, opt *BucketPutOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/", + method: http.MethodPut, + optHeader: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// Delete Bucket请求可以在指定账号下删除Bucket，删除之前要求Bucket为空。 +// +// https://www.qcloud.com/document/product/436/7732 +func (s *BucketService) Delete(ctx context.Context) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/", + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// Head Bucket请求可以确认是否存在该Bucket，是否有权限访问，Head的权限与Read一致。 +// +// 当其存在时，返回 HTTP 状态码200； +// 当无权限时，返回 HTTP 状态码403； +// 当不存在时，返回 HTTP 状态码404。 +// +// https://www.qcloud.com/document/product/436/7735 +func (s *BucketService) Head(ctx context.Context) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/", + method: http.MethodHead, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// Bucket is the meta info of Bucket +type Bucket struct { + Name string + Region string `xml:"Location,omitempty"` + CreationDate string `xml:",omitempty"` +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_acl.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_acl.go new file mode 100644 index 0000000000..285b9064b4 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_acl.go @@ -0,0 +1,62 @@ +package cos + +import ( + "context" + "net/http" +) + +// BucketGetACLResult is same to the ACLXml +type BucketGetACLResult ACLXml + +// GetACL 使用API读取Bucket的ACL表，只有所有者有权操作。 +// +// https://www.qcloud.com/document/product/436/7733 +func (s *BucketService) GetACL(ctx context.Context) (*BucketGetACLResult, *Response, error) { + var res BucketGetACLResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?acl", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// BucketPutACLOptions is the option of PutBucketACL +type BucketPutACLOptions struct { + Header *ACLHeaderOptions `url:"-" xml:"-"` + Body *ACLXml `url:"-" header:"-"` +} + +// PutACL 使用API写入Bucket的ACL表，您可以通过Header："x-cos-acl","x-cos-grant-read", +// "x-cos-grant-write","x-cos-grant-full-control"传入ACL信息，也可以通过body以XML格式传入ACL信息， +// +// 但是只能选择Header和Body其中一种，否则返回冲突。 +// +// Put Bucket ACL是一个覆盖操作，传入新的ACL将覆盖原有ACL。只有所有者有权操作。 +// +// "x-cos-acl"：枚举值为public-read，private；public-read意味这个Bucket有公有读私有写的权限， +// private意味这个Bucket有私有读写的权限。 +// +// "x-cos-grant-read"：意味被赋予权限的用户拥有该Bucket的读权限 +// "x-cos-grant-write"：意味被赋予权限的用户拥有该Bucket的写权限 +// "x-cos-grant-full-control"：意味被赋予权限的用户拥有该Bucket的读写权限 +// +// https://www.qcloud.com/document/product/436/7737 +func (s *BucketService) PutACL(ctx context.Context, opt *BucketPutACLOptions) (*Response, error) { + header := opt.Header + body := opt.Body + if body != nil { + header = nil + } + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?acl", + method: http.MethodPut, + body: body, + optHeader: header, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_cors.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_cors.go new file mode 100644 index 0000000000..1d688c9b20 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_cors.go @@ -0,0 +1,71 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// BucketCORSRule is the rule of BucketCORS +type BucketCORSRule struct { + ID string `xml:"ID,omitempty"` + AllowedMethods []string `xml:"AllowedMethod"` + AllowedOrigins []string `xml:"AllowedOrigin"` + AllowedHeaders []string `xml:"AllowedHeader,omitempty"` + MaxAgeSeconds int `xml:"MaxAgeSeconds,omitempty"` + ExposeHeaders []string `xml:"ExposeHeader,omitempty"` +} + +// BucketGetCORSResult is the result of GetBucketCORS +type BucketGetCORSResult struct { + XMLName xml.Name `xml:"CORSConfiguration"` + Rules []BucketCORSRule `xml:"CORSRule,omitempty"` +} + +// GetCORS 实现 Bucket 跨域访问配置读取。 +// +// https://www.qcloud.com/document/product/436/8274 +func (s *BucketService) GetCORS(ctx context.Context) (*BucketGetCORSResult, *Response, error) { + var res BucketGetCORSResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?cors", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// BucketPutCORSOptions is the option of PutBucketCORS +type BucketPutCORSOptions struct { + XMLName xml.Name `xml:"CORSConfiguration"` + Rules []BucketCORSRule `xml:"CORSRule,omitempty"` +} + +// PutCORS 实现 Bucket 跨域访问设置，您可以通过传入XML格式的配置文件实现配置，文件大小限制为64 KB。 +// +// https://www.qcloud.com/document/product/436/8279 +func (s *BucketService) PutCORS(ctx context.Context, opt *BucketPutCORSOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?cors", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// DeleteCORS 实现 Bucket 跨域访问配置删除。 +// +// https://www.qcloud.com/document/product/436/8283 +func (s *BucketService) DeleteCORS(ctx context.Context) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?cors", + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_inventory.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_inventory.go new file mode 100644 index 0000000000..17ed781e98 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_inventory.go @@ -0,0 +1,134 @@ +package cos + +import ( + "context" + "encoding/xml" + "fmt" + "net/http" +) + +// Notice bucket_inventory only for test. can not use + +// BucketGetInventoryResult same struct to options +type BucketGetInventoryResult BucketPutInventoryOptions + +// BucketListInventoryConfiguartion same struct to options +type BucketListInventoryConfiguartion BucketPutInventoryOptions + +// BucketInventoryFilter ... +type BucketInventoryFilter struct { + Prefix string `xml:"Prefix,omitempty"` +} + +// BucketInventoryOptionalFields ... +type BucketInventoryOptionalFields struct { + XMLName xml.Name `xml:"OptionalFields,omitempty"` + BucketInventoryFields []string `xml:"Field,omitempty"` +} + +// BucketInventorySchedule ... +type BucketInventorySchedule struct { + Frequency string `xml:"Frequency"` +} + +// BucketInventoryEncryption ... +type BucketInventoryEncryption struct { + XMLName xml.Name `xml:"Encryption"` + SSECOS string `xml:"SSE-COS,omitempty"` +} + +// BucketInventoryDestinationContent ... +type BucketInventoryDestinationContent struct { + Bucket string `xml:"Bucket"` + AccountId string `xml:"AccountId,omitempty"` + Prefix string `xml:"Prefix,omitempty"` + Format string `xml:"Format"` + Encryption *BucketInventoryEncryption `xml:"Encryption,omitempty"` +} + +// BucketInventoryDestination ... +type BucketInventoryDestination struct { + XMLName xml.Name `xml:"Destination"` + BucketDestination *BucketInventoryDestinationContent `xml:"COSBucketDestination"` +} + +// BucketPutInventoryOptions ... +type BucketPutInventoryOptions struct { + XMLName xml.Name `xml:"InventoryConfiguration"` + ID string `xml:"Id"` + IsEnabled string `xml:"IsEnabled"` + IncludedObjectVersions string `xml:"IncludedObjectVersions"` + Filter *BucketInventoryFilter `xml:"Filter,omitempty"` + OptionalFields *BucketInventoryOptionalFields `xml:"OptionalFields,omitempty"` + Schedule *BucketInventorySchedule `xml:"Schedule"` + Destination *BucketInventoryDestination `xml:"Destination"` +} + +// ListBucketInventoryConfigResult result of ListBucketInventoryConfiguration +type ListBucketInventoryConfigResult struct { + XMLName xml.Name `xml:"ListInventoryConfigurationResult"` + InventoryConfigurations []BucketListInventoryConfiguartion `xml:"InventoryConfiguration,omitempty"` + IsTruncated bool `xml:"IsTruncated,omitempty"` + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` +} + +// PutBucketInventory https://cloud.tencent.com/document/product/436/33707 +func (s *BucketService) PutBucketInventoryTest(ctx context.Context, id string, opt *BucketPutInventoryOptions) (*Response, error) { + u := fmt.Sprintf("/?inventory&id=%s", id) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err + +} + +// GetBucketInventory https://cloud.tencent.com/document/product/436/33705 +func (s *BucketService) GetBucketInventoryTest(ctx context.Context, id string) (*BucketGetInventoryResult, *Response, error) { + u := fmt.Sprintf("/?inventory&id=%s", id) + var res BucketGetInventoryResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// DeleteBucketInventory https://cloud.tencent.com/document/product/436/33704 +func (s *BucketService) DeleteBucketInventoryTest(ctx context.Context, id string) (*Response, error) { + u := fmt.Sprintf("/?inventory&id=%s", id) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// ListBucketInventoryConfigurations https://cloud.tencent.com/document/product/436/33706 +func (s *BucketService) ListBucketInventoryConfigurationsTest(ctx context.Context, token string) (*ListBucketInventoryConfigResult, *Response, error) { + var res ListBucketInventoryConfigResult + var u string + if token == "" { + u = "/?inventory" + } else { + u = fmt.Sprintf("/?inventory&continuation-token=%s", encodeURIComponent(token)) + } + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err + +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go new file mode 100644 index 0000000000..c9ca97f482 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go @@ -0,0 +1,92 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// BucketLifecycleFilter is the param of BucketLifecycleRule +type BucketLifecycleFilter struct { + Prefix string `xml:"Prefix,omitempty"` +} + +// BucketLifecycleExpiration is the param of BucketLifecycleRule +type BucketLifecycleExpiration struct { + Date string `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` +} + +// BucketLifecycleTransition is the param of BucketLifecycleRule +type BucketLifecycleTransition struct { + Date string `xml:"Date,omitempty"` + Days int `xml:"Days,omitempty"` + StorageClass string +} + +// BucketLifecycleAbortIncompleteMultipartUpload is the param of BucketLifecycleRule +type BucketLifecycleAbortIncompleteMultipartUpload struct { + DaysAfterInitiation string `xml:"DaysAfterInititation,omitempty"` +} + +// BucketLifecycleRule is the rule of BucketLifecycle +type BucketLifecycleRule struct { + ID string `xml:"ID,omitempty"` + Status string + Filter *BucketLifecycleFilter `xml:"Filter,omitempty"` + Transition *BucketLifecycleTransition `xml:"Transition,omitempty"` + Expiration *BucketLifecycleExpiration `xml:"Expiration,omitempty"` + AbortIncompleteMultipartUpload *BucketLifecycleAbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"` +} + +// BucketGetLifecycleResult is the result of BucketGetLifecycle +type BucketGetLifecycleResult struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + Rules []BucketLifecycleRule `xml:"Rule,omitempty"` +} + +// GetLifecycle 请求实现读取生命周期管理的配置。当配置不存在时，返回404 Not Found。 +// https://www.qcloud.com/document/product/436/8278 +func (s *BucketService) GetLifecycle(ctx context.Context) (*BucketGetLifecycleResult, *Response, error) { + var res BucketGetLifecycleResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?lifecycle", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// BucketPutLifecycleOptions is the option of PutBucketLifecycle +type BucketPutLifecycleOptions struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + Rules []BucketLifecycleRule `xml:"Rule,omitempty"` +} + +// PutLifecycle 请求实现设置生命周期管理的功能。您可以通过该请求实现数据的生命周期管理配置和定期删除。 +// 此请求为覆盖操作，上传新的配置文件将覆盖之前的配置文件。生命周期管理对文件和文件夹同时生效。 +// https://www.qcloud.com/document/product/436/8280 +func (s *BucketService) PutLifecycle(ctx context.Context, opt *BucketPutLifecycleOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?lifecycle", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// DeleteLifecycle 请求实现删除生命周期管理。 +// https://www.qcloud.com/document/product/436/8284 +func (s *BucketService) DeleteLifecycle(ctx context.Context) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?lifecycle", + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_location.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_location.go new file mode 100644 index 0000000000..dd4b5a55d4 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_location.go @@ -0,0 +1,28 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// BucketGetLocationResult is the result of BucketGetLocation +type BucketGetLocationResult struct { + XMLName xml.Name `xml:"LocationConstraint"` + Location string `xml:",chardata"` +} + +// GetLocation 接口获取Bucket所在地域信息，只有Bucket所有者有权限读取信息。 +// +// https://www.qcloud.com/document/product/436/8275 +func (s *BucketService) GetLocation(ctx context.Context) (*BucketGetLocationResult, *Response, error) { + var res BucketGetLocationResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?location", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_logging.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_logging.go new file mode 100644 index 0000000000..d4ea51300a --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_logging.go @@ -0,0 +1,53 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// Notice bucket logging function is testing, can not use. + +// BucketLoggingEnabled main struct of logging +type BucketLoggingEnabled struct { + TargetBucket string `xml:"TargetBucket"` + TargetPrefix string `xml:"TargetPrefix"` +} + +// BucketPutLoggingOptions is the options of PutBucketLogging +type BucketPutLoggingOptions struct { + XMLName xml.Name `xml:"BucketLoggingStatus"` + LoggingEnabled *BucketLoggingEnabled `xml:"LoggingEnabled"` +} + +// BucketGetLoggingResult is the result of GetBucketLogging +type BucketGetLoggingResult struct { + XMLName xml.Name `xml:"BucketLoggingStatus"` + LoggingEnabled *BucketLoggingEnabled `xml:"LoggingEnabled"` +} + +// PutBucketLogging https://cloud.tencent.com/document/product/436/17054 +func (s *BucketService) PutBucketLoggingTest(ctx context.Context, opt *BucketPutLoggingOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?logging", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// GetBucketLogging https://cloud.tencent.com/document/product/436/17053 +func (s *BucketService) GetBucketLoggingTest(ctx context.Context) (*BucketGetLoggingResult, *Response, error) { + var res BucketGetLoggingResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?logging", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err + +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_part.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_part.go new file mode 100644 index 0000000000..e77b4207a2 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_part.go @@ -0,0 +1,57 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// ListMultipartUploadsResult is the result of ListMultipartUploads +type ListMultipartUploadsResult struct { + XMLName xml.Name `xml:"ListMultipartUploadsResult"` + Bucket string `xml:"Bucket"` + EncodingType string `xml:"Encoding-Type"` + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKeyMarker string + NextUploadIDMarker string `xml:"NextUploadIdMarker"` + MaxUploads int + IsTruncated bool + Uploads []struct { + Key string + UploadID string `xml:"UploadId"` + StorageClass string + Initiator *Initiator + Owner *Owner + Initiated string + } `xml:"Upload,omitempty"` + Prefix string + Delimiter string `xml:"delimiter,omitempty"` + CommonPrefixes []string `xml:"CommonPrefixs>Prefix,omitempty"` +} + +// ListMultipartUploadsOptions is the option of ListMultipartUploads +type ListMultipartUploadsOptions struct { + Delimiter string `url:"delimiter,omitempty"` + EncodingType string `url:"encoding-type,omitempty"` + Prefix string `url:"prefix,omitempty"` + MaxUploads int `url:"max-uploads,omitempty"` + KeyMarker string `url:"key-marker,omitempty"` + UploadIDMarker string `url:"upload-id-marker,omitempty"` +} + +// ListMultipartUploads 用来查询正在进行中的分块上传。单次最多列出1000个正在进行中的分块上传。 +// +// https://www.qcloud.com/document/product/436/7736 +func (s *BucketService) ListMultipartUploads(ctx context.Context, opt *ListMultipartUploadsOptions) (*ListMultipartUploadsResult, *Response, error) { + var res ListMultipartUploadsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?uploads", + method: http.MethodGet, + result: &res, + optQuery: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_replication.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_replication.go new file mode 100644 index 0000000000..d0a1a9af91 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_replication.go @@ -0,0 +1,73 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// ReplicationDestination is the sub struct of BucketReplicationRule +type ReplicationDestination struct { + Bucket string `xml:"Bucket"` + StorageClass string `xml:"StorageClass,omitempty"` +} + +// BucketReplicationRule is the main param of replication +type BucketReplicationRule struct { + ID string `xml:"ID,omitempty"` + Status string `xml:"Status"` + Prefix string `xml:"Prefix"` + Destination *ReplicationDestination `xml:"Destination"` +} + +// PutBucketReplicationOptions is the options of PutBucketReplication +type PutBucketReplicationOptions struct { + XMLName xml.Name `xml:"ReplicationConfiguration"` + Role string `xml:"Role"` + Rule []BucketReplicationRule `xml:"Rule"` +} + +// GetBucketReplicationResult is the result of GetBucketReplication +type GetBucketReplicationResult struct { + XMLName xml.Name `xml:"ReplicationConfiguration"` + Role string `xml:"Role"` + Rule []BucketReplicationRule `xml:"Rule"` +} + +// PutBucketReplication https://cloud.tencent.com/document/product/436/19223 +func (s *BucketService) PutBucketReplication(ctx context.Context, opt *PutBucketReplicationOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?replication", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err + +} + +// GetBucketReplication https://cloud.tencent.com/document/product/436/19222 +func (s *BucketService) GetBucketReplication(ctx context.Context) (*GetBucketReplicationResult, *Response, error) { + var res GetBucketReplicationResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?replication", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err + +} + +// DeleteBucketReplication https://cloud.tencent.com/document/product/436/19221 +func (s *BucketService) DeleteBucketReplication(ctx context.Context) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?replication", + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_tagging.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_tagging.go new file mode 100644 index 0000000000..1d8873175e --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_tagging.go @@ -0,0 +1,69 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// BucketTaggingTag is the tag of BucketTagging +type BucketTaggingTag struct { + Key string + Value string +} + +// BucketGetTaggingResult is the result of BucketGetTagging +type BucketGetTaggingResult struct { + XMLName xml.Name `xml:"Tagging"` + TagSet []BucketTaggingTag `xml:"TagSet>Tag,omitempty"` +} + +// GetTagging 接口实现获取指定Bucket的标签。 +// +// https://www.qcloud.com/document/product/436/8277 +func (s *BucketService) GetTagging(ctx context.Context) (*BucketGetTaggingResult, *Response, error) { + var res BucketGetTaggingResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?tagging", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// BucketPutTaggingOptions is the option of BucketPutTagging +type BucketPutTaggingOptions struct { + XMLName xml.Name `xml:"Tagging"` + TagSet []BucketTaggingTag `xml:"TagSet>Tag,omitempty"` +} + +// PutTagging 接口实现给用指定Bucket打标签。用来组织和管理相关Bucket。 +// +// 当该请求设置相同Key名称，不同Value时，会返回400。请求成功，则返回204。 +// +// https://www.qcloud.com/document/product/436/8281 +func (s *BucketService) PutTagging(ctx context.Context, opt *BucketPutTaggingOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?tagging", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// DeleteTagging 接口实现删除指定Bucket的标签。 +// +// https://www.qcloud.com/document/product/436/8286 +func (s *BucketService) DeleteTagging(ctx context.Context) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?tagging", + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_version.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_version.go new file mode 100644 index 0000000000..b74527bd88 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_version.go @@ -0,0 +1,45 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// BucketPutVersionOptions is the options of PutBucketVersioning +type BucketPutVersionOptions struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status"` +} + +// BucketGetVersionResult is the result of GetBucketVersioning +type BucketGetVersionResult struct { + XMLName xml.Name `xml:"VersioningConfiguration"` + Status string `xml:"Status"` +} + +// PutVersion https://cloud.tencent.com/document/product/436/19889 +// Status has Suspended\Enabled +func (s *BucketService) PutVersioning(ctx context.Context, opt *BucketPutVersionOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?versioning", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// GetVersion https://cloud.tencent.com/document/product/436/19888 +func (s *BucketService) GetVersioning(ctx context.Context) (*BucketGetVersionResult, *Response, error) { + var res BucketGetVersionResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?versioning", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go new file mode 100644 index 0000000000..adc381fafc --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go @@ -0,0 +1,346 @@ +package cos + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "text/template" + + "strconv" + + "github.com/google/go-querystring/query" + "github.com/mozillazg/go-httpheader" +) + +const ( + // Version current go sdk version + Version = "0.7.3" + userAgent = "cos-go-sdk-v5/" + Version + contentTypeXML = "application/xml" + defaultServiceBaseURL = "http://service.cos.myqcloud.com" +) + +var bucketURLTemplate = template.Must( + template.New("bucketURLFormat").Parse( + "{{.Schema}}://{{.BucketName}}.cos.{{.Region}}.myqcloud.com", + ), +) + +// BaseURL 访问各 API 所需的基础 URL +type BaseURL struct { + // 访问 bucket, object 相关 API 的基础 URL（不包含 path 部分）: http://example.com + BucketURL *url.URL + // 访问 service API 的基础 URL（不包含 path 部分）: http://example.com + ServiceURL *url.URL +} + +// NewBucketURL 生成 BaseURL 所需的 BucketURL +// +// bucketName: bucket名称, bucket的命名规则为{name}-{appid} ，此处填写的存储桶名称必须为此格式 +// Region: 区域代码: ap-beijing-1,ap-beijing,ap-shanghai,ap-guangzhou... +// secure: 是否使用 https +func NewBucketURL(bucketName, region string, secure bool) *url.URL { + schema := "https" + if !secure { + schema = "http" + } + + w := bytes.NewBuffer(nil) + bucketURLTemplate.Execute(w, struct { + Schema string + BucketName string + Region string + }{ + schema, bucketName, region, + }) + + u, _ := url.Parse(w.String()) + return u +} + +// Client is a client manages communication with the COS API. +type Client struct { + client *http.Client + + UserAgent string + BaseURL *BaseURL + + common service + + Service *ServiceService + Bucket *BucketService + Object *ObjectService +} + +type service struct { + client *Client +} + +// NewClient returns a new COS API client. +func NewClient(uri *BaseURL, httpClient *http.Client) *Client { + if httpClient == nil { + httpClient = &http.Client{} + } + + baseURL := &BaseURL{} + if uri != nil { + baseURL.BucketURL = uri.BucketURL + baseURL.ServiceURL = uri.ServiceURL + } + if baseURL.ServiceURL == nil { + baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL) + } + + c := &Client{ + client: httpClient, + UserAgent: userAgent, + BaseURL: baseURL, + } + c.common.client = c + c.Service = (*ServiceService)(&c.common) + c.Bucket = (*BucketService)(&c.common) + c.Object = (*ObjectService)(&c.common) + return c +} + +func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method string, body interface{}, optQuery interface{}, optHeader interface{}) (req *http.Request, err error) { + uri, err = addURLOptions(uri, optQuery) + if err != nil { + return + } + u, _ := url.Parse(uri) + urlStr := baseURL.ResolveReference(u).String() + + var reader io.Reader + contentType := "" + contentMD5 := "" + if body != nil { + // 上传文件 + if r, ok := body.(io.Reader); ok { + reader = r + } else { + b, err := xml.Marshal(body) + if err != nil { + return nil, err + } + contentType = contentTypeXML + reader = bytes.NewReader(b) + contentMD5 = base64.StdEncoding.EncodeToString(calMD5Digest(b)) + } + } + + req, err = http.NewRequest(method, urlStr, reader) + if err != nil { + return + } + + req.Header, err = addHeaderOptions(req.Header, optHeader) + if err != nil { + return + } + if v := req.Header.Get("Content-Length"); req.ContentLength == 0 && v != "" && v != "0" { + req.ContentLength, _ = strconv.ParseInt(v, 10, 64) + } + + if contentMD5 != "" { + req.Header["Content-MD5"] = []string{contentMD5} + } + if c.UserAgent != "" { + req.Header.Set("User-Agent", c.UserAgent) + } + if req.Header.Get("Content-Type") == "" && contentType != "" { + req.Header.Set("Content-Type", contentType) + } + return +} + +func (c *Client) doAPI(ctx context.Context, req *http.Request, result interface{}, closeBody bool) (*Response, error) { + req = req.WithContext(ctx) + + resp, err := c.client.Do(req) + if err != nil { + // If we got an error, and the context has been canceled, + // the context's error is probably more useful. + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + return nil, err + } + + defer func() { + if closeBody { + // Close the body to let the Transport reuse the connection + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() + } + }() + + response := newResponse(resp) + + err = checkResponse(resp) + if err != nil { + // even though there was an error, we still return the response + // in case the caller wants to inspect it further + return response, err + } + + if result != nil { + if w, ok := result.(io.Writer); ok { + io.Copy(w, resp.Body) + } else { + err = xml.NewDecoder(resp.Body).Decode(result) + if err == io.EOF { + err = nil // ignore EOF errors caused by empty response body + } + } + } + + return response, err +} + +type sendOptions struct { + // 基础 URL + baseURL *url.URL + // URL 中除基础 URL 外的剩余部分 + uri string + // 请求方法 + method string + + body interface{} + // url 查询参数 + optQuery interface{} + // http header 参数 + optHeader interface{} + // 用 result 反序列化 resp.Body + result interface{} + // 是否禁用自动调用 resp.Body.Close() + // 自动调用 Close() 是为了能够重用连接 + disableCloseBody bool +} + +func (c *Client) send(ctx context.Context, opt *sendOptions) (resp *Response, err error) { + req, err := c.newRequest(ctx, opt.baseURL, opt.uri, opt.method, opt.body, opt.optQuery, opt.optHeader) + if err != nil { + return + } + + resp, err = c.doAPI(ctx, req, opt.result, !opt.disableCloseBody) + if err != nil { + return + } + return +} + +// addURLOptions adds the parameters in opt as URL query parameters to s. opt +// must be a struct whose fields may contain "url" tags. +func addURLOptions(s string, opt interface{}) (string, error) { + v := reflect.ValueOf(opt) + if v.Kind() == reflect.Ptr && v.IsNil() { + return s, nil + } + + u, err := url.Parse(s) + if err != nil { + return s, err + } + + qs, err := query.Values(opt) + if err != nil { + return s, err + } + + // 保留原有的参数，并且放在前面。因为 cos 的 url 路由是以第一个参数作为路由的 + // e.g. /?uploads + q := u.RawQuery + rq := qs.Encode() + if q != "" { + if rq != "" { + u.RawQuery = fmt.Sprintf("%s&%s", q, qs.Encode()) + } + } else { + u.RawQuery = rq + } + return u.String(), nil +} + +// addHeaderOptions adds the parameters in opt as Header fields to req. opt +// must be a struct whose fields may contain "header" tags. +func addHeaderOptions(header http.Header, opt interface{}) (http.Header, error) { + v := reflect.ValueOf(opt) + if v.Kind() == reflect.Ptr && v.IsNil() { + return header, nil + } + + h, err := httpheader.Header(opt) + if err != nil { + return nil, err + } + + for key, values := range h { + for _, value := range values { + header.Add(key, value) + } + } + return header, nil +} + +// Owner defines Bucket/Object's owner +type Owner struct { + UIN string `xml:"uin,omitempty"` + ID string `xml:",omitempty"` + DisplayName string `xml:",omitempty"` +} + +// Initiator same to the Owner struct +type Initiator Owner + +// Response API 响应 +type Response struct { + *http.Response +} + +func newResponse(resp *http.Response) *Response { + return &Response{ + Response: resp, + } +} + +// ACLHeaderOptions is the option of ACLHeader +type ACLHeaderOptions struct { + XCosACL string `header:"x-cos-acl,omitempty" url:"-" xml:"-"` + XCosGrantRead string `header:"x-cos-grant-read,omitempty" url:"-" xml:"-"` + XCosGrantWrite string `header:"x-cos-grant-write,omitempty" url:"-" xml:"-"` + XCosGrantFullControl string `header:"x-cos-grant-full-control,omitempty" url:"-" xml:"-"` +} + +// ACLGrantee is the param of ACLGrant +type ACLGrantee struct { + Type string `xml:"type,attr"` + UIN string `xml:"uin,omitempty"` + URI string `xml:"URI,omitempty"` + ID string `xml:",omitempty"` + DisplayName string `xml:",omitempty"` + SubAccount string `xml:"Subaccount,omitempty"` +} + +// ACLGrant is the param of ACLXml +type ACLGrant struct { + Grantee *ACLGrantee + Permission string +} + +// ACLXml is the ACL body struct +type ACLXml struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner *Owner + AccessControlList []ACLGrant `xml:"AccessControlList>Grant,omitempty"` +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/debug/http.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/debug/http.go new file mode 100644 index 0000000000..a0c012a9bd --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/debug/http.go @@ -0,0 +1,70 @@ +package debug + +import ( + "fmt" + "io" + "net/http" + "net/http/httputil" + "os" +) + +// DebugRequestTransport 会打印请求和响应信息, 方便调试. +type DebugRequestTransport struct { + RequestHeader bool + RequestBody bool // RequestHeader 为 true 时,这个选项才会生效 + ResponseHeader bool + ResponseBody bool // ResponseHeader 为 true 时,这个选项才会生效 + + // debug 信息输出到 Writer 中, 默认是 os.Stderr + Writer io.Writer + + Transport http.RoundTripper +} + +// RoundTrip implements the RoundTripper interface. +func (t *DebugRequestTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req = cloneRequest(req) // per RoundTrip contract + w := t.Writer + if w == nil { + w = os.Stderr + } + + if t.RequestHeader { + a, _ := httputil.DumpRequest(req, t.RequestBody) + fmt.Fprintf(w, "%s\n\n", string(a)) + } + + resp, err := t.transport().RoundTrip(req) + if err != nil { + return resp, err + } + + if t.ResponseHeader { + + b, _ := httputil.DumpResponse(resp, t.ResponseBody) + fmt.Fprintf(w, "%s\n", string(b)) + } + + return resp, err +} + +func (t *DebugRequestTransport) transport() http.RoundTripper { + if t.Transport != nil { + return t.Transport + } + return http.DefaultTransport +} + +// cloneRequest returns a clone of the provided *http.Request. The clone is a +// shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/doc.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/doc.go new file mode 100644 index 0000000000..99005ab3a1 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/doc.go @@ -0,0 +1,3 @@ +// Package cos is COS(Cloud Object Storage) Go SDK. The V5 version(XML API). +// There are examples of using each API in the project's 'example' directory. +package cos diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/error.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/error.go new file mode 100644 index 0000000000..9d06194ebd --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/error.go @@ -0,0 +1,49 @@ +package cos + +import ( + "encoding/xml" + "fmt" + "io/ioutil" + "net/http" +) + +// ErrorResponse 包含 API 返回的错误信息 +// +// https://www.qcloud.com/document/product/436/7730 +type ErrorResponse struct { + XMLName xml.Name `xml:"Error"` + Response *http.Response `xml:"-"` + Code string + Message string + Resource string + RequestID string `header:"x-cos-request-id,omitempty" url:"-" xml:"-"` + TraceID string `xml:"TraceId,omitempty"` +} + +// Error returns the error msg +func (r *ErrorResponse) Error() string { + RequestID := r.RequestID + if RequestID == "" { + RequestID = r.Response.Header.Get("X-Cos-Request-Id") + } + TraceID := r.TraceID + if TraceID == "" { + TraceID = r.Response.Header.Get("X-Cos-Trace-Id") + } + return fmt.Sprintf("%v %v: %d %v(Message: %v, RequestId: %v, TraceId: %v)", + r.Response.Request.Method, r.Response.Request.URL, + r.Response.StatusCode, r.Code, r.Message, RequestID, TraceID) +} + +// 检查 response 是否是出错时的返回的 response +func checkResponse(r *http.Response) error { + if c := r.StatusCode; 200 <= c && c <= 299 { + return nil + } + errorResponse := &ErrorResponse{Response: r} + data, err := ioutil.ReadAll(r.Body) + if err == nil && data != nil { + xml.Unmarshal(data, errorResponse) + } + return errorResponse +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go new file mode 100644 index 0000000000..08c5a35df8 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go @@ -0,0 +1,85 @@ +package cos + +import ( + "bytes" + "crypto/md5" + "crypto/sha1" + "fmt" + "net/http" +) + +// 计算 md5 或 sha1 时的分块大小 +const calDigestBlockSize = 1024 * 1024 * 10 + +func calMD5Digest(msg []byte) []byte { + // TODO: 分块计算,减少内存消耗 + m := md5.New() + m.Write(msg) + return m.Sum(nil) +} + +func calSHA1Digest(msg []byte) []byte { + // TODO: 分块计算,减少内存消耗 + m := sha1.New() + m.Write(msg) + return m.Sum(nil) +} + +// cloneRequest returns a clone of the provided *http.Request. The clone is a +// shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + r2 := new(http.Request) + *r2 = *r + // deep copy of the Header + r2.Header = make(http.Header, len(r.Header)) + for k, s := range r.Header { + r2.Header[k] = append([]string(nil), s...) + } + return r2 +} + +// encodeURIComponent like same function in javascript +// +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent +// +// http://www.ecma-international.org/ecma-262/6.0/#sec-uri-syntax-and-semantics +func encodeURIComponent(s string) string { + var b bytes.Buffer + written := 0 + + for i, n := 0, len(s); i < n; i++ { + c := s[i] + + switch c { + case '-', '_', '.', '!', '~', '*', '\'', '(', ')': + continue + default: + // Unreserved according to RFC 3986 sec 2.3 + if 'a' <= c && c <= 'z' { + + continue + + } + if 'A' <= c && c <= 'Z' { + + continue + + } + if '0' <= c && c <= '9' { + + continue + } + } + + b.WriteString(s[written:i]) + fmt.Fprintf(&b, "%%%02X", c) + written = i + 1 + } + + if written == 0 { + return s + } + b.WriteString(s[written:]) + return b.String() +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go new file mode 100644 index 0000000000..6932a93151 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go @@ -0,0 +1,483 @@ +package cos + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// ObjectService 相关 API +type ObjectService service + +// ObjectGetOptions is the option of GetObject +type ObjectGetOptions struct { + ResponseContentType string `url:"response-content-type,omitempty" header:"-"` + ResponseContentLanguage string `url:"response-content-language,omitempty" header:"-"` + ResponseExpires string `url:"response-expires,omitempty" header:"-"` + ResponseCacheControl string `url:"response-cache-control,omitempty" header:"-"` + ResponseContentDisposition string `url:"response-content-disposition,omitempty" header:"-"` + ResponseContentEncoding string `url:"response-content-encoding,omitempty" header:"-"` + Range string `url:"-" header:"Range,omitempty"` + IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"` +} + +// presignedURLTestingOptions is the opt of presigned url +type presignedURLTestingOptions struct { + authTime *AuthTime +} + +// Get Object 请求可以将一个文件（Object）下载至本地。 +// 该操作需要对目标 Object 具有读权限或目标 Object 对所有人都开放了读权限（公有读）。 +// +// https://www.qcloud.com/document/product/436/7753 +func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOptions, id ...string) (*Response, error) { + var u string + if len(id) == 1 { + u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0]) + } else if len(id) == 0 { + u = "/" + encodeURIComponent(name) + } else { + return nil, errors.New("wrong params") + } + + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodGet, + optQuery: opt, + optHeader: opt, + disableCloseBody: true, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// GetToFile download the object to local file +func (s *ObjectService) GetToFile(ctx context.Context, name, localpath string, opt *ObjectGetOptions, id ...string) (*Response, error) { + resp, err := s.Get(ctx, name, opt, id...) + if err != nil { + return resp, err + } + defer resp.Body.Close() + + // If file exist, overwrite it + fd, err := os.OpenFile(localpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) + if err != nil { + return resp, err + } + + _, err = io.Copy(fd, resp.Body) + fd.Close() + if err != nil { + return resp, err + } + + return resp, nil +} + +// GetPresignedURL get the object presigned to down or upload file by url +func (s *ObjectService) GetPresignedURL(ctx context.Context, httpMethod, name, ak, sk string, expired time.Duration, opt interface{}) (*url.URL, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: httpMethod, + optQuery: opt, + optHeader: opt, + } + req, err := s.client.newRequest(ctx, sendOpt.baseURL, sendOpt.uri, sendOpt.method, sendOpt.body, sendOpt.optQuery, sendOpt.optHeader) + if err != nil { + return nil, err + } + + var authTime *AuthTime + if opt != nil { + if opt, ok := opt.(*presignedURLTestingOptions); ok { + authTime = opt.authTime + } + } + if authTime == nil { + authTime = NewAuthTime(expired) + } + authorization := newAuthorization(ak, sk, req, authTime) + sign := encodeURIComponent(authorization) + + if req.URL.RawQuery == "" { + req.URL.RawQuery = fmt.Sprintf("sign=%s", sign) + } else { + req.URL.RawQuery = fmt.Sprintf("%s&sign=%s", req.URL.RawQuery, sign) + } + return req.URL, nil + +} + +// ObjectPutHeaderOptions the options of header of the put object +type ObjectPutHeaderOptions struct { + CacheControl string `header:"Cache-Control,omitempty" url:"-"` + ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"` + ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"` + ContentType string `header:"Content-Type,omitempty" url:"-"` + ContentMD5 string `header:"Content-MD5,omitempty" url:"-"` + ContentLength int `header:"Content-Length,omitempty" url:"-"` + Expect string `header:"Expect,omitempty" url:"-"` + Expires string `header:"Expires,omitempty" url:"-"` + XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"` + // 自定义的 x-cos-meta-* header + XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"` + XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-"` + // 可选值: Normal, Appendable + //XCosObjectType string `header:"x-cos-object-type,omitempty" url:"-"` +} + +// ObjectPutOptions the options of put object +type ObjectPutOptions struct { + *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"` + *ObjectPutHeaderOptions `header:",omitempty" url:"-" xml:"-"` +} + +// Put Object请求可以将一个文件（Oject）上传至指定Bucket。 +// +// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength +// +// https://www.qcloud.com/document/product/436/7749 +func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: http.MethodPut, + body: r, + optHeader: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// PutFromFile put object from local file +// Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory +func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*Response, error) { + fd, err := os.Open(filePath) + if err != nil { + return nil, err + } + defer fd.Close() + + return s.Put(ctx, name, fd, opt) +} + +// ObjectCopyHeaderOptions is the head option of the Copy +type ObjectCopyHeaderOptions struct { + // When use replace directive to update meta infos + CacheControl string `header:"Cache-Control,omitempty" url:"-"` + ContentDisposition string `header:"Content-Disposition,omitempty" url:"-"` + ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"` + ContentType string `header:"Content-Type,omitempty" url:"-"` + Expires string `header:"Expires,omitempty" url:"-"` + Expect string `header:"Expect,omitempty" url:"-"` + XCosMetadataDirective string `header:"x-cos-metadata-directive,omitempty" url:"-" xml:"-"` + XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since,omitempty" url:"-" xml:"-"` + XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since,omitempty" url:"-" xml:"-"` + XCosCopySourceIfMatch string `header:"x-cos-copy-source-If-Match,omitempty" url:"-" xml:"-"` + XCosCopySourceIfNoneMatch string `header:"x-cos-copy-source-If-None-Match,omitempty" url:"-" xml:"-"` + XCosStorageClass string `header:"x-cos-storage-class,omitempty" url:"-" xml:"-"` + // 自定义的 x-cos-meta-* header + XCosMetaXXX *http.Header `header:"x-cos-meta-*,omitempty" url:"-"` + XCosCopySource string `header:"x-cos-copy-source" url:"-" xml:"-"` + XCosServerSideEncryption string `header:"x-cos-server-side-encryption,omitempty" url:"-" xml:"-"` +} + +// ObjectCopyOptions is the option of Copy, choose header or body +type ObjectCopyOptions struct { + *ObjectCopyHeaderOptions `header:",omitempty" url:"-" xml:"-"` + *ACLHeaderOptions `header:",omitempty" url:"-" xml:"-"` +} + +// ObjectCopyResult is the result of Copy +type ObjectCopyResult struct { + XMLName xml.Name `xml:"CopyObjectResult"` + ETag string `xml:"ETag,omitempty"` + LastModified string `xml:"LastModified,omitempty"` +} + +// Copy 调用 PutObjectCopy 请求实现将一个文件从源路径复制到目标路径。建议文件大小 1M 到 5G， +// 超过 5G 的文件请使用分块上传 Upload - Copy。在拷贝的过程中，文件元属性和 ACL 可以被修改。 +// +// 用户可以通过该接口实现文件移动，文件重命名，修改文件属性和创建副本。 +// +// 注意：在跨帐号复制的时候，需要先设置被复制文件的权限为公有读，或者对目标帐号赋权，同帐号则不需要。 +// +// https://cloud.tencent.com/document/product/436/10881 +func (s *ObjectService) Copy(ctx context.Context, name, sourceURL string, opt *ObjectCopyOptions) (*ObjectCopyResult, *Response, error) { + var res ObjectCopyResult + if opt == nil { + opt = new(ObjectCopyOptions) + } + if opt.ObjectCopyHeaderOptions == nil { + opt.ObjectCopyHeaderOptions = new(ObjectCopyHeaderOptions) + } + opt.XCosCopySource = encodeURIComponent(sourceURL) + + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: http.MethodPut, + body: nil, + optHeader: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error. + if err == nil && resp.StatusCode == 200 { + if res.ETag == "" { + return &res, resp, errors.New("response 200 OK, but body contains an error") + } + } + return &res, resp, err +} + +// Delete Object请求可以将一个文件（Object）删除。 +// +// https://www.qcloud.com/document/product/436/7743 +func (s *ObjectService) Delete(ctx context.Context, name string) (*Response, error) { + // When use "" string might call the delete bucket interface + if len(name) == 0 { + return nil, errors.New("empty object name") + } + + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// ObjectHeadOptions is the option of HeadObject +type ObjectHeadOptions struct { + IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"` +} + +// Head Object请求可以取回对应Object的元数据，Head的权限与Get的权限一致 +// +// https://www.qcloud.com/document/product/436/7745 +func (s *ObjectService) Head(ctx context.Context, name string, opt *ObjectHeadOptions, id ...string) (*Response, error) { + var u string + if len(id) == 1 { + u = fmt.Sprintf("/%s?versionId=%s", encodeURIComponent(name), id[0]) + } else if len(id) == 0 { + u = "/" + encodeURIComponent(name) + } else { + return nil, errors.New("wrong params") + } + + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodHead, + optHeader: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + if resp != nil && resp.Header["X-Cos-Object-Type"] != nil && resp.Header["X-Cos-Object-Type"][0] == "appendable" { + resp.Header.Add("x-cos-next-append-position", resp.Header["Content-Length"][0]) + } + + return resp, err +} + +// ObjectOptionsOptions is the option of object options +type ObjectOptionsOptions struct { + Origin string `url:"-" header:"Origin"` + AccessControlRequestMethod string `url:"-" header:"Access-Control-Request-Method"` + AccessControlRequestHeaders string `url:"-" header:"Access-Control-Request-Headers,omitempty"` +} + +// Options Object请求实现跨域访问的预请求。即发出一个 OPTIONS 请求给服务器以确认是否可以进行跨域操作。 +// +// 当CORS配置不存在时，请求返回403 Forbidden。 +// +// https://www.qcloud.com/document/product/436/8288 +func (s *ObjectService) Options(ctx context.Context, name string, opt *ObjectOptionsOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: http.MethodOptions, + optHeader: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// CASJobParameters support three way: Standard(in 35 hours), Expedited(quick way, in 15 mins), Bulk(in 5-12 hours_ +type CASJobParameters struct { + Tier string `xml:"Tier"` +} + +// ObjectRestoreOptions is the option of object restore +type ObjectRestoreOptions struct { + XMLName xml.Name `xml:"RestoreRequest"` + Days int `xml:"Days"` + Tier *CASJobParameters `xml:"CASJobParameters"` +} + +// PutRestore API can recover an object of type archived by COS archive. +// +// https://cloud.tencent.com/document/product/436/12633 +func (s *ObjectService) PostRestore(ctx context.Context, name string, opt *ObjectRestoreOptions) (*Response, error) { + u := fmt.Sprintf("/%s?restore", encodeURIComponent(name)) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodPost, + body: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + + return resp, err +} + +// TODO Append 接口在优化未开放使用 +// +// Append请求可以将一个文件（Object）以分块追加的方式上传至 Bucket 中。使用Append Upload的文件必须事前被设定为Appendable。 +// 当Appendable的文件被执行Put Object的操作以后，文件被覆盖，属性改变为Normal。 +// +// 文件属性可以在Head Object操作中被查询到，当您发起Head Object请求时，会返回自定义Header『x-cos-object-type』，该Header只有两个枚举值：Normal或者Appendable。 +// +// 追加上传建议文件大小1M - 5G。如果position的值和当前Object的长度不致，COS会返回409错误。 +// 如果Append一个Normal的Object，COS会返回409 ObjectNotAppendable。 +// +// Appendable的文件不可以被复制，不参与版本管理，不参与生命周期管理，不可跨区域复制。 +// +// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength +// +// https://www.qcloud.com/document/product/436/7741 +// func (s *ObjectService) Append(ctx context.Context, name string, position int, r io.Reader, opt *ObjectPutOptions) (*Response, error) { +// u := fmt.Sprintf("/%s?append&position=%d", encodeURIComponent(name), position) +// if position != 0{ +// opt = nil +// } +// sendOpt := sendOptions{ +// baseURL: s.client.BaseURL.BucketURL, +// uri: u, +// method: http.MethodPost, +// optHeader: opt, +// body: r, +// } +// resp, err := s.client.send(ctx, &sendOpt) +// return resp, err +// } + +// ObjectDeleteMultiOptions is the option of DeleteMulti +type ObjectDeleteMultiOptions struct { + XMLName xml.Name `xml:"Delete" header:"-"` + Quiet bool `xml:"Quiet" header:"-"` + Objects []Object `xml:"Object" header:"-"` + //XCosSha1 string `xml:"-" header:"x-cos-sha1"` +} + +// ObjectDeleteMultiResult is the result of DeleteMulti +type ObjectDeleteMultiResult struct { + XMLName xml.Name `xml:"DeleteResult"` + DeletedObjects []Object `xml:"Deleted,omitempty"` + Errors []struct { + Key string + Code string + Message string + } `xml:"Error,omitempty"` +} + +// DeleteMulti 请求实现批量删除文件，最大支持单次删除1000个文件。 +// 对于返回结果，COS提供Verbose和Quiet两种结果模式。Verbose模式将返回每个Object的删除结果； +// Quiet模式只返回报错的Object信息。 +// https://www.qcloud.com/document/product/436/8289 +func (s *ObjectService) DeleteMulti(ctx context.Context, opt *ObjectDeleteMultiOptions) (*ObjectDeleteMultiResult, *Response, error) { + var res ObjectDeleteMultiResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?delete", + method: http.MethodPost, + body: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// Object is the meta info of the object +type Object struct { + Key string `xml:",omitempty"` + ETag string `xml:",omitempty"` + Size int `xml:",omitempty"` + PartNumber int `xml:",omitempty"` + LastModified string `xml:",omitempty"` + StorageClass string `xml:",omitempty"` + Owner *Owner `xml:",omitempty"` +} + +type MultiUploadOptions struct { + OptIni *InitiateMultipartUploadOptions + PartSize int +} + +// MultiUpload 为高级upload接口，并发分块上传 +// +// 需要指定分块大小 partSize >= 1 ,单位为MB +// 同时请确认分块数量不超过10000 +// + +func (s *ObjectService) MultiUpload(ctx context.Context, name string, r io.Reader, opt *MultiUploadOptions) (*CompleteMultipartUploadResult, *Response, error) { + + optini := opt.OptIni + res, _, err := s.InitiateMultipartUpload(ctx, name, optini) + if err != nil { + return nil, nil, err + } + uploadID := res.UploadID + bufSize := opt.PartSize * 1024 * 1024 + buffer := make([]byte, bufSize) + optcom := &CompleteMultipartUploadOptions{} + + PartUpload := func(ch chan *Response, ctx context.Context, name string, uploadId string, partNumber int, data io.Reader, opt *ObjectUploadPartOptions) { + + defer func() { + if err := recover(); err != nil { + fmt.Println(err) + } + }() + resp, _ := s.UploadPart(context.Background(), name, uploadId, partNumber, data, nil) + ch <- resp + } + + chs := make([]chan *Response, 10000) + PartNumber := 0 + for i := 1; true; i++ { + bytesread, err := r.Read(buffer) + if err != nil { + if err != io.EOF { + return nil, nil, err + } + PartNumber = i + break + } + chs[i] = make(chan *Response) + go PartUpload(chs[i], context.Background(), name, uploadID, i, strings.NewReader(string(buffer[:bytesread])), nil) + } + + for i := 1; i < PartNumber; i++ { + resp := <-chs[i] + // Notice one part fail can not get the etag according. + etag := resp.Header.Get("ETag") + optcom.Parts = append(optcom.Parts, Object{ + PartNumber: i, ETag: etag}, + ) + } + + v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom) + + return v, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_acl.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_acl.go new file mode 100644 index 0000000000..2dc5935a4b --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_acl.go @@ -0,0 +1,63 @@ +package cos + +import ( + "context" + "net/http" +) + +// ObjectGetACLResult is the result of GetObjectACL +type ObjectGetACLResult ACLXml + +// GetACL Get Object ACL接口实现使用API读取Object的ACL表，只有所有者有权操作。 +// +// https://www.qcloud.com/document/product/436/7744 +func (s *ObjectService) GetACL(ctx context.Context, name string) (*ObjectGetACLResult, *Response, error) { + var res ObjectGetACLResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name) + "?acl", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// ObjectPutACLOptions the options of put object acl +type ObjectPutACLOptions struct { + Header *ACLHeaderOptions `url:"-" xml:"-"` + Body *ACLXml `url:"-" header:"-"` +} + +// PutACL 使用API写入Object的ACL表，您可以通过Header："x-cos-acl", "x-cos-grant-read" , +// "x-cos-grant-write" ,"x-cos-grant-full-control"传入ACL信息， +// 也可以通过body以XML格式传入ACL信息，但是只能选择Header和Body其中一种，否则，返回冲突。 +// +// Put Object ACL是一个覆盖操作，传入新的ACL将覆盖原有ACL。只有所有者有权操作。 +// +// "x-cos-acl"：枚举值为public-read，private；public-read意味这个Object有公有读私有写的权限， +// private意味这个Object有私有读写的权限。 +// +// "x-cos-grant-read"：意味被赋予权限的用户拥有该Object的读权限 +// +// "x-cos-grant-write"：意味被赋予权限的用户拥有该Object的写权限 +// +// "x-cos-grant-full-control"：意味被赋予权限的用户拥有该Object的读写权限 +// +// https://www.qcloud.com/document/product/436/7748 +func (s *ObjectService) PutACL(ctx context.Context, name string, opt *ObjectPutACLOptions) (*Response, error) { + header := opt.Header + body := opt.Body + if body != nil { + header = nil + } + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name) + "?acl", + method: http.MethodPut, + optHeader: header, + body: body, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go new file mode 100644 index 0000000000..6ef5773563 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go @@ -0,0 +1,191 @@ +package cos + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" +) + +// InitiateMultipartUploadOptions is the option of InitateMultipartUpload +type InitiateMultipartUploadOptions struct { + *ACLHeaderOptions + *ObjectPutHeaderOptions +} + +// InitiateMultipartUploadResult is the result of InitateMultipartUpload +type InitiateMultipartUploadResult struct { + XMLName xml.Name `xml:"InitiateMultipartUploadResult"` + Bucket string + Key string + UploadID string `xml:"UploadId"` +} + +// InitiateMultipartUpload 请求实现初始化分片上传，成功执行此请求以后会返回Upload ID用于后续的Upload Part请求。 +// +// https://www.qcloud.com/document/product/436/7746 +func (s *ObjectService) InitiateMultipartUpload(ctx context.Context, name string, opt *InitiateMultipartUploadOptions) (*InitiateMultipartUploadResult, *Response, error) { + var res InitiateMultipartUploadResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name) + "?uploads", + method: http.MethodPost, + optHeader: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// ObjectUploadPartOptions is the options of upload-part +type ObjectUploadPartOptions struct { + Expect string `header:"Expect,omitempty" url:"-"` + XCosContentSHA1 string `header:"x-cos-content-sha1" url:"-"` + ContentLength int `header:"Content-Length,omitempty" url:"-"` +} + +// UploadPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。 +// 在每次请求Upload Part时候，需要携带partNumber和uploadID，partNumber为块的编号，支持乱序上传。 +// +// 当传入uploadID和partNumber都相同的时候，后传入的块将覆盖之前传入的块。当uploadID不存在时会返回404错误，NoSuchUpload. +// +// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ContentLength +// +// https://www.qcloud.com/document/product/436/7750 +func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *ObjectUploadPartOptions) (*Response, error) { + u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodPut, + optHeader: opt, + body: r, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} + +// ObjectListPartsOptions is the option of ListParts +type ObjectListPartsOptions struct { + EncodingType string `url:"Encoding-type,omitempty"` + MaxParts string `url:"max-parts,omitempty"` + PartNumberMarker string `url:"part-number-marker,omitempty"` +} + +// ObjectListPartsResult is the result of ListParts +type ObjectListPartsResult struct { + XMLName xml.Name `xml:"ListPartsResult"` + Bucket string + EncodingType string `xml:"Encoding-type,omitempty"` + Key string + UploadID string `xml:"UploadId"` + Initiator *Initiator `xml:"Initiator,omitempty"` + Owner *Owner `xml:"Owner,omitempty"` + StorageClass string + PartNumberMarker string + NextPartNumberMarker string `xml:"NextPartNumberMarker,omitempty"` + MaxParts string + IsTruncated bool + Parts []Object `xml:"Part,omitempty"` +} + +// ListParts 用来查询特定分块上传中的已上传的块。 +// +// https://www.qcloud.com/document/product/436/7747 +func (s *ObjectService) ListParts(ctx context.Context, name, uploadID string, opt *ObjectListPartsOptions) (*ObjectListPartsResult, *Response, error) { + u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID) + var res ObjectListPartsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodGet, + result: &res, + optQuery: opt, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// CompleteMultipartUploadOptions is the option of CompleteMultipartUpload +type CompleteMultipartUploadOptions struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts []Object `xml:"Part"` +} + +// CompleteMultipartUploadResult is the result CompleteMultipartUpload +type CompleteMultipartUploadResult struct { + XMLName xml.Name `xml:"CompleteMultipartUploadResult"` + Location string + Bucket string + Key string + ETag string +} + +// ObjectList can used for sort the parts which needs in complete upload part +// sort.Sort(cos.ObjectList(opt.Parts)) +type ObjectList []Object + +func (o ObjectList) Len() int { + return len(o) +} + +func (o ObjectList) Swap(i, j int) { + o[i], o[j] = o[j], o[i] +} + +func (o ObjectList) Less(i, j int) bool { // rewrite the Less method from small to big + return o[i].PartNumber < o[j].PartNumber +} + +// CompleteMultipartUpload 用来实现完成整个分块上传。当您已经使用Upload Parts上传所有块以后，你可以用该API完成上传。 +// 在使用该API时，您必须在Body中给出每一个块的PartNumber和ETag，用来校验块的准确性。 +// +// 由于分块上传的合并需要数分钟时间，因而当合并分块开始的时候，COS就立即返回200的状态码，在合并的过程中， +// COS会周期性的返回空格信息来保持连接活跃，直到合并完成，COS会在Body中返回合并后块的内容。 +// +// 当上传块小于1 MB的时候，在调用该请求时，会返回400 EntityTooSmall； +// 当上传块编号不连续的时候，在调用该请求时，会返回400 InvalidPart； +// 当请求Body中的块信息没有按序号从小到大排列的时候，在调用该请求时，会返回400 InvalidPartOrder； +// 当UploadId不存在的时候，在调用该请求时，会返回404 NoSuchUpload。 +// +// 建议您及时完成分块上传或者舍弃分块上传，因为已上传但是未终止的块会占用存储空间进而产生存储费用。 +// +// https://www.qcloud.com/document/product/436/7742 +func (s *ObjectService) CompleteMultipartUpload(ctx context.Context, name, uploadID string, opt *CompleteMultipartUploadOptions) (*CompleteMultipartUploadResult, *Response, error) { + u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID) + var res CompleteMultipartUploadResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodPost, + body: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error. + if err == nil && resp.StatusCode == 200 { + if res.ETag == "" { + return &res, resp, errors.New("response 200 OK, but body contains an error") + } + } + return &res, resp, err +} + +// AbortMultipartUpload 用来实现舍弃一个分块上传并删除已上传的块。当您调用Abort Multipart Upload时， +// 如果有正在使用这个Upload Parts上传块的请求，则Upload Parts会返回失败。当该UploadID不存在时，会返回404 NoSuchUpload。 +// +// 建议您及时完成分块上传或者舍弃分块上传，因为已上传但是未终止的块会占用存储空间进而产生存储费用。 +// +// https://www.qcloud.com/document/product/436/7740 +func (s *ObjectService) AbortMultipartUpload(ctx context.Context, name, uploadID string) (*Response, error) { + u := fmt.Sprintf("/%s?uploadId=%s", encodeURIComponent(name), uploadID) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodDelete, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/service.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/service.go new file mode 100644 index 0000000000..defe938aec --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/service.go @@ -0,0 +1,35 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +// Service 相关 API +type ServiceService service + +// ServiceGetResult is the result of Get Service +type ServiceGetResult struct { + XMLName xml.Name `xml:"ListAllMyBucketsResult"` + Owner *Owner `xml:"Owner"` + Buckets []Bucket `xml:"Buckets>Bucket,omitempty"` +} + +// Get Service 接口实现获取该用户下所有Bucket列表。 +// +// 该API接口需要使用Authorization签名认证， +// 且只能获取签名中AccessID所属账户的Bucket列表。 +// +// https://www.qcloud.com/document/product/436/8291 +func (s *ServiceService) Get(ctx context.Context) (*ServiceGetResult, *Response, error) { + var res ServiceGetResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.ServiceURL, + uri: "/", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6f0cdaf08a..c728f526ff 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -98,6 +98,7 @@ github.com/apache/thrift/lib/go/thrift # github.com/aws/aws-sdk-go v1.15.73 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/credentials +github.com/aws/aws-sdk-go/aws/request github.com/aws/aws-sdk-go/aws/session github.com/aws/aws-sdk-go/service/ec2 github.com/aws/aws-sdk-go/service/iam @@ -108,14 +109,13 @@ github.com/aws/aws-sdk-go/aws/endpoints github.com/aws/aws-sdk-go/internal/sdkio github.com/aws/aws-sdk-go/internal/ini github.com/aws/aws-sdk-go/internal/shareddefaults +github.com/aws/aws-sdk-go/aws/awsutil +github.com/aws/aws-sdk-go/aws/client/metadata github.com/aws/aws-sdk-go/aws/client github.com/aws/aws-sdk-go/aws/corehandlers github.com/aws/aws-sdk-go/aws/credentials/stscreds github.com/aws/aws-sdk-go/aws/csm github.com/aws/aws-sdk-go/aws/defaults -github.com/aws/aws-sdk-go/aws/request -github.com/aws/aws-sdk-go/aws/awsutil -github.com/aws/aws-sdk-go/aws/client/metadata github.com/aws/aws-sdk-go/aws/signer/v4 github.com/aws/aws-sdk-go/internal/sdkrand github.com/aws/aws-sdk-go/private/protocol @@ -170,11 +170,11 @@ github.com/coredns/coredns/plugin/pkg/dnsutil github.com/coredns/coredns/plugin/pkg/fall github.com/coredns/coredns/plugin/pkg/upstream github.com/coredns/coredns/request -github.com/coredns/coredns/plugin/pkg/log github.com/coredns/coredns/pb github.com/coredns/coredns/plugin/metrics/vars github.com/coredns/coredns/plugin/pkg/doh github.com/coredns/coredns/plugin/pkg/edns +github.com/coredns/coredns/plugin/pkg/log github.com/coredns/coredns/plugin/pkg/nonwriter github.com/coredns/coredns/plugin/pkg/parse github.com/coredns/coredns/plugin/pkg/rcode @@ -245,6 +245,8 @@ github.com/gin-gonic/gin/json github.com/gin-gonic/gin/render # github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 github.com/glycerine/go-unsnap-stream +# github.com/go-ini/ini v1.44.0 +github.com/go-ini/ini # github.com/go-logfmt/logfmt v0.4.0 github.com/go-logfmt/logfmt # github.com/go-ole/go-ole v1.2.2 @@ -280,6 +282,8 @@ github.com/golang/protobuf/ptypes/timestamp github.com/golang/snappy # github.com/google/btree v1.0.0 github.com/google/btree +# github.com/google/go-querystring v1.0.0 +github.com/google/go-querystring/query # github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf github.com/google/gofuzz # github.com/google/gopacket v1.1.17 @@ -364,6 +368,15 @@ github.com/mholt/caddy/caddyfile github.com/mholt/caddy/onevent/hook # github.com/miekg/dns v1.1.1 github.com/miekg/dns +# github.com/minio/minio-go v6.0.14+incompatible +github.com/minio/minio-go +github.com/minio/minio-go/pkg/credentials +github.com/minio/minio-go/pkg/encrypt +github.com/minio/minio-go/pkg/s3signer +github.com/minio/minio-go/pkg/s3utils +github.com/minio/minio-go/pkg/set +# github.com/mitchellh/go-homedir v1.1.0 +github.com/mitchellh/go-homedir # github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/concurrent # github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 @@ -372,12 +385,12 @@ github.com/modern-go/reflect2 github.com/mohae/deepcopy # github.com/moul/http2curl v1.0.0 github.com/moul/http2curl +# github.com/mozillazg/go-httpheader v0.2.1 +github.com/mozillazg/go-httpheader # github.com/mozillazg/go-pinyin v0.15.0 github.com/mozillazg/go-pinyin # github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae github.com/mschoch/smat -# github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80 -github.com/nelsonken/cos-go-sdk-v5/cos # github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 github.com/op/go-logging # github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 @@ -456,6 +469,9 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile +# github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f +github.com/tencentyun/cos-go-sdk-v5 +github.com/tencentyun/cos-go-sdk-v5/debug # github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e github.com/texttheater/golang-levenshtein/levenshtein # github.com/tinylib/msgp v1.1.0 @@ -503,13 +519,17 @@ golang.org/x/crypto/pkcs12 golang.org/x/crypto/blowfish golang.org/x/crypto/ed25519/internal/edwards25519 golang.org/x/crypto/internal/subtle +golang.org/x/crypto/argon2 golang.org/x/crypto/pkcs12/internal/rc2 +golang.org/x/crypto/blake2b # golang.org/x/net v0.0.0-20190603091049-60506f45cf65 golang.org/x/net/bpf golang.org/x/net/ipv4 golang.org/x/net/proxy golang.org/x/net/ipv6 golang.org/x/net/trace +golang.org/x/net/http/httpguts +golang.org/x/net/publicsuffix golang.org/x/net/internal/iana golang.org/x/net/internal/socket golang.org/x/net/http2 @@ -517,7 +537,6 @@ golang.org/x/net/context golang.org/x/net/internal/socks golang.org/x/net/internal/timeseries golang.org/x/net/http2/hpack -golang.org/x/net/http/httpguts golang.org/x/net/idna golang.org/x/net/context/ctxhttp # golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 @@ -731,10 +750,10 @@ yunion.io/x/log/hooks yunion.io/x/pkg/util/version yunion.io/x/pkg/utils yunion.io/x/pkg/util/regutils +yunion.io/x/pkg/util/signalutils yunion.io/x/pkg/util/timeutils yunion.io/x/pkg/util/sets yunion.io/x/pkg/trace -yunion.io/x/pkg/util/signalutils yunion.io/x/pkg/util/errors yunion.io/x/pkg/util/netutils yunion.io/x/pkg/util/seclib @@ -759,5 +778,5 @@ yunion.io/x/pkg/util/runtime yunion.io/x/pkg/util/clock # yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda yunion.io/x/sqlchemy -# yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe +# yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d yunion.io/x/structarg diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 64b34fda1d..c65badf90d 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -907,7 +907,7 @@ func (this *ArgumentParser) ParseArgs2(args []string, ignore_unknown bool, setDe } func isQuotedByChar(str string, quoteChar byte) bool { - return str[0] == quoteChar && str[len(str)-1] == quoteChar + return len(str) >= 2 && str[0] == quoteChar && str[len(str)-1] == quoteChar } func isQuoted(str string) bool {