diff --git a/go.mod b/go.mod index 8eff17b256..d76b38de2e 100644 --- a/go.mod +++ b/go.mod @@ -114,7 +114,7 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.5.1 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible - github.com/tencentyun/cos-go-sdk-v5 v0.7.10 + github.com/tencentyun/cos-go-sdk-v5 v0.7.24 github.com/tinylib/msgp v1.1.0 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c @@ -146,13 +146,13 @@ require ( k8s.io/apimachinery v0.19.3 k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 - yunion.io/x/executor v0.0.0-20201231064744-df32f32165a9 + yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v0.0.0-20210204030051-4fb14f92a88f + yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce ) diff --git a/go.sum b/go.sum index c2ab969d6b..90aa09cb0e 100644 --- a/go.sum +++ b/go.sum @@ -588,8 +588,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M= github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8= github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= -github.com/tencentyun/cos-go-sdk-v5 v0.7.10 h1:hJPLdR4RPsnubB3M9qXNBeUbcOJMVANnO/L6Att/PZU= -github.com/tencentyun/cos-go-sdk-v5 v0.7.10/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA= +github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo= +github.com/tencentyun/cos-go-sdk-v5 v0.7.24/go.mod h1:wQBO5HdAkLjj2q6XQiIfDSP8DXDNrppDRw2Kp/1BODA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -910,8 +910,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/executor v0.0.0-20201231064744-df32f32165a9 h1:TyKy58HHgjkTJYrZgD0haz4bItoC+QwcOQ+GtMuGTjw= -yunion.io/x/executor v0.0.0-20201231064744-df32f32165a9/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= +yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ= +yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4= yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= @@ -928,7 +928,7 @@ yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c h1:J/joqbA1N2mAlOl0Uqd4LpAq3+ yunion.io/x/pkg v0.0.0-20210218105412-13a69f60034c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20210204030051-4fb14f92a88f h1:RH3RY/AReFPwASyakFnHyFaHb0N+juxWmwiPlOHZqaY= -yunion.io/x/sqlchemy v0.0.0-20210204030051-4fb14f92a88f/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3 h1:C4yasW3gXronDIlIVUqHSbkA38PM1X9gxUWTYGsuIuI= +yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index 47621af59f..8b02a22a7b 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -268,7 +268,7 @@ func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, s ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{}, } if sizeBytes > 0 { - opts.ContentLength = int(sizeBytes) + opts.ContentLength = sizeBytes } if meta != nil { extraHdr := http.Header{} @@ -364,7 +364,7 @@ func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, p return "", errors.Wrap(err, "GetCosClient") } opts := &cos.ObjectUploadPartOptions{} - opts.ContentLength = int(partSize) + opts.ContentLength = partSize resp, err := coscli.Object.UploadPart(ctx, key, uploadId, partIndex, input, opts) if err != nil { return "", errors.Wrap(err, "UploadPart") diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/CHANGELOG.md b/vendor/github.com/tencentyun/cos-go-sdk-v5/CHANGELOG.md new file mode 100644 index 0000000000..d5b29414a1 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/CHANGELOG.md @@ -0,0 +1,352 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). + +## [v0.7.23](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.22...v0.7.23) - 2021-03-23 + +a stable release + +### Merged + +- latest 0.7.23 stable [`#113`](https://github.com/tencentyun/cos-go-sdk-v5/pull/113) + +### Commits + +- update latest 0.7.23 stable [`17c220d`](https://github.com/tencentyun/cos-go-sdk-v5/commit/17c220d588c93afcdfce86154245daf44b196e0c) +- Updated CHANGELOG.md [`f42ee97`](https://github.com/tencentyun/cos-go-sdk-v5/commit/f42ee9765b9267fc64345d4bb5ca389ed0528c07) + +## [v0.7.22](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.21...v0.7.22) - 2021-03-12 + +update crc + +### Merged + +- Cos v4 dev [`#111`](https://github.com/tencentyun/cos-go-sdk-v5/pull/111) +- update [`#109`](https://github.com/tencentyun/cos-go-sdk-v5/pull/109) + +### Commits + +- Updated CHANGELOG.md [`d58178d`](https://github.com/tencentyun/cos-go-sdk-v5/commit/d58178d869994363c70a4e1b846b4f8ff87ccb84) +- update crc [`23ad6e2`](https://github.com/tencentyun/cos-go-sdk-v5/commit/23ad6e2c72a815f790345446045db132abdec0dd) + +## [v0.7.21](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.20...v0.7.21) - 2021-03-08 + +高级接口增加数据校验功能 + +### Merged + +- Cos v4 dev [`#108`](https://github.com/tencentyun/cos-go-sdk-v5/pull/108) +- Cos v4 dev [`#107`](https://github.com/tencentyun/cos-go-sdk-v5/pull/107) +- Cos v4 dev [`#106`](https://github.com/tencentyun/cos-go-sdk-v5/pull/106) + +### Commits + +- add:支持媒体处理任务接口 [`917abd1`](https://github.com/tencentyun/cos-go-sdk-v5/commit/917abd14626ac6b1dd08c21d46b9a9988f537b1a) +- add:redeme里增加链接 [`d795d20`](https://github.com/tencentyun/cos-go-sdk-v5/commit/d795d20dca35ef730c8b58ba8a1e7e27fd9810d3) +- update ci [`33b02c0`](https://github.com/tencentyun/cos-go-sdk-v5/commit/33b02c0be26631712f47c410e6d1f32f6fae477b) +- update ci put & select [`62fb57e`](https://github.com/tencentyun/cos-go-sdk-v5/commit/62fb57e373a9b86dafbd356820c0e96b20da9ba2) +- add upload verification [`2bfb93f`](https://github.com/tencentyun/cos-go-sdk-v5/commit/2bfb93f7d05ffdd4db0a21cbec410f97f2d51cb6) +- Updated CHANGELOG.md [`e0e66ae`](https://github.com/tencentyun/cos-go-sdk-v5/commit/e0e66ae920c8611459d505d9f847e16e62cb666f) +- add:redeme里增加链接 [`a060fa0`](https://github.com/tencentyun/cos-go-sdk-v5/commit/a060fa03a93c6b7bb2e303e7c270fa0df8895e24) +- update [`45e6cb6`](https://github.com/tencentyun/cos-go-sdk-v5/commit/45e6cb65c3d9758743bba26342ec02ad38f75d22) +- update [`685c58a`](https://github.com/tencentyun/cos-go-sdk-v5/commit/685c58af29276c23ff69aee377b03ed31c7eb856) +- update ci [`deaede4`](https://github.com/tencentyun/cos-go-sdk-v5/commit/deaede440c412277e93e9c0a891089ddb92381f3) + +## [v0.7.20](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.19...v0.7.20) - 2021-02-25 + +fix multicopy and multiupload send on closed channel error + +### Merged + +- fix multicopy and multiupload send on closed channel panic [`#104`](https://github.com/tencentyun/cos-go-sdk-v5/pull/104) +- update DividePart to 64M [`#103`](https://github.com/tencentyun/cos-go-sdk-v5/pull/103) +- update test [`#102`](https://github.com/tencentyun/cos-go-sdk-v5/pull/102) + +### Commits + +- Updated CHANGELOG.md [`eb667d9`](https://github.com/tencentyun/cos-go-sdk-v5/commit/eb667d9ed9fae2a338bee4fffc84381d00dec897) + +## [v0.7.19](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.18...v0.7.19) - 2021-02-01 + +Add Get Object Optional Header && Add Test + +### Merged + +- add get object optional and test [`#101`](https://github.com/tencentyun/cos-go-sdk-v5/pull/101) + +### Commits + +- Updated CHANGELOG.md [`d2273eb`](https://github.com/tencentyun/cos-go-sdk-v5/commit/d2273eb6855df9c5db147ae4e1477eefc2490403) + +## [v0.7.18](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.17...v0.7.18) - 2021-01-11 + +### Merged + +- update upload [`#100`](https://github.com/tencentyun/cos-go-sdk-v5/pull/100) + +### Commits + +- Updated CHANGELOG.md [`5172396`](https://github.com/tencentyun/cos-go-sdk-v5/commit/51723968313c10a106ecf7fdb5227c41247ec066) + +## [v0.7.17](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.16...v0.7.17) - 2020-12-29 + +add multicopy + +### Merged + +- add MultiCopy [`#99`](https://github.com/tencentyun/cos-go-sdk-v5/pull/99) + +### Commits + +- Updated CHANGELOG.md [`91fc87b`](https://github.com/tencentyun/cos-go-sdk-v5/commit/91fc87ba8af042adb0d0bf147f8fe6aa49057976) + +## [v0.7.16](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.15...v0.7.16) - 2020-12-24 + +add bucket accelerate && update upload progress + +### Merged + +- Cos v4 dev [`#98`](https://github.com/tencentyun/cos-go-sdk-v5/pull/98) + +### Commits + +- Add Bucket Accelerate [`c07e497`](https://github.com/tencentyun/cos-go-sdk-v5/commit/c07e49771c809fab640ac9f2c31d776de9dea23d) +- update upload progress && single object length [`72e7751`](https://github.com/tencentyun/cos-go-sdk-v5/commit/72e77516044f833f60577906800a638ebb31dc83) +- Updated CHANGELOG.md [`5057561`](https://github.com/tencentyun/cos-go-sdk-v5/commit/50575619064fbd54e69745b1884756e6f6222a99) +- update version [`6da3d40`](https://github.com/tencentyun/cos-go-sdk-v5/commit/6da3d4094cd8ca8e6840dedcb8b540c14e1f4c93) + +## [v0.7.15](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.14...v0.7.15) - 2020-12-11 + +update ci & ci document + +### Merged + +- update ci [`#96`](https://github.com/tencentyun/cos-go-sdk-v5/pull/96) + +### Commits + +- Updated CHANGELOG.md [`b8afb3f`](https://github.com/tencentyun/cos-go-sdk-v5/commit/b8afb3f85050cee4884a46c8ed49d26bf76d10a4) + +## [v0.7.14](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.13...v0.7.14) - 2020-12-09 + +fix bucket lifecycle + +### Merged + +- update version [`#95`](https://github.com/tencentyun/cos-go-sdk-v5/pull/95) +- fix bucket lifecycle [`#94`](https://github.com/tencentyun/cos-go-sdk-v5/pull/94) + +### Commits + +- Updated CHANGELOG.md [`e3a89ee`](https://github.com/tencentyun/cos-go-sdk-v5/commit/e3a89ee58b4f524c7ad5f2b1f0bdc688a2e39c32) + +## [v0.7.13](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.12...v0.7.13) - 2020-12-08 + +add ci document && add progress + +### Merged + +- Cos v4 dev [`#93`](https://github.com/tencentyun/cos-go-sdk-v5/pull/93) + +### Commits + +- add ci doc [`dad5b1f`](https://github.com/tencentyun/cos-go-sdk-v5/commit/dad5b1f3fbbf30958d3d8ae930b06abcaf8db5ac) +- add progress [`2afc5e1`](https://github.com/tencentyun/cos-go-sdk-v5/commit/2afc5e192cd9bc8d6630fa929e10028ec79bde8e) +- Updated CHANGELOG.md [`e39f3e3`](https://github.com/tencentyun/cos-go-sdk-v5/commit/e39f3e3585e0478abe50b33962e1e3981a2d432c) +- update test [`4899c22`](https://github.com/tencentyun/cos-go-sdk-v5/commit/4899c226f70ccc1bb4ac8489a2b8ff000003bc97) +- add auto changelog workflow [`ca3ea38`](https://github.com/tencentyun/cos-go-sdk-v5/commit/ca3ea38770afcd11e50570a835090c68a377157f) + +## [v0.7.12](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.11...v0.7.12) - 2020-11-25 + +update presignedurl && copy + +### Merged + +- fix ci_test [`#89`](https://github.com/tencentyun/cos-go-sdk-v5/pull/89) +- update version [`#92`](https://github.com/tencentyun/cos-go-sdk-v5/pull/92) +- update presignedurl && copy [`#91`](https://github.com/tencentyun/cos-go-sdk-v5/pull/91) + +## [v0.7.11](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.10...v0.7.11) - 2020-11-15 + +断点续传、select、CI内容审核、CI云上处理、单链接限制 + +### Merged + +- Cos v4 dev [`#90`](https://github.com/tencentyun/cos-go-sdk-v5/pull/90) + +### Commits + +- add select, and x-cos-traffic-limit [`e6f823c`](https://github.com/tencentyun/cos-go-sdk-v5/commit/e6f823c7c1ee66fe68de4aafd45b009dee4b1ac2) +- add ListMultiUploads && 断点续传 [`64d31f3`](https://github.com/tencentyun/cos-go-sdk-v5/commit/64d31f318a223866ac16c64bbe9effcd168b696d) +- ci 内容审核 [`695c446`](https://github.com/tencentyun/cos-go-sdk-v5/commit/695c4466f5cec17acd5143777d4eae90f389f119) +- add checkpoint multi upload [`287669a`](https://github.com/tencentyun/cos-go-sdk-v5/commit/287669a677af36153d645aabd4b296114aea75fe) +- add ci post [`dd7c41c`](https://github.com/tencentyun/cos-go-sdk-v5/commit/dd7c41ca0824b607633e6a7bd0d3a838d52a91fd) +- add CI图片审核 [`98eab28`](https://github.com/tencentyun/cos-go-sdk-v5/commit/98eab2886c628a3fbbbf581aeea56759ff38700c) +- fix list uploads [`5e69c19`](https://github.com/tencentyun/cos-go-sdk-v5/commit/5e69c19d3450d06c3da270ca12e43647432d0ca4) +- add decodeURIComponent [`f6c91c9`](https://github.com/tencentyun/cos-go-sdk-v5/commit/f6c91c92d6e46869c8f7acf000eccce2d351051b) +- add checkpoint multi upload: update [`52c110b`](https://github.com/tencentyun/cos-go-sdk-v5/commit/52c110b7ede92af97e3685d4afd73a128ddd443b) + +## [v0.7.10](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.9...v0.7.10) - 2020-09-29 + +add delete with versionid & fix MultiUpload when filesize equal 0 & Bucket/Object ACL transform + +### Merged + +- ACL转换 [`#87`](https://github.com/tencentyun/cos-go-sdk-v5/pull/87) +- fix bucket encryption & test [`#86`](https://github.com/tencentyun/cos-go-sdk-v5/pull/86) + +### Commits + +- fix MultiUpload when filesize=0 [`cb662cd`](https://github.com/tencentyun/cos-go-sdk-v5/commit/cb662cdad5225cf92be335911d1805a29e445d14) +- 多版本删除 [`0e9536d`](https://github.com/tencentyun/cos-go-sdk-v5/commit/0e9536d989b8e47b71fa02d12cb75d8ffda8b5fc) +- update travis.yml [`b0a399e`](https://github.com/tencentyun/cos-go-sdk-v5/commit/b0a399e92dd143fb55b1ed173497644e51450835) +- update version [`5804e86`](https://github.com/tencentyun/cos-go-sdk-v5/commit/5804e86747f587402b962dcb680ebc0d7c04035d) + +## [v0.7.9](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.8...v0.7.9) - 2020-09-14 + +add bucket intelligenttiering + +### Merged + +- add bucket intelligenttiering [`#85`](https://github.com/tencentyun/cos-go-sdk-v5/pull/85) + +## [v0.7.8](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.7...v0.7.8) - 2020-08-31 + +x-cos-copy-source urlencode修正 + +### Merged + +- x-cos-copy-source urlencode修正 [`#83`](https://github.com/tencentyun/cos-go-sdk-v5/pull/83) +- Common dev [`#82`](https://github.com/tencentyun/cos-go-sdk-v5/pull/82) + +### Commits + +- add host to signature [`f9f6178`](https://github.com/tencentyun/cos-go-sdk-v5/commit/f9f617878dad23f915ecea978f065aa70486843c) + +## [v0.7.7](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.6...v0.7.7) - 2020-06-04 + +add object tagging && bucket origin && add stsv3 demo + +### Merged + +- add object tagging && bucket origin && add stsv3 demo [`#80`](https://github.com/tencentyun/cos-go-sdk-v5/pull/80) + +## [v0.7.6](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.5...v0.7.6) - 2020-05-20 + +Add PutBucketObjectVersions + +### Merged + +- add GetBucketObjectVersions & update tips when failed [`#78`](https://github.com/tencentyun/cos-go-sdk-v5/pull/78) + +### Commits + +- update version [`bcc1ed2`](https://github.com/tencentyun/cos-go-sdk-v5/commit/bcc1ed2b8317cea8a32ab193ffad0a6482d63560) + +## [v0.7.5](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.4...v0.7.5) - 2020-05-14 + +add HeaderOptions to Object.Delete + +### Merged + +- update tag 0.7.5 [`#77`](https://github.com/tencentyun/cos-go-sdk-v5/pull/77) +- feat: Object.Delete 增加 optHeader 传递 [`#75`](https://github.com/tencentyun/cos-go-sdk-v5/pull/75) + +### Commits + +- feat: 改为可选参数，向下兼容 [`2eee514`](https://github.com/tencentyun/cos-go-sdk-v5/commit/2eee5149d727a045e80d28729bb60c90614e8275) + +## [v0.7.4](https://github.com/tencentyun/cos-go-sdk-v5/compare/v0.7.3...v0.7.4) - 2020-04-12 + +add put/get bucket eferer and put/get/delete bucket encryption + +### Merged + +- add encryption and referer [`#74`](https://github.com/tencentyun/cos-go-sdk-v5/pull/74) + +### Commits + +- version 0.7.4 [`da7722b`](https://github.com/tencentyun/cos-go-sdk-v5/commit/da7722b10af59762b8e4f687891850650f0100a9) + +## v0.7.3 - 2020-04-02 + +update ci + +### Merged + +- update ci and error [`#73`](https://github.com/tencentyun/cos-go-sdk-v5/pull/73) +- add policy and IsNoSuchKeyError function [`#72`](https://github.com/tencentyun/cos-go-sdk-v5/pull/72) +- add restore object for batch [`#69`](https://github.com/tencentyun/cos-go-sdk-v5/pull/69) +- Addbatch [`#68`](https://github.com/tencentyun/cos-go-sdk-v5/pull/68) +- inventory/logging/replication/versioning/tagging [`#66`](https://github.com/tencentyun/cos-go-sdk-v5/pull/66) +- add content-language [`#65`](https://github.com/tencentyun/cos-go-sdk-v5/pull/65) +- add host [`#64`](https://github.com/tencentyun/cos-go-sdk-v5/pull/64) +- update MultiUpload [`#63`](https://github.com/tencentyun/cos-go-sdk-v5/pull/63) +- add bucket website and domain interface [`#53`](https://github.com/tencentyun/cos-go-sdk-v5/pull/53) +- 改进：增加对Upload part - copy API的支持 [`#54`](https://github.com/tencentyun/cos-go-sdk-v5/pull/54) +- add sts policy describe [`#60`](https://github.com/tencentyun/cos-go-sdk-v5/pull/60) +- update the sts example [`#59`](https://github.com/tencentyun/cos-go-sdk-v5/pull/59) +- Automatic partitioning for MultiUpload [`#58`](https://github.com/tencentyun/cos-go-sdk-v5/pull/58) +- add sse-c and optional header for put object [`#57`](https://github.com/tencentyun/cos-go-sdk-v5/pull/57) +- fix object upload copy versionid [`#56`](https://github.com/tencentyun/cos-go-sdk-v5/pull/56) +- Fix the multiupload property [`#50`](https://github.com/tencentyun/cos-go-sdk-v5/pull/50) +- Fix the mulupload out of index panic [`#49`](https://github.com/tencentyun/cos-go-sdk-v5/pull/49) +- fix the example of get, reorder the import package [`#48`](https://github.com/tencentyun/cos-go-sdk-v5/pull/48) +- spec the test bucket only used for test [`#47`](https://github.com/tencentyun/cos-go-sdk-v5/pull/47) +- reopen the coverage report [`#46`](https://github.com/tencentyun/cos-go-sdk-v5/pull/46) +- Fix the ci test [`#45`](https://github.com/tencentyun/cos-go-sdk-v5/pull/45) +- add Server Side Encryption Option to ObjectPutHeaderOptions and fixed testing [`#44`](https://github.com/tencentyun/cos-go-sdk-v5/pull/44) +- Fix the outindex of the resp addr in head object [`#42`](https://github.com/tencentyun/cos-go-sdk-v5/pull/42) +- Add the check for anonymous user [`#41`](https://github.com/tencentyun/cos-go-sdk-v5/pull/41) +- Change the param type in listParts [`#38`](https://github.com/tencentyun/cos-go-sdk-v5/pull/38) +- Fix the options param to listparts interface [`#37`](https://github.com/tencentyun/cos-go-sdk-v5/pull/37) +- Add the error when return 200OK but body contains the error [`#34`](https://github.com/tencentyun/cos-go-sdk-v5/pull/34) +- cgi already compact, so remove the sprit [`#32`](https://github.com/tencentyun/cos-go-sdk-v5/pull/32) +- fix the blink in prefix which lead the wrong auth [`#31`](https://github.com/tencentyun/cos-go-sdk-v5/pull/31) +- Support Versioning, Replication, Inventory and Logging API [`#30`](https://github.com/tencentyun/cos-go-sdk-v5/pull/30) +- fix the panic of getting out of index with response header [`#28`](https://github.com/tencentyun/cos-go-sdk-v5/pull/28) +- According the versionid to head and get object, add the opt header of copy object [`#26`](https://github.com/tencentyun/cos-go-sdk-v5/pull/26) +- rm the panic in interface [`#24`](https://github.com/tencentyun/cos-go-sdk-v5/pull/24) +- Add presigned url and demo use it to get object [`#23`](https://github.com/tencentyun/cos-go-sdk-v5/pull/23) +- Check delete object param and fix restore name [`#22`](https://github.com/tencentyun/cos-go-sdk-v5/pull/22) +- Add PostObjectRestore interface [`#21`](https://github.com/tencentyun/cos-go-sdk-v5/pull/21) +- Add vendor files [`#19`](https://github.com/tencentyun/cos-go-sdk-v5/pull/19) +- Add the comment to declare the out of memory issue when put large loc… [`#17`](https://github.com/tencentyun/cos-go-sdk-v5/pull/17) +- Need to improve the interface in the future [`#16`](https://github.com/tencentyun/cos-go-sdk-v5/pull/16) +- Add ci test with the testify and travis, Add get/put by localfile. [`#15`](https://github.com/tencentyun/cos-go-sdk-v5/pull/15) +- url encode without / [`#14`](https://github.com/tencentyun/cos-go-sdk-v5/pull/14) +- solve the oom when send the big file with the debug request header option [`#10`](https://github.com/tencentyun/cos-go-sdk-v5/pull/10) +- remove the content-type when the request body is empty [`#9`](https://github.com/tencentyun/cos-go-sdk-v5/pull/9) +- fix format [`#6`](https://github.com/tencentyun/cos-go-sdk-v5/pull/6) +- fix format of doc [`#5`](https://github.com/tencentyun/cos-go-sdk-v5/pull/5) +- update the comment for godoc [`#4`](https://github.com/tencentyun/cos-go-sdk-v5/pull/4) +- fix test [`#3`](https://github.com/tencentyun/cos-go-sdk-v5/pull/3) +- update rm the append interface [`#2`](https://github.com/tencentyun/cos-go-sdk-v5/pull/2) + +### Commits + +- fix the vendor files [`a5b3ad3`](https://github.com/tencentyun/cos-go-sdk-v5/commit/a5b3ad37d64c5aa82b12a57a6fdb0d2dba5ca1a8) +- first to commit project [`6ad265f`](https://github.com/tencentyun/cos-go-sdk-v5/commit/6ad265fc0c275a4b1615c0f4702a7a992d8d299c) +- add batch [`c57ac81`](https://github.com/tencentyun/cos-go-sdk-v5/commit/c57ac81a16da82605cc42bf9e0c8fd3af2cf62c5) +- Support Versioning, Replication, Inventory and Logging api [`c131870`](https://github.com/tencentyun/cos-go-sdk-v5/commit/c131870916d7c29eb8c4a9512ea59ed7d7b09e18) +- fix path [`121a7d8`](https://github.com/tencentyun/cos-go-sdk-v5/commit/121a7d861c8e0fb048aac5a8bc1a644414b16dc3) +- fix the vendor file [`830b5c5`](https://github.com/tencentyun/cos-go-sdk-v5/commit/830b5c54a51d38b91e1432c7b216350a5c479ad0) +- update bucket inventory/logging/replication/versioning/tagging and test [`d37fd23`](https://github.com/tencentyun/cos-go-sdk-v5/commit/d37fd23cdfe570886024c2c0586a2f14429a10ed) +- Add ci test with the testify and travis, test the working [`4cc9e08`](https://github.com/tencentyun/cos-go-sdk-v5/commit/4cc9e08da1a0ce670cf6d20567059b7529b440d5) +- Add ci test with the testify and travis, test the working [`c323f1b`](https://github.com/tencentyun/cos-go-sdk-v5/commit/c323f1b90cd97b98708216a72c666d89bdbcad8b) +- rm inventory and logging example [`7739e84`](https://github.com/tencentyun/cos-go-sdk-v5/commit/7739e84d42b7a33121b16817f25129050db93db5) +- add policy [`b427226`](https://github.com/tencentyun/cos-go-sdk-v5/commit/b427226ae44b1c0f25b8681147e828f4603a9014) +- update batch [`156deae`](https://github.com/tencentyun/cos-go-sdk-v5/commit/156deaea3caacc5249050ae1e7378ae8af909c5d) +- update bucket domain and website struct [`b97a490`](https://github.com/tencentyun/cos-go-sdk-v5/commit/b97a490e28370fecc475bf8b1a82df0d5a52a424) +- presigned url, and demo [`0c2d28c`](https://github.com/tencentyun/cos-go-sdk-v5/commit/0c2d28caf108cff2a95507a9ff286be679d7925b) +- fix ci_test [`c17efed`](https://github.com/tencentyun/cos-go-sdk-v5/commit/c17efed0aedee60e819d3e9504720ff47958da14) +- fix [`9ebd645`](https://github.com/tencentyun/cos-go-sdk-v5/commit/9ebd6456b6aa788d0d2e97f4a8fb652c1b9a1976) +- update batch [`699fef2`](https://github.com/tencentyun/cos-go-sdk-v5/commit/699fef215987bcf7f2459b9843ce8641900135f7) +- Add the comment to declare the out of memory issue when put large local file [`6c894b8`](https://github.com/tencentyun/cos-go-sdk-v5/commit/6c894b8e4d5f0bb1ca24d444a9f8e18f95027cde) +- go module support [`54101f9`](https://github.com/tencentyun/cos-go-sdk-v5/commit/54101f9739b3e59f201092f79d683a0cf377d7d0) +- optional header for complete upload part [`44546a4`](https://github.com/tencentyun/cos-go-sdk-v5/commit/44546a40372260158652ddc6d805ddfc19f13fa3) diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md b/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md index 0316a137ba..72b2b888c0 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/README.md @@ -93,3 +93,14 @@ Object API: * [x] Complete Multipart Upload（使用示例：[object/completeMultipartUpload.go](./example/object/completeMultipartUpload.go)） * [x] Abort Multipart Upload（使用示例：[object/abortMultipartUpload.go](./example/object/abortMultipartUpload.go)） * [x] Mutipart Upload（使用示例：[object/MutiUpload.go](./example/object/MutiUpload.go)） + + +数据处理 API: + +* [x] 媒体处理（使用示例：[media_process.go](./example/CI/media_process/media_process.go)） +* [x] 文档处理（使用示例：[ci_doc_process.go](./example/CI/doc_process/ci_doc_process.go)） + + +内容审核 API: + +* [x] 视频审核（使用示例：[ci_video_auditing_job.go](./example/CI/content_auditing/ci_video_auditing_job.go)） diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/batch.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/batch.go index 05089dd223..ea10c1942d 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/batch.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/batch.go @@ -13,7 +13,7 @@ type BatchRequestHeaders struct { XCosAppid int `header:"x-cos-appid" xml:"-" url:"-"` ContentLength string `header:"Content-Length,omitempty" xml:"-" url:"-"` ContentType string `header:"Content-Type,omitempty" xml:"-" url:"-"` - Headers *http.Header `header:"-" xml:"-", url:"-"` + Headers *http.Header `header:"-" xml:"-" url:"-"` } // BatchProgressSummary @@ -244,7 +244,7 @@ func (s *BatchService) UpdateJobPriority(ctx context.Context, opt *BatchUpdatePr type BatchUpdateStatusOptions struct { JobId string `header:"-" url:"-" xml:"-"` RequestedJobStatus string `url:"requestedJobStatus" header:"-" xml:"-"` - StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-", xml:"-"` + StatusUpdateReason string `url:"statusUpdateReason,omitempty" header:"-" xml:"-"` } type BatchUpdateStatusResult struct { XMLName xml.Name `xml:"UpdateJobStatusResult"` diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_accelerate.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_accelerate.go new file mode 100644 index 0000000000..0eb4c442c5 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_accelerate.go @@ -0,0 +1,37 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +type BucketPutAccelerateOptions struct { + XMLName xml.Name `xml:"AccelerateConfiguration"` + Status string `xml:"Status,omitempty"` + Type string `xml:"Type,omitempty"` +} +type BucketGetAccelerateResult BucketPutAccelerateOptions + +func (s *BucketService) PutAccelerate(ctx context.Context, opt *BucketPutAccelerateOptions) (*Response, error) { + sendOpt := &sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?accelerate", + method: http.MethodPut, + body: opt, + } + resp, err := s.client.send(ctx, sendOpt) + return resp, err +} + +func (s *BucketService) GetAccelerate(ctx context.Context) (*BucketGetAccelerateResult, *Response, error) { + var res BucketGetAccelerateResult + sendOpt := &sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?accelerate", + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, sendOpt) + return &res, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go index c9ca97f482..d5d9096ca7 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/bucket_lifecycle.go @@ -26,7 +26,7 @@ type BucketLifecycleTransition struct { // BucketLifecycleAbortIncompleteMultipartUpload is the param of BucketLifecycleRule type BucketLifecycleAbortIncompleteMultipartUpload struct { - DaysAfterInitiation string `xml:"DaysAfterInititation,omitempty"` + DaysAfterInitiation int `xml:"DaysAfterInitiation,omitempty"` } // BucketLifecycleRule is the rule of BucketLifecycle diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/ci.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/ci.go index 2cf17133cd..9516c8e0c6 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/ci.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/ci.go @@ -1,14 +1,21 @@ package cos import ( + "context" "encoding/json" + "encoding/xml" + "hash/crc64" + "io" + "net/http" + "os" ) +type CIService service + type PicOperations struct { IsPicInfo int `json:"is_pic_info,omitempty"` Rules []PicOperationsRules `json:"rules,omitemtpy"` } - type PicOperationsRules struct { Bucket string `json:"bucket,omitempty"` FileId string `json:"fileid"` @@ -16,9 +23,233 @@ type PicOperationsRules struct { } func EncodePicOperations(pic *PicOperations) string { + if pic == nil { + return "" + } bs, err := json.Marshal(pic) if err != nil { return "" } return string(bs) } + +type ImageProcessResult struct { + XMLName xml.Name `xml:"UploadResult"` + OriginalInfo *PicOriginalInfo `xml:"OriginalInfo,omitempty"` + ProcessResults *PicProcessObject `xml:"ProcessResults>Object,omitempty"` +} +type PicOriginalInfo struct { + Key string `xml:"Key,omitempty"` + Location string `xml:"Location,omitempty"` + ImageInfo *PicImageInfo `xml:"ImageInfo,omitempty"` + ETag string `xml:"ETag,omitempty"` +} +type PicImageInfo struct { + Format string `xml:"Format,omitempty"` + Width int `xml:"Width,omitempty"` + Height int `xml:"Height,omitempty"` + Quality int `xml:"Quality,omitempty"` + Ave string `xml:"Ave,omitempty"` + Orientation int `xml:"Orientation,omitempty"` +} +type PicProcessObject struct { + Key string `xml:"Key,omitempty"` + Location string `xml:"Location,omitempty"` + Format string `xml:"Format,omitempty"` + Width int `xml:"Width,omitempty"` + Height int `xml:"Height,omitempty"` + Size int `xml:"Size,omitempty"` + Quality int `xml:"Quality,omitempty"` + ETag string `xml:"ETag,omitempty"` + WatermarkStatus int `xml:"WatermarkStatus,omitempty"` +} + +type picOperationsHeader struct { + PicOperations string `header:"Pic-Operations" xml:"-" url:"-"` +} + +type ImageProcessOptions = PicOperations + +// 云上数据处理 https://cloud.tencent.com/document/product/460/18147 +func (s *CIService) ImageProcess(ctx context.Context, name string, opt *ImageProcessOptions) (*ImageProcessResult, *Response, error) { + header := &picOperationsHeader{ + PicOperations: EncodePicOperations(opt), + } + var res ImageProcessResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name) + "?image_process", + method: http.MethodPost, + optHeader: header, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type ImageRecognitionOptions struct { + CIProcess string `url:"ci-process,omitempty"` + DetectType string `url:"detect-type,omitempty"` +} + +type ImageRecognitionResult struct { + XMLName xml.Name `xml:"RecognitionResult"` + PornInfo *RecognitionInfo `xml:"PornInfo,omitempty"` + TerroristInfo *RecognitionInfo `xml:"TerroristInfo,omitempty"` + PoliticsInfo *RecognitionInfo `xml:"PoliticsInfo,omitempty"` + AdsInfo *RecognitionInfo `xml:"AdsInfo,omitempty"` +} +type RecognitionInfo struct { + Code int `xml:"Code,omitempty"` + Msg string `xml:"Msg,omitempty"` + HitFlag int `xml:"HitFlag,omitempty"` + Score int `xml:"Score,omitempty"` + Label string `xml:"Label,omitempty"` + Count int `xml:"Count,omitempty"` +} + +// 图片审核 https://cloud.tencent.com/document/product/460/37318 +func (s *CIService) ImageRecognition(ctx context.Context, name string, opt *ImageRecognitionOptions) (*ImageRecognitionResult, *Response, error) { + if opt != nil && opt.CIProcess == "" { + opt.CIProcess = "sensitive-content-recognition" + } + var res ImageRecognitionResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: http.MethodGet, + optQuery: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type PutVideoAuditingJobOptions struct { + XMLName xml.Name `xml:"Request"` + InputObject string `xml:"Input>Object"` + Conf *VideoAuditingJobConf `xml:"Conf"` +} +type VideoAuditingJobConf struct { + DetectType string `xml:",omitempty"` + Snapshot *PutVideoAuditingJobSnapshot `xml:",omitempty"` + Callback string `xml:",omitempty"` +} +type PutVideoAuditingJobSnapshot struct { + Mode string `xml:",omitempty"` + Count int `xml:",omitempty"` + TimeInterval float32 `xml:",omitempty"` + Start float32 `xml:",omitempty"` +} + +type PutVideoAuditingJobResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail struct { + JobId string `xml:"JobId,omitempty"` + State string `xml:"State,omitempty"` + CreationTime string `xml:"CreationTime,omitempty"` + Object string `xml:"Object,omitempty"` + } `xml:"JobsDetail,omitempty"` +} + +func (s *CIService) PutVideoAuditingJob(ctx context.Context, opt *PutVideoAuditingJobOptions) (*PutVideoAuditingJobResult, *Response, error) { + var res PutVideoAuditingJobResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/video/auditing", + method: http.MethodPost, + body: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type GetVideoAuditingJobResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail *VideoAuditingJobDetail `xml:",omitempty"` + NonExistJobIds string `xml:",omitempty"` +} +type VideoAuditingJobDetail struct { + Code string `xml:",omitempty"` + Message string `xml:",omitempty"` + JobId string `xml:",omitempty"` + State string `xml:",omitempty"` + CreationTime string `xml:",omitempty"` + Object string `xml:",omitempty"` + SnapshotCount string `xml:",omitempty"` + Result int `xml:",omitempty"` + PornInfo *RecognitionInfo `xml:",omitempty"` + TerrorismInfo *RecognitionInfo `xml:",omitempty"` + PoliticsInfo *RecognitionInfo `xml:",omitempty"` + AdsInfo *RecognitionInfo `xml:",omitempty"` + Snapshot *GetVideoAuditingJobSnapshot `xml:",omitempty"` +} +type GetVideoAuditingJobSnapshot struct { + Url string `xml:",omitempty"` + PornInfo *RecognitionInfo `xml:",omitempty"` + TerrorismInfo *RecognitionInfo `xml:",omitempty"` + PoliticsInfo *RecognitionInfo `xml:",omitempty"` + AdsInfo *RecognitionInfo `xml:",omitempty"` +} + +func (s *CIService) GetVideoAuditingJob(ctx context.Context, jobid string) (*GetVideoAuditingJobResult, *Response, error) { + var res GetVideoAuditingJobResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/video/auditing/" + jobid, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +// ci put https://cloud.tencent.com/document/product/460/18147 +func (s *CIService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*ImageProcessResult, *Response, error) { + if err := CheckReaderLen(r); err != nil { + return nil, nil, err + } + opt := cloneObjectPutOptions(uopt) + totalBytes, err := GetReaderLen(r) + if err != nil && opt != nil && opt.Listener != nil { + return nil, nil, err + } + if err == nil { + // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传 + if opt != nil && opt.ContentLength == 0 && IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + reader := TeeReader(r, nil, totalBytes, nil) + if s.client.Conf.EnableCRC { + reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA)) + } + if opt != nil && opt.Listener != nil { + reader.listener = opt.Listener + } + + var res ImageProcessResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name), + method: http.MethodPut, + body: reader, + optHeader: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + + return &res, resp, err +} + +// ci put object from local file +func (s *CIService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*ImageProcessResult, *Response, error) { + fd, err := os.Open(filePath) + if err != nil { + return nil, nil, err + } + defer fd.Close() + + return s.Put(ctx, name, fd, opt) +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/ci_doc.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/ci_doc.go new file mode 100644 index 0000000000..30b3214ea8 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/ci_doc.go @@ -0,0 +1,272 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +type DocProcessJobInput struct { + Object string `xml:"Object,omitempty"` +} + +type DocProcessJobOutput struct { + Region string `xml:"Region,omitempty"` + Bucket string `xml:"Bucket,omitempty"` + Object string `xml:"Object,omitempty"` +} + +type DocProcessJobDocProcess struct { + SrcType string `xml:"SrcType,omitempty"` + TgtType string `xml:"TgtType,omitempty"` + SheetId int `xml:"SheetId,omitempty"` + StartPage int `xml:"StartPage,omitempty"` + EndPage int `xml:"EndPage,omitempty"` + ImageParams string `xml:"ImageParams,omitempty"` + DocPassword string `xml:"DocPassword,omitempty"` + Comments int `xml:"Comments,omitempty"` + PaperDirection int `xml:"PaperDirection,omitempty"` + Quality int `xml:"Quality,omitempty"` + Zoom int `xml:"Zoom,omitempty"` +} + +type DocProcessJobDocProcessResult struct { + FailPageCount int `xml:",omitempty"` + SuccPageCount int `xml:"SuccPageCount,omitempty"` + TaskId string `xml:"TaskId,omitempty"` + TgtType string `xml:"TgtType,omitempty"` + TotalPageCount int `xml:"TotalPageCount,omitempty"` + PageInfo struct { + PageNo int `xml:"PageNo,omitempty"` + TgtUri string `xml:"TgtUri,omitempty"` + } `xml:"PageInfo,omitempty"` +} + +type DocProcessJobOperation struct { + Output *DocProcessJobOutput `xml:"Output,omitempty"` + DocProcess *DocProcessJobDocProcess `xml:"DocProcess,omitempty"` + DocProcessResult *DocProcessJobDocProcessResult `xml:"DocProcessResult,omitempty"` +} + +type DocProcessJobDetail struct { + Code string `xml:"Code,omitempty"` + Message string `xml:"Message,omitempty"` + JobId string `xml:"JobId,omitempty"` + Tag string `xml:"Tag,omitempty"` + State string `xml:"State,omitempty"` + CreationTime string `xml:"CreationTime,omitempty"` + QueueId string `xml:"QueueId,omitempty"` + Input *DocProcessJobInput `xml:"Input,omitempty"` + Operation *DocProcessJobOperation `xml:"Operation,omitempty"` +} + +type CreateDocProcessJobsOptions struct { + XMLName xml.Name `xml:"Request"` + Tag string `xml:"Tag,omitempty"` + Input *DocProcessJobInput `xml:"Input,omitempty"` + Operation *DocProcessJobOperation `xml:"Operation,omitempty"` + QueueId string `xml:"QueueId,omitempty"` +} + +type CreateDocProcessJobsResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail DocProcessJobDetail `xml:"JobsDetail,omitempty"` +} + +func (s *CIService) CreateDocProcessJobs(ctx context.Context, opt *CreateDocProcessJobsOptions) (*CreateDocProcessJobsResult, *Response, error) { + var res CreateDocProcessJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/doc_jobs", + method: http.MethodPost, + body: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessJobResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail *DocProcessJobDetail `xml:"JobsDetail,omitempty"` + NonExistJobIds string `xml:"NonExistJobIds,omitempty"` +} + +func (s *CIService) DescribeDocProcessJob(ctx context.Context, jobid string) (*DescribeDocProcessJobResult, *Response, error) { + var res DescribeDocProcessJobResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/doc_jobs/" + jobid, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessJobsOptions struct { + QueueId string `url:"queueId,omitempty"` + Tag string `url:"tag,omitempty"` + OrderByTime string `url:"orderByTime,omitempty"` + NextToken string `url:"nextToken,omitempty"` + Size int `url:"size,omitempty"` + States string `url:"states,omitempty"` + StartCreationTime string `url:"startCreationTime,omitempty"` + EndCreationTime string `url:"endCreationTime,omitempty"` +} + +type DescribeDocProcessJobsResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail []DocProcessJobDetail `xml:"JobsDetail,omitempty"` + NextToken string `xml:"NextToken,omitempty"` +} + +func (s *CIService) DescribeDocProcessJobs(ctx context.Context, opt *DescribeDocProcessJobsOptions) (*DescribeDocProcessJobsResult, *Response, error) { + var res DescribeDocProcessJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/doc_jobs", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessQueuesOptions struct { + QueueIds string `url:"queueIds,omitempty"` + State string `url:"state,omitempty"` + PageNumber int `url:"pageNumber,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +type DescribeDocProcessQueuesResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId,omitempty"` + TotalCount int `xml:"TotalCount,omitempty"` + PageNumber int `xml:"PageNumber,omitempty"` + PageSize int `xml:"PageSize,omitempty"` + QueueList []DocProcessQueue `xml:"QueueList,omitempty"` + NonExistPIDs []string `xml:"NonExistPIDs,omitempty"` +} + +type DocProcessQueue struct { + QueueId string `xml:"QueueId,omitempty"` + Name string `xml:"Name,omitempty"` + State string `xml:"State,omitempty"` + MaxSize int `xml:"MaxSize,omitempty"` + MaxConcurrent int `xml:"MaxConcurrent,omitempty"` + UpdateTime string `xml:"UpdateTime,omitempty"` + CreateTime string `xml:"CreateTime,omitempty"` + NotifyConfig *DocProcessQueueNotifyConfig `xml:"NotifyConfig,omitempty"` +} + +type DocProcessQueueNotifyConfig struct { + Url string `xml:"Url,omitempty"` + State string `xml:"State,omitempty"` + Type string `xml:"Type,omitempty"` + Event string `xml:"Event,omitempty"` +} + +func (s *CIService) DescribeDocProcessQueues(ctx context.Context, opt *DescribeDocProcessQueuesOptions) (*DescribeDocProcessQueuesResult, *Response, error) { + var res DescribeDocProcessQueuesResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/docqueue", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type UpdateDocProcessQueueOptions struct { + XMLName xml.Name `xml:"Request"` + Name string `xml:"Name,omitempty"` + QueueID string `xml:"QueueID,omitempty"` + State string `xml:"State,omitempty"` + NotifyConfig *DocProcessQueueNotifyConfig `xml:"NotifyConfig,omitempty"` +} + +type UpdateDocProcessQueueResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId"` + Queue *DocProcessQueue `xml:"Queue"` +} + +func (s *CIService) UpdateDocProcessQueue(ctx context.Context, opt *UpdateDocProcessQueueOptions) (*UpdateDocProcessQueueResult, *Response, error) { + var res UpdateDocProcessQueueResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/docqueue/" + opt.QueueID, + body: opt, + method: http.MethodPut, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeDocProcessBucketsOptions struct { + Regions string `url:"regions,omitempty"` + BucketNames string `url:"bucketNames,omitempty"` + BucketName string `url:"bucketName,omitempty"` + PageNumber int `url:"pageNumber,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +type DescribeDocProcessBucketsResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId,omitempty"` + TotalCount int `xml:"TotalCount,omitempty"` + PageNumber int `xml:"PageNumber,omitempty"` + PageSize int `xml:"PageSize,omitempty"` + DocBucketList []DocProcessBucket `xml:"DocBucketList,omitempty"` +} +type DocProcessBucket struct { + BucketId string `xml:"BucketId,omitempty"` + Name string `xml:"Name,omitempty"` + Region string `xml:"Region,omitempty"` + CreateTime string `xml:"CreateTime,omitempty"` + AliasBucketId string `xml:"AliasBucketId,omitempty"` +} + +func (s *CIService) DescribeDocProcessBuckets(ctx context.Context, opt *DescribeDocProcessBucketsOptions) (*DescribeDocProcessBucketsResult, *Response, error) { + var res DescribeDocProcessBucketsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/docbucket", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DocPreviewOptions struct { + SrcType string `url:"srcType,omitempty"` + Page int `url:"page,omitempty"` + ImageParams string `url:"ImageParams,omitempty"` + Sheet int `url:"sheet,omitempty"` + DstType string `url:"dstType,omitempty"` + Password string `url:"password,omitempty"` + Comment int `url:"comment,omitempty"` + ExcelPaperDirection int `url:"excelPaperDirection,omitempty"` + Quality int `url:"quality,omitempty"` + Zoom int `url:"zoom,omitempty"` +} + +func (s *CIService) DocPreview(ctx context.Context, name string, opt *DocPreviewOptions) (*Response, error) { + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/" + encodeURIComponent(name) + "?ci-process=doc-preview", + optQuery: opt, + method: http.MethodGet, + disableCloseBody: true, + } + resp, err := s.client.send(ctx, &sendOpt) + return resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/ci_media.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/ci_media.go new file mode 100644 index 0000000000..9a10c61d2e --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/ci_media.go @@ -0,0 +1,303 @@ +package cos + +import ( + "context" + "encoding/xml" + "net/http" +) + +type JobInput struct { + Object string `xml:"Object,omitempty"` +} + +type JobOutput struct { + Region string `xml:"Region,omitempty"` + Bucket string `xml:"Bucket,omitempty"` + Object string `xml:"Object,omitempty"` +} + +type Container struct { + Format string `xml:"Format"` +} + +type Video struct { + Codec string `xml:"Codec"` + Width string `xml:"Width"` + Height string `xml:"Height"` + Fps string `xml:"Fps"` + Remove string `xml:"Remove"` + Profile string `xml:"Profile"` + Bitrate string `xml:"Bitrate"` + Crf string `xml:"Crf"` + Gop string `xml:"Gop"` + Preset string `xml:"Preset"` + Bufsize string `xml:"Bufsize"` + Maxrate string `xml:"Maxrate"` + HlsTsTime string `xml:"HlsTsTime"` + Pixfmt string `xml:"Pixfmt"` + LongShortMode string `xml:"LongShortMode"` +} + +type TimeInterval struct { + Start string `xml:"Start"` + Duration string `xml:"Duration"` +} + +type Audio struct { + Codec string `xml:"Codec"` + Samplerate string `xml:"Samplerate"` + Bitrate string `xml:"Bitrate"` + Channels string `xml:"Channels"` + Remove string `xml:"Remove"` +} + +type TransConfig struct { + AdjDarMethod string `xml:"AdjDarMethod"` + IsCheckReso string `xml:"IsCheckReso"` + ResoAdjMethod string `xml:"ResoAdjMethod"` + IsCheckVideoBitrate string `xml:"IsCheckVideoBitrate"` + VideoBitrateAdjMethod string `xml:"VideoBitrateAdjMethod"` + IsCheckAudioBitrate string `xml:"IsCheckAudioBitrate"` + AudioBitrateAdjMethod string `xml:"AudioBitrateAdjMethod"` +} + +type Transcode struct { + Container *Container `xml:"Container,omitempty"` + Video *Video `xml:"Video,omitempty"` + TimeInterval *TimeInterval `xml:"TimeInterval,omitempty"` + Audio *Audio `xml:"Audio,omitempty"` + TransConfig *TransConfig `xml:"TransConfig,omitempty"` +} + +type Image struct { + Url string `xml:"Url,omitempty"` + Mode string `xml:"Mode,omitempty"` + Width string `xml:"Width,omitempty"` + Height string `xml:"Height,omitempty"` + Transparency string `xml:"Transparency,omitempty"` + Background string `xml:"Background,omitempty"` +} + +type Text struct { + FontSize string `xml:"FontSize,omitempty"` + FontType string `xml:"FontType,omitempty"` + FontColor string `xml:"FontColor,omitempty"` + Transparency string `xml:"Transparency,omitempty"` + Text string `xml:"Text,omitempty"` +} + +type Watermark struct { + Type string `xml:"Type,omitempty"` + Pos string `xml:"Pos,omitempty"` + LocMode string `xml:"LocMode,omitempty"` + Dx string `xml:"Dx,omitempty"` + Dy string `xml:"Dy,omitempty"` + StartTime string `xml:"StartTime,omitempty"` + EndTime string `xml:"EndTime,omitempty"` + Image *Image `xml:"Image,omitempty"` + Text *Text `xml:"Text,omitempty"` +} + +type MediaProcessJobOperation struct { + Output *JobOutput `xml:"Output,omitempty"` + Transcode *Transcode `xml:"Transcode,omitempty"` + Watermark *Watermark `xml:"Watermark,omitempty"` + TemplateId string `xml:"TemplateId,omitempty"` + WatermarkTemplateId []string `xml:"WatermarkTemplateId,omitempty"` +} + +type CreateMediaJobsOptions struct { + XMLName xml.Name `xml:"Request"` + Tag string `xml:"Tag,omitempty"` + Input *JobInput `xml:"Input,omitempty"` + Operation *MediaProcessJobOperation `xml:"Operation,omitempty"` + QueueId string `xml:"QueueId,omitempty"` +} + +type MediaProcessJobDetail struct { + Code string `xml:"Code,omitempty"` + Message string `xml:"Message,omitempty"` + JobId string `xml:"JobId,omitempty"` + Tag string `xml:"Tag,omitempty"` + State string `xml:"State,omitempty"` + CreationTime string `xml:"CreationTime,omitempty"` + QueueId string `xml:"QueueId,omitempty"` + Input *JobInput `xml:"Input,omitempty"` + Operation *MediaProcessJobOperation `xml:"Operation,omitempty"` +} + +type CreateMediaJobsResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail MediaProcessJobDetail `xml:"JobsDetail,omitempty"` +} + +func (s *CIService) CreateMediaJobs(ctx context.Context, opt *CreateMediaJobsOptions) (*CreateMediaJobsResult, *Response, error) { + var res CreateMediaJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/jobs", + method: http.MethodPost, + body: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeMediaProcessJobResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail *MediaProcessJobDetail `xml:"JobsDetail,omitempty"` + NonExistJobIds string `xml:"NonExistJobIds,omitempty"` +} + +func (s *CIService) DescribeMediaJobs(ctx context.Context, jobid string) (*DescribeMediaProcessJobResult, *Response, error) { + var res DescribeMediaProcessJobResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/jobs/" + jobid, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeMediaProcessJobsOptions struct { + QueueId string `url:"queueId,omitempty"` + Tag string `url:"tag,omitempty"` + OrderByTime string `url:"orderByTime,omitempty"` + NextToken string `url:"nextToken,omitempty"` + Size int `url:"size,omitempty"` + States string `url:"states,omitempty"` + StartCreationTime string `url:"startCreationTime,omitempty"` + EndCreationTime string `url:"endCreationTime,omitempty"` +} + +type DescribeMediaProcessJobsResult struct { + XMLName xml.Name `xml:"Response"` + JobsDetail []DocProcessJobDetail `xml:"JobsDetail,omitempty"` + NextToken string `xml:"NextToken,omitempty"` +} + +func (s *CIService) DescribeMediaProcessJobs(ctx context.Context, opt *DescribeMediaProcessJobsOptions) (*DescribeMediaProcessJobsResult, *Response, error) { + var res DescribeMediaProcessJobsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/jobs", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeMediaProcessQueuesOptions struct { + QueueIds string `url:"queueIds,omitempty"` + State string `url:"state,omitempty"` + PageNumber int `url:"pageNumber,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +type DescribeMediaProcessQueuesResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId,omitempty"` + TotalCount int `xml:"TotalCount,omitempty"` + PageNumber int `xml:"PageNumber,omitempty"` + PageSize int `xml:"PageSize,omitempty"` + QueueList []MediaProcessQueue `xml:"QueueList,omitempty"` + NonExistPIDs []string `xml:"NonExistPIDs,omitempty"` +} + +type MediaProcessQueue struct { + QueueId string `xml:"QueueId,omitempty"` + Name string `xml:"Name,omitempty"` + State string `xml:"State,omitempty"` + MaxSize int `xml:"MaxSize,omitempty"` + MaxConcurrent int `xml:"MaxConcurrent,omitempty"` + UpdateTime string `xml:"UpdateTime,omitempty"` + CreateTime string `xml:"CreateTime,omitempty"` + NotifyConfig *MediaProcessQueueNotifyConfig `xml:"NotifyConfig,omitempty"` +} + +type MediaProcessQueueNotifyConfig struct { + Url string `xml:"Url,omitempty"` + State string `xml:"State,omitempty"` + Type string `xml:"Type,omitempty"` + Event string `xml:"Event,omitempty"` +} + +func (s *CIService) DescribeMediaProcessQueues(ctx context.Context, opt *DescribeMediaProcessQueuesOptions) (*DescribeMediaProcessQueuesResult, *Response, error) { + var res DescribeMediaProcessQueuesResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/queue", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type UpdateMediaProcessQueueOptions struct { + XMLName xml.Name `xml:"Request"` + Name string `xml:"Name,omitempty"` + QueueID string `xml:"QueueID,omitempty"` + State string `xml:"State,omitempty"` + NotifyConfig *MediaProcessQueueNotifyConfig `xml:"NotifyConfig,omitempty"` +} + +type UpdateMediaProcessQueueResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId"` + Queue *MediaProcessQueue `xml:"Queue"` +} + +func (s *CIService) UpdateMediaProcessQueue(ctx context.Context, opt *UpdateMediaProcessQueueOptions) (*UpdateMediaProcessQueueResult, *Response, error) { + var res UpdateMediaProcessQueueResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/queue/" + opt.QueueID, + body: opt, + method: http.MethodPut, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} + +type DescribeMediaProcessBucketsOptions struct { + Regions string `url:"regions,omitempty"` + BucketNames string `url:"bucketNames,omitempty"` + BucketName string `url:"bucketName,omitempty"` + PageNumber int `url:"pageNumber,omitempty"` + PageSize int `url:"pageSize,omitempty"` +} + +type DescribeMediaProcessBucketsResult struct { + XMLName xml.Name `xml:"Response"` + RequestId string `xml:"RequestId,omitempty"` + TotalCount int `xml:"TotalCount,omitempty"` + PageNumber int `xml:"PageNumber,omitempty"` + PageSize int `xml:"PageSize,omitempty"` + MediaBucketList []MediaProcessBucket `xml:"MediaBucketList,omitempty"` +} +type MediaProcessBucket struct { + BucketId string `xml:"BucketId,omitempty"` + Region string `xml:"Region,omitempty"` + CreateTime string `xml:"CreateTime,omitempty"` +} + +func (s *CIService) DescribeMediaProcessBuckets(ctx context.Context, opt *DescribeMediaProcessBucketsOptions) (*DescribeMediaProcessBucketsResult, *Response, error) { + var res DescribeMediaProcessBucketsResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.CIURL, + uri: "/mediabucket", + optQuery: opt, + method: http.MethodGet, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + return &res, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go index 665084f6fb..72df0bf315 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/cos.go @@ -22,7 +22,7 @@ import ( const ( // Version current go sdk version - Version = "0.7.10" + Version = "0.7.24" userAgent = "cos-go-sdk-v5/" + Version contentTypeXML = "application/xml" defaultServiceBaseURL = "http://service.cos.myqcloud.com" @@ -42,6 +42,8 @@ type BaseURL struct { ServiceURL *url.URL // 访问 job API 的基础 URL （不包含 path 部分）: http://example.com BatchURL *url.URL + // 访问 CI 的基础 URL + CIURL *url.URL } // NewBucketURL 生成 BaseURL 所需的 BucketURL @@ -68,6 +70,11 @@ func NewBucketURL(bucketName, region string, secure bool) *url.URL { return u } +type Config struct { + EnableCRC bool + RequestBodyClose bool +} + // Client is a client manages communication with the COS API. type Client struct { client *http.Client @@ -82,6 +89,9 @@ type Client struct { Bucket *BucketService Object *ObjectService Batch *BatchService + CI *CIService + + Conf *Config } type service struct { @@ -99,6 +109,7 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { baseURL.BucketURL = uri.BucketURL baseURL.ServiceURL = uri.ServiceURL baseURL.BatchURL = uri.BatchURL + baseURL.CIURL = uri.CIURL } if baseURL.ServiceURL == nil { baseURL.ServiceURL, _ = url.Parse(defaultServiceBaseURL) @@ -108,12 +119,17 @@ func NewClient(uri *BaseURL, httpClient *http.Client) *Client { client: httpClient, UserAgent: userAgent, BaseURL: baseURL, + Conf: &Config{ + EnableCRC: true, + RequestBodyClose: false, + }, } c.common.client = c c.Service = (*ServiceService)(&c.common) c.Bucket = (*BucketService)(&c.common) c.Object = (*ObjectService)(&c.common) c.Batch = (*BatchService)(&c.common) + c.CI = (*CIService)(&c.common) return c } @@ -168,6 +184,9 @@ func (c *Client) newRequest(ctx context.Context, baseURL *url.URL, uri, method s if c.Host != "" { req.Host = c.Host } + if c.Conf.RequestBodyClose { + req.Close = true + } return } @@ -203,6 +222,18 @@ func (c *Client) doAPI(ctx context.Context, req *http.Request, result interface{ return response, err } + // need CRC64 verification + if reader, ok := req.Body.(*teeReader); ok { + if c.Conf.EnableCRC && reader.writer != nil { + localcrc := reader.Crc64() + scoscrc := response.Header.Get("x-cos-hash-crc64ecma") + icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64) + if icoscrc != localcrc { + return response, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc) + } + } + } + if result != nil { if w, ok := result.(io.Writer); ok { io.Copy(w, resp.Body) @@ -244,9 +275,6 @@ func (c *Client) send(ctx context.Context, opt *sendOptions) (resp *Response, er } resp, err = c.doAPI(ctx, req, opt.result, !opt.disableCloseBody) - if err != nil { - return - } return } diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go index 08c5a35df8..7b4acbf016 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/helper.go @@ -4,10 +4,20 @@ import ( "bytes" "crypto/md5" "crypto/sha1" + "errors" "fmt" + "hash/crc64" + "io" "net/http" + "net/url" + "os" + "strings" ) +// 单次上传文件最大为5GB +const singleUploadMaxLength = 5 * 1024 * 1024 * 1024 +const singleUploadThreshold = 32 * 1024 * 1024 + // 计算 md5 或 sha1 时的分块大小 const calDigestBlockSize = 1024 * 1024 * 10 @@ -25,6 +35,17 @@ func calSHA1Digest(msg []byte) []byte { return m.Sum(nil) } +func calCRC64(fd io.Reader) (uint64, error) { + tb := crc64.MakeTable(crc64.ECMA) + hash := crc64.New(tb) + _, err := io.Copy(hash, fd) + if err != nil { + return 0, err + } + sum := hash.Sum64() + return sum, nil +} + // cloneRequest returns a clone of the provided *http.Request. The clone is a // shallow copy of the struct and its Header map. func cloneRequest(r *http.Request) *http.Request { @@ -44,7 +65,7 @@ func cloneRequest(r *http.Request) *http.Request { // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent // // http://www.ecma-international.org/ecma-262/6.0/#sec-uri-syntax-and-semantics -func encodeURIComponent(s string) string { +func encodeURIComponent(s string, excluded ...[]byte) string { var b bytes.Buffer written := 0 @@ -70,6 +91,18 @@ func encodeURIComponent(s string) string { continue } + if len(excluded) > 0 { + conti := false + for _, ch := range excluded[0] { + if ch == c { + conti = true + break + } + } + if conti { + continue + } + } } b.WriteString(s[written:i]) @@ -83,3 +116,124 @@ func encodeURIComponent(s string) string { b.WriteString(s[written:]) return b.String() } + +func decodeURIComponent(s string) (string, error) { + decodeStr, err := url.QueryUnescape(s) + if err != nil { + return s, err + } + return decodeStr, err +} + +func DecodeURIComponent(s string) (string, error) { + return decodeURIComponent(s) +} + +func EncodeURIComponent(s string) string { + return encodeURIComponent(s) +} + +func GetReaderLen(reader io.Reader) (length int64, err error) { + switch v := reader.(type) { + case *bytes.Buffer: + length = int64(v.Len()) + case *bytes.Reader: + length = int64(v.Len()) + case *strings.Reader: + length = int64(v.Len()) + case *os.File: + stat, ferr := v.Stat() + if ferr != nil { + err = fmt.Errorf("can't get reader length: %s", ferr.Error()) + } else { + length = stat.Size() + } + case *io.LimitedReader: + length = int64(v.N) + case *LimitedReadCloser: + length = int64(v.N) + case FixedLengthReader: + length = v.Size() + default: + err = fmt.Errorf("can't get reader content length, unkown reader type") + } + return +} + +func IsLenReader(reader io.Reader) bool { + switch reader.(type) { + case *bytes.Buffer: + return true + case *bytes.Reader: + return true + case *strings.Reader: + return true + default: + return false + } + return false +} + +func CheckReaderLen(reader io.Reader) error { + nlen, err := GetReaderLen(reader) + if err != nil || nlen < singleUploadMaxLength { + return nil + } + return errors.New("The single object size you upload can not be larger than 5GB") +} + +func CopyOptionsToMulti(opt *ObjectCopyOptions) *InitiateMultipartUploadOptions { + if opt == nil { + return nil + } + optini := &InitiateMultipartUploadOptions{ + opt.ACLHeaderOptions, + &ObjectPutHeaderOptions{}, + } + if opt.ObjectCopyHeaderOptions == nil { + return optini + } + optini.ObjectPutHeaderOptions = &ObjectPutHeaderOptions{ + CacheControl: opt.ObjectCopyHeaderOptions.CacheControl, + ContentDisposition: opt.ObjectCopyHeaderOptions.ContentDisposition, + ContentEncoding: opt.ObjectCopyHeaderOptions.ContentEncoding, + ContentType: opt.ObjectCopyHeaderOptions.ContentType, + ContentLanguage: opt.ObjectCopyHeaderOptions.ContentLanguage, + Expect: opt.ObjectCopyHeaderOptions.Expect, + Expires: opt.ObjectCopyHeaderOptions.Expires, + XCosMetaXXX: opt.ObjectCopyHeaderOptions.XCosMetaXXX, + XCosStorageClass: opt.ObjectCopyHeaderOptions.XCosStorageClass, + XCosServerSideEncryption: opt.ObjectCopyHeaderOptions.XCosServerSideEncryption, + XCosSSECustomerAglo: opt.ObjectCopyHeaderOptions.XCosSSECustomerAglo, + XCosSSECustomerKey: opt.ObjectCopyHeaderOptions.XCosSSECustomerKey, + XCosSSECustomerKeyMD5: opt.ObjectCopyHeaderOptions.XCosSSECustomerKeyMD5, + XOptionHeader: opt.ObjectCopyHeaderOptions.XOptionHeader, + } + return optini +} + +// 浅拷贝ObjectPutOptions +func cloneObjectPutOptions(opt *ObjectPutOptions) *ObjectPutOptions { + res := &ObjectPutOptions{ + &ACLHeaderOptions{}, + &ObjectPutHeaderOptions{}, + } + if opt != nil { + if opt.ACLHeaderOptions != nil { + *res.ACLHeaderOptions = *opt.ACLHeaderOptions + } + if opt.ObjectPutHeaderOptions != nil { + *res.ObjectPutHeaderOptions = *opt.ObjectPutHeaderOptions + } + } + return res +} + +// 浅拷贝ObjectUploadPartOptions +func cloneObjectUploadPartOptions(opt *ObjectUploadPartOptions) *ObjectUploadPartOptions { + var res ObjectUploadPartOptions + if opt != nil { + res = *opt + } + return &res +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go index d44e351ce6..bf9ef66c1d 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object.go @@ -2,14 +2,18 @@ package cos import ( "context" + "crypto/md5" "encoding/xml" "errors" "fmt" + "hash/crc64" "io" + "io/ioutil" "net/http" "net/url" "os" "sort" + "strconv" "strings" "time" ) @@ -31,6 +35,13 @@ type ObjectGetOptions struct { XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"` XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` + + //兼容其他自定义头部 + XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` + XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` + + // 下载进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil + Listener ProgressListener `header:"-" url:"-" xml:"-"` } // presignedURLTestingOptions is the opt of presigned url @@ -61,6 +72,14 @@ func (s *ObjectService) Get(ctx context.Context, name string, opt *ObjectGetOpti disableCloseBody: true, } resp, err := s.client.send(ctx, &sendOpt) + + if opt != nil && opt.Listener != nil { + if err == nil && resp != nil { + if totalBytes, e := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64); e == nil { + resp.Body = TeeReader(resp.Body, nil, totalBytes, opt.Listener) + } + } + } return resp, err } @@ -111,12 +130,12 @@ func (s *ObjectService) GetPresignedURL(ctx context.Context, httpMethod, name, a authTime = NewAuthTime(expired) } authorization := newAuthorization(ak, sk, req, authTime) - sign := encodeURIComponent(authorization) + sign := encodeURIComponent(authorization, []byte{'&', '='}) if req.URL.RawQuery == "" { - req.URL.RawQuery = fmt.Sprintf("sign=%s", sign) + req.URL.RawQuery = fmt.Sprintf("%s", sign) } else { - req.URL.RawQuery = fmt.Sprintf("%s&sign=%s", req.URL.RawQuery, sign) + req.URL.RawQuery = fmt.Sprintf("%s&%s", req.URL.RawQuery, sign) } return req.URL, nil @@ -129,7 +148,7 @@ type ObjectPutHeaderOptions struct { ContentEncoding string `header:"Content-Encoding,omitempty" url:"-"` ContentType string `header:"Content-Type,omitempty" url:"-"` ContentMD5 string `header:"Content-MD5,omitempty" url:"-"` - ContentLength int `header:"Content-Length,omitempty" url:"-"` + ContentLength int64 `header:"Content-Length,omitempty" url:"-"` ContentLanguage string `header:"Content-Language,omitempty" url:"-"` Expect string `header:"Expect,omitempty" url:"-"` Expires string `header:"Expires,omitempty" url:"-"` @@ -146,7 +165,11 @@ type ObjectPutHeaderOptions struct { XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` //兼容其他自定义头部 - XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` + XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` + XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` + + // 上传进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil + Listener ProgressListener `header:"-" url:"-" xml:"-"` } // ObjectPutOptions the options of put object @@ -157,31 +180,61 @@ type ObjectPutOptions struct { // Put Object请求可以将一个文件（Oject）上传至指定Bucket。 // -// 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ObjectPutHeaderOptions.ContentLength -// // https://www.qcloud.com/document/product/436/7749 -func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, opt *ObjectPutOptions) (*Response, error) { +func (s *ObjectService) Put(ctx context.Context, name string, r io.Reader, uopt *ObjectPutOptions) (*Response, error) { + if err := CheckReaderLen(r); err != nil { + return nil, err + } + opt := cloneObjectPutOptions(uopt) + totalBytes, err := GetReaderLen(r) + if err != nil && opt != nil && opt.Listener != nil { + return nil, err + } + if err == nil { + // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader由用户指定ContentLength, 或使用 Chunk 上传 + if opt != nil && opt.ContentLength == 0 && IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + reader := TeeReader(r, nil, totalBytes, nil) + if s.client.Conf.EnableCRC { + reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA)) + } + if opt != nil && opt.Listener != nil { + reader.listener = opt.Listener + } sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: "/" + encodeURIComponent(name), method: http.MethodPut, - body: r, + body: reader, optHeader: opt, } resp, err := s.client.send(ctx, &sendOpt) + return resp, err } // PutFromFile put object from local file // Notice that when use this put large file need set non-body of debug req/resp, otherwise will out of memory -func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (*Response, error) { - fd, err := os.Open(filePath) - if err != nil { - return nil, err +func (s *ObjectService) PutFromFile(ctx context.Context, name string, filePath string, opt *ObjectPutOptions) (resp *Response, err error) { + nr := 0 + for nr < 3 { + fd, e := os.Open(filePath) + if e != nil { + err = e + return + } + resp, err = s.Put(ctx, name, fd, opt) + if err != nil { + nr++ + fd.Close() + continue + } + fd.Close() + break } - defer fd.Close() - - return s.Put(ctx, name, fd, opt) + return } // ObjectCopyHeaderOptions is the head option of the Copy @@ -226,6 +279,8 @@ type ObjectCopyResult struct { XMLName xml.Name `xml:"CopyObjectResult"` ETag string `xml:"ETag,omitempty"` LastModified string `xml:"LastModified,omitempty"` + CRC64 string `xml:"CRC64,omitempty"` + VersionId string `xml:"VersionId,omitempty"` } // Copy 调用 PutObjectCopy 请求实现将一个文件从源路径复制到目标路径。建议文件大小 1M 到 5G， @@ -251,20 +306,26 @@ func (s *ObjectService) Copy(ctx context.Context, name, sourceURL string, opt *O } var res ObjectCopyResult - if opt == nil { - opt = new(ObjectCopyOptions) + copyOpt := &ObjectCopyOptions{ + &ObjectCopyHeaderOptions{}, + &ACLHeaderOptions{}, } - if opt.ObjectCopyHeaderOptions == nil { - opt.ObjectCopyHeaderOptions = new(ObjectCopyHeaderOptions) + if opt != nil { + if opt.ObjectCopyHeaderOptions != nil { + *copyOpt.ObjectCopyHeaderOptions = *opt.ObjectCopyHeaderOptions + } + if opt.ACLHeaderOptions != nil { + *copyOpt.ACLHeaderOptions = *opt.ACLHeaderOptions + } } - opt.XCosCopySource = u + copyOpt.XCosCopySource = u sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: "/" + encodeURIComponent(name), method: http.MethodPut, body: nil, - optHeader: opt, + optHeader: copyOpt, result: &res, } resp, err := s.client.send(ctx, &sendOpt) @@ -315,9 +376,10 @@ func (s *ObjectService) Delete(ctx context.Context, name string, opt ...*ObjectD type ObjectHeadOptions struct { IfModifiedSince string `url:"-" header:"If-Modified-Since,omitempty"` // SSE-C - XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"` - XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` - XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` + XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"` + XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` + XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` + XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` } // Head Object请求可以取回对应Object的元数据，Head的权限与Get的权限一致 @@ -470,7 +532,7 @@ func (s *ObjectService) DeleteMulti(ctx context.Context, opt *ObjectDeleteMultiO type Object struct { Key string `xml:",omitempty"` ETag string `xml:",omitempty"` - Size int `xml:",omitempty"` + Size int64 `xml:",omitempty"` PartNumber int `xml:",omitempty"` LastModified string `xml:",omitempty"` StorageClass string `xml:",omitempty"` @@ -481,15 +543,19 @@ type Object struct { // MultiUploadOptions is the option of the multiupload, // ThreadPoolSize default is one type MultiUploadOptions struct { - OptIni *InitiateMultipartUploadOptions - PartSize int64 - ThreadPoolSize int + OptIni *InitiateMultipartUploadOptions + PartSize int64 + ThreadPoolSize int + CheckPoint bool + EnableVerification bool } type Chunk struct { Number int OffSet int64 Size int64 + Done bool + ETag string } // jobs @@ -506,46 +572,65 @@ type Jobs struct { type Results struct { PartNumber int Resp *Response + err error +} + +func LimitReadCloser(r io.Reader, n int64) io.Reader { + var lc LimitedReadCloser + lc.R = r + lc.N = n + return &lc +} + +type LimitedReadCloser struct { + io.LimitedReader +} + +func (lc *LimitedReadCloser) Close() error { + if r, ok := lc.R.(io.ReadCloser); ok { + return r.Close() + } + return nil } func worker(s *ObjectService, jobs <-chan *Jobs, results chan<- *Results) { for j := range jobs { - fd, err := os.Open(j.FilePath) - var res Results - if err != nil { - res.PartNumber = j.Chunk.Number - res.Resp = nil - results <- &res - } - - fd.Seek(j.Chunk.OffSet, os.SEEK_SET) - // UploadPart do not support the chunk trsf, so need to add the content-length - j.Opt.ContentLength = int(j.Chunk.Size) + j.Opt.ContentLength = j.Chunk.Size rt := j.RetryTimes for { + // http.Request.Body can be Closed in request + fd, err := os.Open(j.FilePath) + var res Results + if err != nil { + res.err = err + res.PartNumber = j.Chunk.Number + res.Resp = nil + results <- &res + break + } + fd.Seek(j.Chunk.OffSet, os.SEEK_SET) resp, err := s.UploadPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number, - &io.LimitedReader{R: fd, N: j.Chunk.Size}, j.Opt) + LimitReadCloser(fd, j.Chunk.Size), j.Opt) res.PartNumber = j.Chunk.Number res.Resp = resp + res.err = err if err != nil { rt-- if rt == 0 { - fd.Close() results <- &res break } continue } - fd.Close() results <- &res break } } } -func DividePart(fileSize int64) (int64, int64) { - partSize := int64(1 * 1024 * 1024) +func DividePart(fileSize int64, last int) (int64, int64) { + partSize := int64(last * 1024 * 1024) partNum := fileSize / partSize for partNum >= 10000 { partSize = partSize * 2 @@ -554,30 +639,30 @@ func DividePart(fileSize int64) (int64, int64) { return partNum, partSize } -func SplitFileIntoChunks(filePath string, partSize int64) ([]Chunk, int, error) { +func SplitFileIntoChunks(filePath string, partSize int64) (int64, []Chunk, int, error) { if filePath == "" { - return nil, 0, errors.New("filePath invalid") + return 0, nil, 0, errors.New("filePath invalid") } file, err := os.Open(filePath) if err != nil { - return nil, 0, err + return 0, nil, 0, err } defer file.Close() stat, err := file.Stat() if err != nil { - return nil, 0, err + return 0, nil, 0, err } var partNum int64 if partSize > 0 { partSize = partSize * 1024 * 1024 partNum = stat.Size() / partSize if partNum >= 10000 { - return nil, 0, errors.New("Too many parts, out of 10000") + return 0, nil, 0, errors.New("Too many parts, out of 10000") } } else { - partNum, partSize = DividePart(stat.Size()) + partNum, partSize = DividePart(stat.Size(), 64) } var chunks []Chunk @@ -597,10 +682,84 @@ func SplitFileIntoChunks(filePath string, partSize int64) ([]Chunk, int, error) partNum++ } - return chunks, int(partNum), nil + return int64(stat.Size()), chunks, int(partNum), nil } +func (s *ObjectService) getResumableUploadID(ctx context.Context, name string) (string, error) { + opt := &ObjectListUploadsOptions{ + Prefix: name, + EncodingType: "url", + } + res, _, err := s.ListUploads(ctx, opt) + if err != nil { + return "", err + } + if len(res.Upload) == 0 { + return "", nil + } + last := len(res.Upload) - 1 + for last >= 0 { + decodeKey, _ := decodeURIComponent(res.Upload[last].Key) + if decodeKey == name { + return decodeURIComponent(res.Upload[last].UploadID) + } + last = last - 1 + } + return "", nil +} + +func (s *ObjectService) checkUploadedParts(ctx context.Context, name, UploadID, filepath string, chunks []Chunk, partNum int) error { + var uploadedParts []Object + isTruncated := true + opt := &ObjectListPartsOptions{ + EncodingType: "url", + } + for isTruncated { + res, _, err := s.ListParts(ctx, name, UploadID, opt) + if err != nil { + return err + } + if len(res.Parts) > 0 { + uploadedParts = append(uploadedParts, res.Parts...) + } + isTruncated = res.IsTruncated + opt.PartNumberMarker = res.NextPartNumberMarker + } + fd, err := os.Open(filepath) + if err != nil { + return err + } + defer fd.Close() + // 某个分块出错, 重置chunks + ret := func(e error) error { + for i, _ := range chunks { + chunks[i].Done = false + chunks[i].ETag = "" + } + return e + } + for _, part := range uploadedParts { + partNumber := part.PartNumber + if partNumber > partNum { + return ret(errors.New("Part Number is not consistent")) + } + partNumber = partNumber - 1 + fd.Seek(chunks[partNumber].OffSet, os.SEEK_SET) + bs, err := ioutil.ReadAll(io.LimitReader(fd, chunks[partNumber].Size)) + if err != nil { + return ret(err) + } + localMD5 := fmt.Sprintf("\"%x\"", md5.Sum(bs)) + if localMD5 != part.ETag { + return ret(errors.New(fmt.Sprintf("CheckSum Failed in Part[%d]", part.PartNumber))) + } + chunks[partNumber].Done = true + chunks[partNumber].ETag = part.ETag + } + return nil +} + // MultiUpload/Upload 为高级upload接口，并发分块上传 // 注意该接口目前只供参考 // @@ -615,12 +774,26 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string if opt == nil { opt = &MultiUploadOptions{} } + var localcrc uint64 // 1.Get the file chunk - chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize) + totalBytes, chunks, partNum, err := SplitFileIntoChunks(filepath, opt.PartSize) if err != nil { return nil, nil, err } - if partNum == 0 { + // 校验 + if s.client.Conf.EnableCRC { + fd, err := os.Open(filepath) + if err != nil { + return nil, nil, err + } + defer fd.Close() + localcrc, err = calCRC64(fd) + if err != nil { + return nil, nil, err + } + } + // filesize=0 , use simple upload + if partNum == 0 || partNum == 1 { var opt0 *ObjectPutOptions if opt.OptIni != nil { opt0 = &ObjectPutOptions{ @@ -636,16 +809,36 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string Key: name, ETag: rsp.Header.Get("ETag"), } + if rsp != nil && s.client.Conf.EnableCRC { + scoscrc := rsp.Header.Get("x-cos-hash-crc64ecma") + icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64) + if icoscrc != localcrc { + return result, rsp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc) + } + } return result, rsp, nil } + var uploadID string + resumableFlag := false + if opt.CheckPoint { + var err error + uploadID, err = s.getResumableUploadID(ctx, name) + if err == nil && uploadID != "" { + err = s.checkUploadedParts(ctx, name, uploadID, filepath, chunks, partNum) + resumableFlag = (err == nil) + } + } + // 2.Init optini := opt.OptIni - res, _, err := s.InitiateMultipartUpload(ctx, name, optini) - if err != nil { - return nil, nil, err + if !resumableFlag { + res, _, err := s.InitiateMultipartUpload(ctx, name, optini) + if err != nil { + return nil, nil, err + } + uploadID = res.UploadID } - uploadID := res.UploadID var poolSize int if opt.ThreadPoolSize > 0 { poolSize = opt.ThreadPoolSize @@ -663,44 +856,97 @@ func (s *ObjectService) Upload(ctx context.Context, name string, filepath string go worker(s, chjobs, chresults) } - // 4.Push jobs - for _, chunk := range chunks { - partOpt := &ObjectUploadPartOptions{} - if optini != nil && optini.ObjectPutHeaderOptions != nil { - partOpt.XCosSSECustomerAglo = optini.XCosSSECustomerAglo - partOpt.XCosSSECustomerKey = optini.XCosSSECustomerKey - partOpt.XCosSSECustomerKeyMD5 = optini.XCosSSECustomerKeyMD5 - } - job := &Jobs{ - Name: name, - RetryTimes: 3, - FilePath: filepath, - UploadId: uploadID, - Chunk: chunk, - Opt: partOpt, - } - chjobs <- job + // progress started event + var listener ProgressListener + var consumedBytes int64 + if opt.OptIni != nil { + listener = opt.OptIni.Listener } - close(chjobs) + event := newProgressEvent(ProgressStartedEvent, 0, 0, totalBytes) + progressCallback(listener, event) + + // 4.Push jobs + go func() { + for _, chunk := range chunks { + if chunk.Done { + continue + } + partOpt := &ObjectUploadPartOptions{} + if optini != nil && optini.ObjectPutHeaderOptions != nil { + partOpt.XCosSSECustomerAglo = optini.XCosSSECustomerAglo + partOpt.XCosSSECustomerKey = optini.XCosSSECustomerKey + partOpt.XCosSSECustomerKeyMD5 = optini.XCosSSECustomerKeyMD5 + partOpt.XCosTrafficLimit = optini.XCosTrafficLimit + } + job := &Jobs{ + Name: name, + RetryTimes: 3, + FilePath: filepath, + UploadId: uploadID, + Chunk: chunk, + Opt: partOpt, + } + chjobs <- job + } + close(chjobs) + }() // 5.Recv the resp etag to complete - for i := 1; i <= partNum; i++ { + err = nil + for i := 0; i < partNum; i++ { + if chunks[i].Done { + optcom.Parts = append(optcom.Parts, Object{ + PartNumber: chunks[i].Number, ETag: chunks[i].ETag}, + ) + if err == nil { + consumedBytes += chunks[i].Size + event = newProgressEvent(ProgressDataEvent, chunks[i].Size, consumedBytes, totalBytes) + progressCallback(listener, event) + } + continue + } res := <-chresults // Notice one part fail can not get the etag according. - if res.Resp == nil { + if res.Resp == nil || res.err != nil { // Some part already fail, can not to get the header inside. - return nil, nil, fmt.Errorf("UploadID %s, part %d failed to get resp content.", uploadID, res.PartNumber) + err = fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error()) + continue } // Notice one part fail can not get the etag according. etag := res.Resp.Header.Get("ETag") optcom.Parts = append(optcom.Parts, Object{ PartNumber: res.PartNumber, ETag: etag}, ) + if err == nil { + consumedBytes += chunks[res.PartNumber-1].Size + event = newProgressEvent(ProgressDataEvent, chunks[res.PartNumber-1].Size, consumedBytes, totalBytes) + progressCallback(listener, event) + } + } + close(chresults) + if err != nil { + event = newProgressEvent(ProgressFailedEvent, 0, consumedBytes, totalBytes, err) + progressCallback(listener, event) + return nil, nil, err } sort.Sort(ObjectList(optcom.Parts)) - v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom) + event = newProgressEvent(ProgressCompletedEvent, 0, consumedBytes, totalBytes) + progressCallback(listener, event) + v, resp, err := s.CompleteMultipartUpload(context.Background(), name, uploadID, optcom) + if err != nil { + s.AbortMultipartUpload(ctx, name, uploadID) + return v, resp, err + } + + if resp != nil && s.client.Conf.EnableCRC { + scoscrc := resp.Header.Get("x-cos-hash-crc64ecma") + icoscrc, _ := strconv.ParseUint(scoscrc, 10, 64) + if icoscrc != localcrc { + return v, resp, fmt.Errorf("verification failed, want:%v, return:%v", localcrc, icoscrc) + } + } return v, resp, err } diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go index 897afcae31..2a6e5bf8f9 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go @@ -5,8 +5,13 @@ import ( "encoding/xml" "errors" "fmt" + "hash/crc64" "io" "net/http" + "net/url" + "sort" + "strings" + "time" ) // InitiateMultipartUploadOptions is the option of InitateMultipartUpload @@ -41,13 +46,19 @@ func (s *ObjectService) InitiateMultipartUpload(ctx context.Context, name string // ObjectUploadPartOptions is the options of upload-part type ObjectUploadPartOptions struct { - Expect string `header:"Expect,omitempty" url:"-"` - XCosContentSHA1 string `header:"x-cos-content-sha1" url:"-"` - ContentLength int `header:"Content-Length,omitempty" url:"-"` - + Expect string `header:"Expect,omitempty" url:"-"` + XCosContentSHA1 string `header:"x-cos-content-sha1,omitempty" url:"-"` + ContentLength int64 `header:"Content-Length,omitempty" url:"-"` + ContentMD5 string `header:"Content-MD5,omitempty" url:"-"` XCosSSECustomerAglo string `header:"x-cos-server-side-encryption-customer-algorithm,omitempty" url:"-" xml:"-"` XCosSSECustomerKey string `header:"x-cos-server-side-encryption-customer-key,omitempty" url:"-" xml:"-"` XCosSSECustomerKeyMD5 string `header:"x-cos-server-side-encryption-customer-key-MD5,omitempty" url:"-" xml:"-"` + + XCosTrafficLimit int `header:"x-cos-traffic-limit,omitempty" url:"-" xml:"-"` + + XOptionHeader *http.Header `header:"-,omitempty" url:"-" xml:"-"` + // 上传进度, ProgressCompleteEvent不能表示对应API调用成功，API是否调用成功的判断标准为返回err==nil + Listener ProgressListener `header:"-" url:"-" xml:"-"` } // UploadPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。 @@ -58,14 +69,37 @@ type ObjectUploadPartOptions struct { // 当 r 不是 bytes.Buffer/bytes.Reader/strings.Reader 时，必须指定 opt.ContentLength // // https://www.qcloud.com/document/product/436/7750 -func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, opt *ObjectUploadPartOptions) (*Response, error) { +func (s *ObjectService) UploadPart(ctx context.Context, name, uploadID string, partNumber int, r io.Reader, uopt *ObjectUploadPartOptions) (*Response, error) { + if err := CheckReaderLen(r); err != nil { + return nil, err + } + // opt 不为 nil + opt := cloneObjectUploadPartOptions(uopt) + totalBytes, err := GetReaderLen(r) + if err != nil && opt.Listener != nil { + return nil, err + } + // 分块上传不支持 Chunk 上传 + if err == nil { + // 与 go http 保持一致, 非bytes.Buffer/bytes.Reader/strings.Reader需用户指定ContentLength + if opt != nil && opt.ContentLength == 0 && IsLenReader(r) { + opt.ContentLength = totalBytes + } + } + reader := TeeReader(r, nil, totalBytes, nil) + if s.client.Conf.EnableCRC { + reader.writer = crc64.New(crc64.MakeTable(crc64.ECMA)) + } + if opt != nil && opt.Listener != nil { + reader.listener = opt.Listener + } u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID) sendOpt := sendOptions{ baseURL: s.client.BaseURL.BucketURL, uri: u, method: http.MethodPut, optHeader: opt, - body: r, + body: reader, } resp, err := s.client.send(ctx, &sendOpt) return resp, err @@ -244,3 +278,253 @@ func (s *ObjectService) CopyPart(ctx context.Context, name, uploadID string, par } return &res, resp, err } + +type ObjectListUploadsOptions struct { + Delimiter string `url:"Delimiter,omitempty"` + EncodingType string `url:"EncodingType,omitempty"` + Prefix string `url:"Prefix"` + MaxUploads int `url:"MaxUploads"` + KeyMarker string `url:"KeyMarker"` + UploadIdMarker string `url:"UploadIDMarker"` +} + +type ObjectListUploadsResult struct { + XMLName xml.Name `xml:"ListMultipartUploadsResult"` + Bucket string `xml:"Bucket,omitempty"` + EncodingType string `xml:"Encoding-Type,omitempty"` + KeyMarker string `xml:"KeyMarker,omitempty"` + UploadIdMarker string `xml:"UploadIdMarker,omitempty"` + NextKeyMarker string `xml:"NextKeyMarker,omitempty"` + NextUploadIdMarker string `xml:"NextUploadIdMarker,omitempty"` + MaxUploads string `xml:"MaxUploads,omitempty"` + IsTruncated bool `xml:"IsTruncated,omitempty"` + Prefix string `xml:"Prefix,omitempty"` + Delimiter string `xml:"Delimiter,omitempty"` + Upload []ListUploadsResultUpload `xml:"Upload,omitempty"` + CommonPrefixes []string `xml:"CommonPrefixes>Prefix,omitempty"` +} + +type ListUploadsResultUpload struct { + Key string `xml:"Key,omitempty"` + UploadID string `xml:"UploadId,omitempty"` + StorageClass string `xml:"StorageClass,omitempty"` + Initiator *Initiator `xml:"Initiator,omitempty"` + Owner *Owner `xml:"Owner,omitempty"` + Initiated string `xml:"Initiated,omitempty"` +} + +func (s *ObjectService) ListUploads(ctx context.Context, opt *ObjectListUploadsOptions) (*ObjectListUploadsResult, *Response, error) { + var res ObjectListUploadsResult + sendOpt := &sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: "/?uploads", + method: http.MethodGet, + optQuery: opt, + result: &res, + } + resp, err := s.client.send(ctx, sendOpt) + return &res, resp, err +} + +type MultiCopyOptions struct { + OptCopy *ObjectCopyOptions + PartSize int64 + ThreadPoolSize int +} + +type CopyJobs struct { + Name string + UploadId string + RetryTimes int + Chunk Chunk + Opt *ObjectCopyPartOptions +} + +type CopyResults struct { + PartNumber int + Resp *Response + err error + res *CopyPartResult +} + +func copyworker(s *ObjectService, jobs <-chan *CopyJobs, results chan<- *CopyResults) { + for j := range jobs { + var copyres CopyResults + j.Opt.XCosCopySourceRange = fmt.Sprintf("bytes=%d-%d", j.Chunk.OffSet, j.Chunk.OffSet+j.Chunk.Size-1) + rt := j.RetryTimes + for { + res, resp, err := s.CopyPart(context.Background(), j.Name, j.UploadId, j.Chunk.Number, j.Opt.XCosCopySource, j.Opt) + copyres.PartNumber = j.Chunk.Number + copyres.Resp = resp + copyres.err = err + copyres.res = res + if err != nil { + rt-- + if rt == 0 { + results <- ©res + break + } + time.Sleep(10 * time.Millisecond) + continue + } + results <- ©res + break + } + } +} + +func (s *ObjectService) innerHead(ctx context.Context, sourceURL string, opt *ObjectHeadOptions, id []string) (resp *Response, err error) { + surl := strings.SplitN(sourceURL, "/", 2) + if len(surl) < 2 { + err = errors.New(fmt.Sprintf("sourceURL format error: %s", sourceURL)) + return + } + + u, err := url.Parse(fmt.Sprintf("https://%s", surl[0])) + if err != nil { + return + } + b := &BaseURL{BucketURL: u} + client := NewClient(b, &http.Client{ + Transport: s.client.client.Transport, + }) + if len(id) > 0 { + resp, err = client.Object.Head(ctx, surl[1], nil, id[0]) + } else { + resp, err = client.Object.Head(ctx, surl[1], nil) + } + return +} + +func SplitCopyFileIntoChunks(totalBytes int64, partSize int64) ([]Chunk, int, error) { + var partNum int64 + if partSize > 0 { + partSize = partSize * 1024 * 1024 + partNum = totalBytes / partSize + if partNum >= 10000 { + return nil, 0, errors.New("Too many parts, out of 10000") + } + } else { + partNum, partSize = DividePart(totalBytes, 128) + } + + var chunks []Chunk + var chunk = Chunk{} + for i := int64(0); i < partNum; i++ { + chunk.Number = int(i + 1) + chunk.OffSet = i * partSize + chunk.Size = partSize + chunks = append(chunks, chunk) + } + + if totalBytes%partSize > 0 { + chunk.Number = len(chunks) + 1 + chunk.OffSet = int64(len(chunks)) * partSize + chunk.Size = totalBytes % partSize + chunks = append(chunks, chunk) + partNum++ + } + return chunks, int(partNum), nil +} + +func (s *ObjectService) MultiCopy(ctx context.Context, name string, sourceURL string, opt *MultiCopyOptions, id ...string) (*ObjectCopyResult, *Response, error) { + resp, err := s.innerHead(ctx, sourceURL, nil, id) + if err != nil { + return nil, nil, err + } + totalBytes := resp.ContentLength + surl := strings.SplitN(sourceURL, "/", 2) + if len(surl) < 2 { + return nil, nil, errors.New(fmt.Sprintf("x-cos-copy-source format error: %s", sourceURL)) + } + var u string + if len(id) == 1 { + u = fmt.Sprintf("%s/%s?versionId=%s", surl[0], encodeURIComponent(surl[1]), id[0]) + } else if len(id) == 0 { + u = fmt.Sprintf("%s/%s", surl[0], encodeURIComponent(surl[1])) + } else { + return nil, nil, errors.New("wrong params") + } + + if opt == nil { + opt = &MultiCopyOptions{} + } + chunks, partNum, err := SplitCopyFileIntoChunks(totalBytes, opt.PartSize) + if err != nil { + return nil, nil, err + } + if partNum == 0 || totalBytes < singleUploadMaxLength { + if len(id) > 0 { + return s.Copy(ctx, name, sourceURL, opt.OptCopy, id[0]) + } else { + return s.Copy(ctx, name, sourceURL, opt.OptCopy) + } + } + optini := CopyOptionsToMulti(opt.OptCopy) + var uploadID string + res, _, err := s.InitiateMultipartUpload(ctx, name, optini) + if err != nil { + return nil, nil, err + } + uploadID = res.UploadID + + var poolSize int + if opt.ThreadPoolSize > 0 { + poolSize = opt.ThreadPoolSize + } else { + poolSize = 1 + } + + chjobs := make(chan *CopyJobs, 100) + chresults := make(chan *CopyResults, 10000) + optcom := &CompleteMultipartUploadOptions{} + + for w := 1; w <= poolSize; w++ { + go copyworker(s, chjobs, chresults) + } + + go func() { + for _, chunk := range chunks { + partOpt := &ObjectCopyPartOptions{ + XCosCopySource: u, + } + job := &CopyJobs{ + Name: name, + RetryTimes: 3, + UploadId: uploadID, + Chunk: chunk, + Opt: partOpt, + } + chjobs <- job + } + close(chjobs) + }() + err = nil + for i := 0; i < partNum; i++ { + res := <-chresults + if res.res == nil || res.err != nil { + err = fmt.Errorf("UploadID %s, part %d failed to get resp content. error: %s", uploadID, res.PartNumber, res.err.Error()) + continue + } + etag := res.res.ETag + optcom.Parts = append(optcom.Parts, Object{ + PartNumber: res.PartNumber, ETag: etag}, + ) + } + close(chresults) + if err != nil { + return nil, nil, err + } + sort.Sort(ObjectList(optcom.Parts)) + + v, resp, err := s.CompleteMultipartUpload(ctx, name, uploadID, optcom) + if err != nil { + s.AbortMultipartUpload(ctx, name, uploadID) + } + cpres := &ObjectCopyResult{ + ETag: v.ETag, + CRC64: resp.Header.Get("x-cos-hash-crc64ecma"), + VersionId: resp.Header.Get("x-cos-version-id"), + } + return cpres, resp, err +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_select.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_select.go new file mode 100644 index 0000000000..a3c3c6dd9f --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_select.go @@ -0,0 +1,444 @@ +package cos + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/xml" + "fmt" + "hash/crc32" + "io" + "io/ioutil" + "net/http" + "os" + "time" +) + +type JSONInputSerialization struct { + Type string `xml:"Type,omitempty"` +} + +type CSVInputSerialization struct { + RecordDelimiter string `xml:"RecordDelimiter,omitempty"` + FieldDelimiter string `xml:"FieldDelimiter,omitempty"` + QuoteCharacter string `xml:"QuoteCharacter,omitempty"` + QuoteEscapeCharacter string `xml:"QuoteEscapeCharacter,omitempty"` + AllowQuotedRecordDelimiter string `xml:"AllowQuotedRecordDelimiter,omitempty"` + FileHeaderInfo string `xml:"FileHeaderInfo,omitempty"` + Comments string `xml:"Comments,omitempty"` +} + +type SelectInputSerialization struct { + CompressionType string `xml:"CompressionType,omitempty"` + CSV *CSVInputSerialization `xml:"CSV,omitempty"` + JSON *JSONInputSerialization `xml:"JSON,omitempty"` +} + +type JSONOutputSerialization struct { + RecordDelimiter string `xml:"RecordDelimiter,omitempty"` +} + +type CSVOutputSerialization struct { + QuoteFields string `xml:"QuoteFields,omitempty"` + RecordDelimiter string `xml:"RecordDelimiter,omitempty"` + FieldDelimiter string `xml:"FieldDelimiter,omitempty"` + QuoteCharacter string `xml:"QuoteCharacter,omitempty"` + QuoteEscapeCharacter string `xml:"QuoteEscapeCharacter,omitempty"` +} + +type SelectOutputSerialization struct { + CSV *CSVOutputSerialization `xml:"CSV,omitempty"` + JSON *JSONOutputSerialization `xml:"JSON,omitempty"` +} + +type ObjectSelectOptions struct { + XMLName xml.Name `xml:"SelectRequest"` + Expression string `xml:"Expression"` + ExpressionType string `xml:"ExpressionType"` + InputSerialization *SelectInputSerialization `xml:"InputSerialization"` + OutputSerialization *SelectOutputSerialization `xml:"OutputSerialization"` + RequestProgress string `xml:"RequestProgress>Enabled,omitempty"` +} + +func (s *ObjectService) Select(ctx context.Context, name string, opt *ObjectSelectOptions) (io.ReadCloser, error) { + u := fmt.Sprintf("/%s?select&select-type=2", encodeURIComponent(name)) + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodPost, + body: opt, + disableCloseBody: true, + } + resp, err := s.client.send(ctx, &sendOpt) + if err != nil { + return nil, err + } + result := &ObjectSelectResponse{ + Headers: resp.Header, + Body: resp.Body, + StatusCode: resp.StatusCode, + Frame: &ObjectSelectResult{ + NextFrame: true, + Payload: []byte{}, + }, + Finish: false, + } + + return result, nil +} + +func (s *ObjectService) SelectToFile(ctx context.Context, name, file string, opt *ObjectSelectOptions) (*ObjectSelectResponse, error) { + resp, err := s.Select(ctx, name, opt) + if err != nil { + return nil, err + } + res, _ := resp.(*ObjectSelectResponse) + defer func() { + io.Copy(ioutil.Discard, resp) + resp.Close() + }() + + fd, err := os.OpenFile(file, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.FileMode(0664)) + if err != nil { + return res, err + } + + _, err = io.Copy(fd, resp) + fd.Close() + res.Finish = true + return res, err +} + +const ( + kReadTimeout = 3 + kMessageType = ":message-type" + kEventType = ":event-type" + kContentType = ":content-type" + + kRecordsFrameType = iota + kContinuationFrameType + kProgressFrameType + kStatsFrameType + kEndFrameType + kErrorFrameType +) + +type ProgressFrame struct { + XMLName xml.Name `xml:"Progress"` + BytesScanned int `xml:"BytesScanned"` + BytesProcessed int `xml:"BytesProcessed"` + BytesReturned int `xml:"BytesReturned"` +} + +type StatsFrame struct { + XMLName xml.Name `xml:"Stats"` + BytesScanned int `xml:"BytesScanned"` + BytesProcessed int `xml:"BytesProcessed"` + BytesReturned int `xml:"BytesReturned"` +} + +type DataFrame struct { + ContentType string + ConsumedBytesLength int32 + LeftBytesLength int32 +} + +type ErrorFrame struct { + Code string + Message string +} + +func (e *ErrorFrame) Error() string { + return fmt.Sprintf("Error Code: %s, Error Message: %s", e.Code, e.Message) +} + +type ObjectSelectResult struct { + TotalFrameLength int32 + TotalHeaderLength int32 + NextFrame bool + FrameType int + Payload []byte + DataFrame DataFrame + ProgressFrame ProgressFrame + StatsFrame StatsFrame + ErrorFrame *ErrorFrame +} + +type ObjectSelectResponse struct { + StatusCode int + Headers http.Header + Body io.ReadCloser + Frame *ObjectSelectResult + Finish bool +} + +func (osr *ObjectSelectResponse) Read(p []byte) (n int, err error) { + n, err = osr.readFrames(p) + return +} +func (osr *ObjectSelectResponse) Close() error { + return osr.Body.Close() +} + +func (osr *ObjectSelectResponse) readFrames(p []byte) (int, error) { + if osr.Finish { + return 0, io.EOF + } + if osr.Frame.ErrorFrame != nil { + return 0, osr.Frame.ErrorFrame + } + + var err error + var nlen int + dlen := len(p) + + for nlen < dlen { + if osr.Frame.NextFrame == true { + osr.Frame.NextFrame = false + err := osr.analysisPrelude() + if err != nil { + return nlen, err + } + err = osr.analysisHeader() + if err != nil { + return nlen, err + } + } + switch osr.Frame.FrameType { + case kRecordsFrameType: + n, err := osr.analysisRecords(p[nlen:]) + if err != nil { + return nlen, err + } + nlen += n + case kContinuationFrameType: + err = osr.payloadChecksum("ContinuationFrame") + if err != nil { + return nlen, err + } + case kProgressFrameType: + err := osr.analysisXml(&osr.Frame.ProgressFrame) + if err != nil { + return nlen, err + } + case kStatsFrameType: + err := osr.analysisXml(&osr.Frame.StatsFrame) + if err != nil { + return nlen, err + } + case kEndFrameType: + err = osr.payloadChecksum("EndFrame") + if err != nil { + return nlen, err + } + osr.Finish = true + return nlen, io.EOF + case kErrorFrameType: + return nlen, osr.Frame.ErrorFrame + } + } + return nlen, err +} + +func (osr *ObjectSelectResponse) analysisPrelude() error { + frame := make([]byte, 12) + _, err := osr.fixedLengthRead(frame, kReadTimeout) + if err != nil { + return err + } + + var preludeCRC uint32 + bytesToInt(frame[0:4], &osr.Frame.TotalFrameLength) + bytesToInt(frame[4:8], &osr.Frame.TotalHeaderLength) + bytesToInt(frame[8:12], &preludeCRC) + osr.Frame.Payload = append(osr.Frame.Payload, frame...) + + return checksum(frame[0:8], preludeCRC, "Prelude") +} + +func (osr *ObjectSelectResponse) analysisHeader() error { + var nlen int32 + headers := make(map[string]string) + for nlen < osr.Frame.TotalHeaderLength { + var headerNameLen int8 + var headerValueLen int16 + bHeaderNameLen := make([]byte, 1) + _, err := osr.fixedLengthRead(bHeaderNameLen, kReadTimeout) + if err != nil { + return err + } + nlen += 1 + bytesToInt(bHeaderNameLen, &headerNameLen) + osr.Frame.Payload = append(osr.Frame.Payload, bHeaderNameLen...) + + bHeaderName := make([]byte, headerNameLen) + _, err = osr.fixedLengthRead(bHeaderName, kReadTimeout) + if err != nil { + return err + } + nlen += int32(headerNameLen) + headerName := string(bHeaderName) + osr.Frame.Payload = append(osr.Frame.Payload, bHeaderName...) + + bValueTypeLen := make([]byte, 3) + _, err = osr.fixedLengthRead(bValueTypeLen, kReadTimeout) + if err != nil { + return err + } + nlen += 3 + bytesToInt(bValueTypeLen[1:], &headerValueLen) + osr.Frame.Payload = append(osr.Frame.Payload, bValueTypeLen...) + + bHeaderValue := make([]byte, headerValueLen) + _, err = osr.fixedLengthRead(bHeaderValue, kReadTimeout) + if err != nil { + return err + } + nlen += int32(headerValueLen) + headers[headerName] = string(bHeaderValue) + osr.Frame.Payload = append(osr.Frame.Payload, bHeaderValue...) + } + htype, ok := headers[kMessageType] + if !ok { + return fmt.Errorf("header parse failed, no message-type, headers: %+v\n", headers) + } + switch { + case htype == "error": + osr.Frame.FrameType = kErrorFrameType + osr.Frame.ErrorFrame = &ErrorFrame{} + osr.Frame.ErrorFrame.Code, _ = headers[":error-code"] + osr.Frame.ErrorFrame.Message, _ = headers[":error-message"] + case htype == "event": + hevent, ok := headers[kEventType] + if !ok { + return fmt.Errorf("header parse failed, no event-type, headers: %+v\n", headers) + } + switch { + case hevent == "Records": + hContentType, ok := headers[kContentType] + if ok { + osr.Frame.DataFrame.ContentType = hContentType + } + osr.Frame.FrameType = kRecordsFrameType + case hevent == "Cont": + osr.Frame.FrameType = kContinuationFrameType + case hevent == "Progress": + osr.Frame.FrameType = kProgressFrameType + case hevent == "Stats": + osr.Frame.FrameType = kStatsFrameType + case hevent == "End": + osr.Frame.FrameType = kEndFrameType + default: + return fmt.Errorf("header parse failed, invalid event-type, headers: %+v\n", headers) + } + default: + return fmt.Errorf("header parse failed, invalid message-type: headers: %+v\n", headers) + } + return nil +} + +func (osr *ObjectSelectResponse) analysisRecords(data []byte) (int, error) { + var needReadLength int32 + dlen := int32(len(data)) + restLen := osr.Frame.TotalFrameLength - 16 - osr.Frame.TotalHeaderLength - osr.Frame.DataFrame.ConsumedBytesLength + if dlen <= restLen { + needReadLength = dlen + } else { + needReadLength = restLen + } + n, err := osr.fixedLengthRead(data[:needReadLength], kReadTimeout) + if err != nil { + return n, fmt.Errorf("read data frame error: %s", err.Error()) + } + osr.Frame.DataFrame.ConsumedBytesLength += int32(n) + osr.Frame.Payload = append(osr.Frame.Payload, data[:needReadLength]...) + // 读完了一帧数据并填充到data中了 + if osr.Frame.DataFrame.ConsumedBytesLength == osr.Frame.TotalFrameLength-16-osr.Frame.TotalHeaderLength { + osr.Frame.DataFrame.ConsumedBytesLength = 0 + err = osr.payloadChecksum("RecordFrame") + } + return n, err +} + +func (osr *ObjectSelectResponse) analysisXml(frame interface{}) error { + payloadLength := osr.Frame.TotalFrameLength - 16 - osr.Frame.TotalHeaderLength + bFrame := make([]byte, payloadLength) + _, err := osr.fixedLengthRead(bFrame, kReadTimeout) + if err != nil { + return err + } + err = xml.Unmarshal(bFrame, frame) + if err != nil { + return err + } + osr.Frame.Payload = append(osr.Frame.Payload, bFrame...) + return osr.payloadChecksum("XmlFrame") +} + +// 调用payloadChecksum时，表示该帧已读完，开始读取下一帧内容 +func (osr *ObjectSelectResponse) payloadChecksum(ftype string) error { + bcrc := make([]byte, 4) + _, err := osr.fixedLengthRead(bcrc, kReadTimeout) + if err != nil { + return err + } + var res uint32 + bytesToInt(bcrc, &res) + err = checksum(osr.Frame.Payload, res, ftype) + + osr.Frame.NextFrame = true + osr.Frame.Payload = []byte{} + + return err +} + +type chanReadIO struct { + readLen int + err error +} + +func (osr *ObjectSelectResponse) fixedLengthRead(p []byte, read_timeout int64) (int, error) { + timeout := time.Duration(read_timeout) + r := osr.Body + ch := make(chan chanReadIO, 1) + defer close(ch) + go func(p []byte) { + var needLen int + readChan := chanReadIO{} + needLen = len(p) + for { + n, err := r.Read(p[readChan.readLen:needLen]) + readChan.readLen += n + if err != nil { + readChan.err = err + ch <- readChan + return + } + + if readChan.readLen == needLen { + break + } + } + ch <- readChan + }(p) + + select { + case <-time.After(time.Second * timeout): + return 0, fmt.Errorf("requestId: %s, readLen timeout, timeout is %d(second),need read:%d", "sr.Headers.Get(HTTPHeaderOssRequestID)", timeout, len(p)) + case result := <-ch: + return result.readLen, result.err + } +} + +func bytesToInt(b []byte, ret interface{}) { + binBuf := bytes.NewBuffer(b) + binary.Read(binBuf, binary.BigEndian, ret) +} + +func checksum(b []byte, rec uint32, ftype string) error { + c := crc32.ChecksumIEEE(b) + if c != rec { + return fmt.Errorf("parse type: %v, checksum failed, cal: %v, rec: %v\n", ftype, c, rec) + } + return nil +} diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/progress.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/progress.go new file mode 100644 index 0000000000..e60b95a972 --- /dev/null +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/progress.go @@ -0,0 +1,149 @@ +package cos + +import ( + "fmt" + "hash" + "io" +) + +type ProgressEventType int + +const ( + // 数据开始传输 + ProgressStartedEvent ProgressEventType = iota + // 数据传输中 + ProgressDataEvent + // 数据传输完成, 但不能表示对应API调用完成 + ProgressCompletedEvent + // 只有在数据传输时发生错误才会返回 + ProgressFailedEvent +) + +type ProgressEvent struct { + EventType ProgressEventType + RWBytes int64 + ConsumedBytes int64 + TotalBytes int64 + Err error +} + +func newProgressEvent(eventType ProgressEventType, rwBytes, consumed, total int64, err ...error) *ProgressEvent { + event := &ProgressEvent{ + EventType: eventType, + RWBytes: rwBytes, + ConsumedBytes: consumed, + TotalBytes: total, + } + if len(err) > 0 { + event.Err = err[0] + } + return event +} + +// 用户自定义Listener需要实现该方法 +type ProgressListener interface { + ProgressChangedCallback(event *ProgressEvent) +} + +func progressCallback(listener ProgressListener, event *ProgressEvent) { + if listener != nil && event != nil { + listener.ProgressChangedCallback(event) + } +} + +type teeReader struct { + reader io.Reader + writer io.Writer + consumedBytes int64 + totalBytes int64 + listener ProgressListener +} + +func (r *teeReader) Read(p []byte) (int, error) { + if r.consumedBytes == 0 { + event := newProgressEvent(ProgressStartedEvent, 0, r.consumedBytes, r.totalBytes) + progressCallback(r.listener, event) + } + + n, err := r.reader.Read(p) + if err != nil && err != io.EOF { + event := newProgressEvent(ProgressFailedEvent, 0, r.consumedBytes, r.totalBytes, err) + progressCallback(r.listener, event) + } + if n > 0 { + r.consumedBytes += int64(n) + if r.writer != nil { + if n, err := r.writer.Write(p[:n]); err != nil { + return n, err + } + } + if r.listener != nil { + event := newProgressEvent(ProgressDataEvent, int64(n), r.consumedBytes, r.totalBytes) + progressCallback(r.listener, event) + } + } + + if err == io.EOF { + event := newProgressEvent(ProgressCompletedEvent, int64(n), r.consumedBytes, r.totalBytes) + progressCallback(r.listener, event) + } + + return n, err +} + +func (r *teeReader) Close() error { + if rc, ok := r.reader.(io.ReadCloser); ok { + return rc.Close() + } + return nil +} + +func (r *teeReader) Size() int64 { + return r.totalBytes +} + +func (r *teeReader) Crc64() uint64 { + if r.writer != nil { + if th, ok := r.writer.(hash.Hash64); ok { + return th.Sum64() + } + } + return 0 +} + +func TeeReader(reader io.Reader, writer io.Writer, total int64, listener ProgressListener) *teeReader { + return &teeReader{ + reader: reader, + writer: writer, + consumedBytes: 0, + totalBytes: total, + listener: listener, + } +} + +type FixedLengthReader interface { + io.Reader + Size() int64 +} + +type DefaultProgressListener struct { +} + +func (l *DefaultProgressListener) ProgressChangedCallback(event *ProgressEvent) { + switch event.EventType { + case ProgressStartedEvent: + fmt.Printf("Transfer Start [ConsumedBytes/TotalBytes: %d/%d]\n", + event.ConsumedBytes, event.TotalBytes) + case ProgressDataEvent: + fmt.Printf("\rTransfer Data [ConsumedBytes/TotalBytes: %d/%d, %d%%]", + event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes) + case ProgressCompletedEvent: + fmt.Printf("\nTransfer Complete [ConsumedBytes/TotalBytes: %d/%d]\n", + event.ConsumedBytes, event.TotalBytes) + case ProgressFailedEvent: + fmt.Printf("\nTransfer Failed [ConsumedBytes/TotalBytes: %d/%d] [Err: %v]\n", + event.ConsumedBytes, event.TotalBytes, event.Err) + default: + fmt.Printf("Progress Changed Error: unknown progress event type\n") + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 29f2bf4820..db33d4fde9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -610,7 +610,7 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/http github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/vpc/v20170312 -# github.com/tencentyun/cos-go-sdk-v5 v0.7.10 +# github.com/tencentyun/cos-go-sdk-v5 v0.7.24 github.com/tencentyun/cos-go-sdk-v5 github.com/tencentyun/cos-go-sdk-v5/debug # github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e @@ -1130,7 +1130,7 @@ k8s.io/utils/integer sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml -# yunion.io/x/executor v0.0.0-20201231064744-df32f32165a9 +# yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e yunion.io/x/executor/apis yunion.io/x/executor/client yunion.io/x/executor/server @@ -1177,7 +1177,7 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20210204030051-4fb14f92a88f +# yunion.io/x/sqlchemy v0.0.0-20210329171939-2c520f93aee3 yunion.io/x/sqlchemy # yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce yunion.io/x/structarg diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index 800b0d5f38..f99d300247 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -575,7 +575,17 @@ func (c *STextColumn) IsSupportDefault() bool { } func (c *STextColumn) ColType() string { - return fmt.Sprintf("%s CHARACTER SET '%s'", c.SBaseWidthColumn.ColType(), c.Charset) + var charset string + var collate string + switch c.Charset { + case "ascii": + charset = "ascii" + collate = "ascii_general_ci" + default: + charset = "utf8mb4" + collate = "utf8mb4_unicode_ci" + } + return fmt.Sprintf("%s CHARACTER SET '%s' COLLATE '%s'", c.SBaseWidthColumn.ColType(), charset, collate) } func (c *STextColumn) IsText() bool { diff --git a/vendor/yunion.io/x/sqlchemy/field_update.go b/vendor/yunion.io/x/sqlchemy/field_update.go index 3697ad7bfb..342f84a53d 100644 --- a/vendor/yunion.io/x/sqlchemy/field_update.go +++ b/vendor/yunion.io/x/sqlchemy/field_update.go @@ -146,7 +146,7 @@ func (ts *STableSpec) updateFields(dt interface{}, fields map[string]interface{} if err != nil { return err } - if aCnt != 1 { + if aCnt > 1 { return errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) } return nil diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index c243bc5054..ced9278cfa 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -179,7 +179,7 @@ func (t *STableSpec) insert(data interface{}, update bool, debug bool) error { // for insertOrUpdate cases, if no duplication, targetCnt=1, else targetCnt=2 targetCnt = 2 } - if affectCnt < 1 || affectCnt > targetCnt { + if (!update && affectCnt < 1) || affectCnt > targetCnt { return errors.Wrapf(ErrUnexpectRowCount, "Insert affected cnt %d != (1, %d)", affectCnt, targetCnt) } diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 790a4ce087..cf397c98e0 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -75,8 +75,10 @@ func (info *SSqlColumnInfo) toColumnSpec() IColumnSpec { charset := "" if info.Collation == "ascii_general_ci" { charset = "ascii" - } else if info.Collation == "utf8_general_ci" { + } else if info.Collation == "utf8_general_ci" || info.Collation == "utf8mb4_unicode_ci" { charset = "utf8" + } else { + charset = "ascii" } if len(charset) > 0 { tagmap[TAG_CHARSET] = charset diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 19292a78f3..193199f7d7 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -150,7 +150,7 @@ func (ts *STableSpec) CreateSQL() string { if len(indexes) > 0 { cols = append(cols, indexes...) } - return fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE=InnoDB DEFAULT CHARSET=utf8%s", ts.name, strings.Join(cols, ",\n"), autoInc) + return fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE=InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci%s", ts.name, strings.Join(cols, ",\n"), autoInc) } func NewTableInstance(ts ITableSpec) *STable { diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index a606969757..cd3f9225d0 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -16,7 +16,6 @@ package sqlchemy import ( "bytes" - "database/sql" "fmt" "reflect" "strings" @@ -193,12 +192,8 @@ func (us *SUpdateSession) saveUpdate(dt interface{}) (UpdateDiffs, error) { if err != nil { return nil, err } - if aCnt != 1 { - if aCnt == 0 { - return nil, sql.ErrNoRows - } else { - return nil, errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) - } + if aCnt > 1 { + return nil, errors.Wrapf(ErrUnexpectRowCount, "affected rows %d != 1", aCnt) } q := us.tableSpec.Query() for k, v := range primaries {