diff --git a/pkg/compute/regiondrivers/aws.go b/pkg/compute/regiondrivers/aws.go index 42b36b6512..e7c6ab86f6 100644 --- a/pkg/compute/regiondrivers/aws.go +++ b/pkg/compute/regiondrivers/aws.go @@ -103,7 +103,7 @@ func validateAwsLbNetwork(ownerId mcclient.IIdentityProvider, data *jsonutils.JS var nets []models.SNetwork wires := models.WireManager.Query().SubQuery() q := models.NetworkManager.Query().IsFalse("pending_deleted") - q = models.NetworkManager.FilterByOwner(q, network.GetOwnerId(), rbacutils.ScopeProject) + q = models.NetworkManager.FilterByOwner(q, ownerId, rbacutils.ScopeProject) q = q.Join(wires, sqlchemy.Equals(q.Field("wire_id"), wires.Field("id"))) q = q.Filter(sqlchemy.Equals(wires.Field("vpc_id"), vpc.GetId())) q = q.Filter(sqlchemy.NotEquals(wires.Field("zone_id"), zone.GetId())) diff --git a/pkg/multicloud/huawei/client/client.go b/pkg/multicloud/huawei/client/client.go index 08c005cbce..129c701a3d 100644 --- a/pkg/multicloud/huawei/client/client.go +++ b/pkg/multicloud/huawei/client/client.go @@ -35,6 +35,7 @@ type Client struct { Balances *modules.SBalanceManager Bandwidths *modules.SBandwidthManager + Credentials *modules.SCredentialManager Disks *modules.SDiskManager Domains *modules.SDomainManager Eips *modules.SEipManager @@ -84,6 +85,7 @@ type Client struct { } func (self *Client) SetHttpClient(httpClient *http.Client) { + self.Credentials.SetHttpClient(httpClient) self.Servers.SetHttpClient(httpClient) self.ServersV2.SetHttpClient(httpClient) self.NovaServers.SetHttpClient(httpClient) @@ -198,6 +200,7 @@ func (self *Client) initManagers() { self.Jobs = modules.NewJobManager(self.regionId, self.projectId, self.signer, self.debug) self.Balances = modules.NewBalanceManager(self.signer, self.debug) self.Bandwidths = modules.NewBandwidthManager(self.regionId, self.projectId, self.signer, self.debug) + self.Credentials = modules.NewCredentialManager(self.signer, self.debug) self.Port = modules.NewPortManager(self.regionId, self.projectId, self.signer, self.debug) self.Flavors = modules.NewFlavorManager(self.regionId, self.projectId, self.signer, self.debug) self.VpcRoutes = modules.NewVpcRouteManager(self.regionId, self.projectId, self.signer, self.debug) diff --git a/pkg/multicloud/huawei/client/modules/mod_credential.go b/pkg/multicloud/huawei/client/modules/mod_credential.go new file mode 100644 index 0000000000..4c3ac8257e --- /dev/null +++ b/pkg/multicloud/huawei/client/modules/mod_credential.go @@ -0,0 +1,23 @@ +package modules + +import ( + "yunion.io/x/onecloud/pkg/multicloud/huawei/client/auth" +) + +type SCredentialManager struct { + SResourceManager +} + +func NewCredentialManager(signer auth.Signer, debug bool) *SCredentialManager { + return &SCredentialManager{SResourceManager: SResourceManager{ + SBaseManager: NewBaseManager(signer, debug), + ServiceName: ServiceNameIAM, + Region: "", + ProjectId: "", + version: "v3.0", + Keyword: "credential", + KeywordPlural: "credentials", + + ResourceKeyword: "OS-CREDENTIAL/credentials", + }} +} diff --git a/pkg/multicloud/huawei/client/modules/mod_users.go b/pkg/multicloud/huawei/client/modules/mod_users.go index 84f65eaafa..22c1474d91 100644 --- a/pkg/multicloud/huawei/client/modules/mod_users.go +++ b/pkg/multicloud/huawei/client/modules/mod_users.go @@ -28,10 +28,10 @@ func NewUserManager(signer auth.Signer, debug bool) *SUserManager { ServiceName: ServiceNameIAM, Region: "", ProjectId: "", - version: "v3", + version: "v3.0", Keyword: "user", KeywordPlural: "users", - ResourceKeyword: "users", + ResourceKeyword: "OS-USER/users", }} } diff --git a/pkg/multicloud/huawei/huawei.go b/pkg/multicloud/huawei/huawei.go index fe22228e2a..da0cb2baa7 100644 --- a/pkg/multicloud/huawei/huawei.go +++ b/pkg/multicloud/huawei/huawei.go @@ -87,8 +87,7 @@ type SHuaweiClient struct { isMainProject bool // whether the project is the main project in the region - ownerId string - ownerName string + ownerId string iregions []cloudprovider.ICloudRegion iBuckets []cloudprovider.ICloudBucket @@ -118,12 +117,12 @@ func (self *SHuaweiClient) init() error { if err != nil { return errors.Wrap(err, "initSigner") } - err = self.fetchBuckets() + err = self.initOwner() if err != nil { return errors.Wrap(err, "fetchOwner") } if self.debug { - log.Debugf("OwnerId: %s OwnerName: %s", self.ownerId, self.ownerName) + log.Debugf("OwnerId: %s", self.ownerId) } return nil } @@ -238,7 +237,6 @@ func (self *SHuaweiClient) fetchBuckets() error { return errors.Wrap(err, "obscli.ListBuckets") } self.ownerId = output.Owner.ID - self.ownerName = output.Owner.DisplayName ret := make([]cloudprovider.ICloudBucket, 0) for i := range output.Buckets { @@ -479,3 +477,57 @@ func (self *SHuaweiClient) GetCapabilities() []string { } return caps } + +func (self *SHuaweiClient) GetUserId() (string, error) { + client, err := self.newGeneralAPIClient() + if err != nil { + return "", errors.Wrap(err, "SHuaweiClient.GetUserId.newGeneralAPIClient") + } + + type cred struct { + UserId string `json:"user_id"` + } + + ret := &cred{} + err = DoGet(client.Credentials.Get, self.accessKey, nil, ret) + if err != nil { + return "", errors.Wrap(err, "SHuaweiClient.GetUserId.DoGet") + } + + return ret.UserId, nil +} + +// owner id == domain_id == account id +func (self *SHuaweiClient) GetOwnerId() (string, error) { + userId, err := self.GetUserId() + if err != nil { + return "", errors.Wrap(err, "SHuaweiClient.GetOwnerId.GetUserId") + } + + client, err := self.newGeneralAPIClient() + if err != nil { + return "", errors.Wrap(err, "SHuaweiClient.GetOwnerId.newGeneralAPIClient") + } + + type user struct { + DomainId string `json:"domain_id"` + } + + ret := &user{} + err = DoGet(client.Users.Get, userId, nil, ret) + if err != nil { + return "", errors.Wrap(err, "SHuaweiClient.GetOwnerId.DoGet") + } + + return ret.DomainId, nil +} + +func (self *SHuaweiClient) initOwner() error { + ownerId, err := self.GetOwnerId() + if err != nil { + return errors.Wrap(err, "SHuaweiClient.initOwner") + } + + self.ownerId = ownerId + return nil +} diff --git a/pkg/multicloud/huawei/obs/auth.go b/pkg/multicloud/huawei/obs/auth.go index 0c7d704d57..17ccde2069 100644 --- a/pkg/multicloud/huawei/obs/auth.go +++ b/pkg/multicloud/huawei/obs/auth.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -24,7 +22,14 @@ import ( func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params map[string]string, headers map[string][]string, expires int64) (requestUrl string, err error) { - + isAkSkEmpty := obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" + if isAkSkEmpty == false && obsClient.conf.securityProvider.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = obsClient.conf.securityProvider.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + } + } requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true) parsedRequestUrl, err := url.Parse(requestUrl) if err != nil { @@ -36,18 +41,28 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, isV4 := obsClient.conf.signature == SignatureV4 prepareHostAndDate(headers, hostName, isV4) - if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" { + if isAkSkEmpty { doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") } else { - if obsClient.conf.securityProvider.securityToken != "" { - params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken - } - if isV4 { - date, _ := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + date, parseDateErr := time.Parse(RFC1123_FORMAT, headers[HEADER_DATE_CAMEL][0]) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } delete(headers, HEADER_DATE_CAMEL) shortDate := date.Format(SHORT_DATE_FORMAT) longDate := date.Format(LONG_DATE_FORMAT) + if len(headers[HEADER_HOST_CAMEL]) != 0 { + index := strings.LastIndex(headers[HEADER_HOST_CAMEL][0], ":") + if index != -1 { + port := headers[HEADER_HOST_CAMEL][0][index+1:] + if port == "80" || port == "443" { + headers[HEADER_HOST_CAMEL] = []string{headers[HEADER_HOST_CAMEL][0][:index]} + } + } + + } signedHeaders, _headers := getSignedHeaders(headers) @@ -59,7 +74,12 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, params[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = strings.Join(signedHeaders, ";") requestUrl, canonicalizedUrl = obsClient.conf.formatUrls(bucketName, objectKey, params, true) - parsedRequestUrl, _ = url.Parse(requestUrl) + parsedRequestUrl, _err := url.Parse(requestUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl with reason: %v", _err) + return "", _err + } + stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, signedHeaders, _headers) signature := getSignature(stringToSign, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) @@ -67,7 +87,11 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, } else { originDate := headers[HEADER_DATE_CAMEL][0] - date, _ := time.Parse(RFC1123_FORMAT, originDate) + date, parseDateErr := time.Parse(RFC1123_FORMAT, originDate) + if parseDateErr != nil { + doLog(LEVEL_WARN, "Failed to parse date with reason: %v", parseDateErr) + return "", parseDateErr + } expires += date.Unix() headers[HEADER_DATE_CAMEL] = []string{Int64ToString(expires)} @@ -79,8 +103,11 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, requestUrl += "&" } delete(headers, HEADER_DATE_CAMEL) - requestUrl += fmt.Sprintf("AWSAccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(obsClient.conf.securityProvider.ak, false), - expires, signature) + + if obsClient.conf.signature != SignatureObs { + requestUrl += "AWS" + } + requestUrl += fmt.Sprintf("AccessKeyId=%s&Expires=%d&Signature=%s", UrlEncode(obsClient.conf.securityProvider.ak, false), expires, signature) } } @@ -89,6 +116,14 @@ func (obsClient ObsClient) doAuthTemporary(method, bucketName, objectKey string, func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params map[string]string, headers map[string][]string, hostName string) (requestUrl string, err error) { + isAkSkEmpty := obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" + if isAkSkEmpty == false && obsClient.conf.securityProvider.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + headers[HEADER_STS_TOKEN_OBS] = []string{obsClient.conf.securityProvider.securityToken} + } else { + headers[HEADER_STS_TOKEN_AMZ] = []string{obsClient.conf.securityProvider.securityToken} + } + } isObs := obsClient.conf.signature == SignatureObs requestUrl, canonicalizedUrl := obsClient.conf.formatUrls(bucketName, objectKey, params, true) parsedRequestUrl, err := url.Parse(requestUrl) @@ -104,12 +139,9 @@ func (obsClient ObsClient) doAuth(method, bucketName, objectKey string, params m isV4 := obsClient.conf.signature == SignatureV4 prepareHostAndDate(headers, hostName, isV4) - if obsClient.conf.securityProvider == nil || obsClient.conf.securityProvider.ak == "" || obsClient.conf.securityProvider.sk == "" { + if isAkSkEmpty { doLog(LEVEL_WARN, "No ak/sk provided, skip to construct authorization") } else { - if obsClient.conf.securityProvider.securityToken != "" { - headers[HEADER_STS_TOKEN_AMZ] = []string{obsClient.conf.securityProvider.securityToken} - } ak := obsClient.conf.securityProvider.ak sk := obsClient.conf.securityProvider.sk var authorization string @@ -287,6 +319,7 @@ func getV4StringToSign(method, canonicalizedUrl, queryUrl, scope, longDate, payl canonicalRequest = append(canonicalRequest, payload) _canonicalRequest := strings.Join(canonicalRequest, "") + doLog(LEVEL_DEBUG, "The v4 auth canonicalRequest:\n%s", _canonicalRequest) stringToSign := make([]string, 0, 7) diff --git a/pkg/multicloud/huawei/obs/client.go b/pkg/multicloud/huawei/obs/client.go index 986c418e33..b5abfe90c9 100644 --- a/pkg/multicloud/huawei/obs/client.go +++ b/pkg/multicloud/huawei/obs/client.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -32,6 +30,7 @@ type ObsClient struct { func New(ak, sk, endpoint string, configurers ...configurer) (*ObsClient, error) { conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, endpoint: endpoint} conf.maxRetryCount = -1 + conf.maxRedirectCount = -1 for _, configurer := range configurers { configurer(conf) } @@ -74,7 +73,7 @@ func (obsClient ObsClient) Close() { func (obsClient ObsClient) ListBuckets(input *ListBucketsInput) (output *ListBucketsOutput, err error) { if input == nil { - input = &ListBucketsInput{QueryLocation: true} + input = &ListBucketsInput{} } output = &ListBucketsOutput{} err = obsClient.doActionWithoutBucket("ListBuckets", HTTP_GET, input, output) @@ -123,6 +122,7 @@ func (obsClient ObsClient) getBucketStoragePolicyS3(bucketName string) (output * err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStoragePolicy), outputS3) if err != nil { output = nil + return } output.BaseModel = outputS3.BaseModel output.StorageClass = fmt.Sprintf("%s", outputS3.StorageClass) @@ -136,6 +136,7 @@ func (obsClient ObsClient) getBucketStoragePolicyObs(bucketName string) (output err = obsClient.doActionWithBucket("GetBucketStoragePolicy", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageClass), outputObs) if err != nil { output = nil + return } output.BaseModel = outputObs.BaseModel output.StorageClass = outputObs.StorageClass @@ -233,6 +234,17 @@ func (obsClient ObsClient) GetBucketMetadata(input *GetBucketMetadataInput) (out return } +func (obsClient ObsClient) SetObjectMetadata(input *SetObjectMetadataInput) (output *SetObjectMetadataOutput, err error) { + output = &SetObjectMetadataOutput{} + err = obsClient.doActionWithBucketAndKey("SetObjectMetadata", HTTP_PUT, input.Bucket, input.Key, input, output) + if err != nil { + output = nil + } else { + ParseSetObjectMetadataOutput(output) + } + return +} + func (obsClient ObsClient) GetBucketStorageInfo(bucketName string) (output *GetBucketStorageInfoOutput, err error) { output = &GetBucketStorageInfoOutput{} err = obsClient.doActionWithBucket("GetBucketStorageInfo", HTTP_GET, bucketName, newSubResourceSerial(SubResourceStorageInfo), output) @@ -295,13 +307,18 @@ func (obsClient ObsClient) getBucketAclObs(bucketName string) (output *GetBucket output = nil } else { output.BaseModel = outputObs.BaseModel - for i, valGrant := range outputObs.Grants { - output.Grants[i].Delivered = valGrant.Delivered - output.Grants[i].Permission = valGrant.Permission - output.Grants[i].Grantee.DisplayName = valGrant.Grantee.DisplayName - output.Grants[i].Grantee.ID = valGrant.Grantee.ID - output.Grants[i].Grantee.Type = valGrant.Grantee.Type - output.Grants[i].Grantee.URI = GroupAllUsers + output.Owner = outputObs.Owner + output.Grants = make([]Grant, 0, len(outputObs.Grants)) + for _, valGrant := range outputObs.Grants { + tempOutput := Grant{} + tempOutput.Delivered = valGrant.Delivered + tempOutput.Permission = valGrant.Permission + tempOutput.Grantee.DisplayName = valGrant.Grantee.DisplayName + tempOutput.Grantee.ID = valGrant.Grantee.ID + tempOutput.Grantee.Type = valGrant.Grantee.Type + tempOutput.Grantee.URI = GroupAllUsers + + output.Grants = append(output.Grants, tempOutput) } } return @@ -661,7 +678,7 @@ func (obsClient ObsClient) PutObject(input *PutObjectInput) (output *PutObjectOu } if input.ContentType == "" && input.Key != "" { - if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok { + if contentType, ok := mime_types[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { input.ContentType = contentType } } @@ -687,6 +704,23 @@ func (obsClient ObsClient) PutObject(input *PutObjectInput) (output *PutObjectOu return } +func (obsClient ObsClient) getContentType(input *PutObjectInput, sourceFile string) (contentType string) { + if contentType, ok := mime_types[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { + return contentType + } + if contentType, ok := mime_types[strings.ToLower(sourceFile[strings.LastIndex(sourceFile, ".")+1:])]; ok { + return contentType + } + return +} + +func (ObsClient ObsClient) isGetContentType(input *PutObjectInput) bool { + if input.ContentType == "" && input.Key != "" { + return true + } + return false +} + func (obsClient ObsClient) PutFile(input *PutFileInput) (output *PutObjectOutput, err error) { if input == nil { return nil, errors.New("PutFileInput is nil") @@ -695,14 +729,21 @@ func (obsClient ObsClient) PutFile(input *PutFileInput) (output *PutObjectOutput var body io.Reader sourceFile := strings.TrimSpace(input.SourceFile) if sourceFile != "" { - fd, err := os.Open(sourceFile) - if err != nil { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err return nil, err } - defer fd.Close() + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() - stat, err := fd.Stat() - if err != nil { + stat, _err := fd.Stat() + if _err != nil { + err = _err return nil, err } fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} @@ -722,12 +763,8 @@ func (obsClient ObsClient) PutFile(input *PutFileInput) (output *PutObjectOutput _input.PutObjectBasicInput = input.PutObjectBasicInput _input.Body = body - if _input.ContentType == "" && _input.Key != "" { - if contentType, ok := mime_types[_input.Key[strings.LastIndex(_input.Key, ".")+1:]]; ok { - _input.ContentType = contentType - } else if contentType, ok := mime_types[sourceFile[strings.LastIndex(sourceFile, ".")+1:]]; ok { - _input.ContentType = contentType - } + if obsClient.isGetContentType(_input) { + _input.ContentType = obsClient.getContentType(_input, sourceFile) } output = &PutObjectOutput{} @@ -783,7 +820,7 @@ func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploa } if input.ContentType == "" && input.Key != "" { - if contentType, ok := mime_types[input.Key[strings.LastIndex(input.Key, ".")+1:]]; ok { + if contentType, ok := mime_types[strings.ToLower(input.Key[strings.LastIndex(input.Key, ".")+1:])]; ok { input.ContentType = contentType } } @@ -798,31 +835,50 @@ func (obsClient ObsClient) InitiateMultipartUpload(input *InitiateMultipartUploa return } -func (obsClient ObsClient) UploadPart(input *UploadPartInput) (output *UploadPartOutput, err error) { - if input == nil { +func (obsClient ObsClient) UploadPart(_input *UploadPartInput) (output *UploadPartOutput, err error) { + if _input == nil { return nil, errors.New("UploadPartInput is nil") } - if input.UploadId == "" { + if _input.UploadId == "" { return nil, errors.New("UploadId is empty") } + input := &UploadPartInput{} + input.Bucket = _input.Bucket + input.Key = _input.Key + input.PartNumber = _input.PartNumber + input.UploadId = _input.UploadId + input.ContentMD5 = _input.ContentMD5 + input.SourceFile = _input.SourceFile + input.Offset = _input.Offset + input.PartSize = _input.PartSize + input.SseHeader = _input.SseHeader + input.Body = _input.Body + output = &UploadPartOutput{} var repeatable bool if input.Body != nil { _, repeatable = input.Body.(*strings.Reader) - if input.PartSize > 0 { + if _, ok := input.Body.(*readerWrapper); !ok && input.PartSize > 0 { input.Body = &readerWrapper{reader: input.Body, totalCount: input.PartSize} } } else if sourceFile := strings.TrimSpace(input.SourceFile); sourceFile != "" { - fd, err := os.Open(sourceFile) - if err != nil { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err return nil, err } - defer fd.Close() + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() - stat, err := fd.Stat() - if err != nil { + stat, _err := fd.Stat() + if _err != nil { + err = _err return nil, err } fileSize := stat.Size() @@ -837,7 +893,9 @@ func (obsClient ObsClient) UploadPart(input *UploadPartInput) (output *UploadPar input.PartSize = fileSize - input.Offset } fileReaderWrapper.totalCount = input.PartSize - fd.Seek(input.Offset, 0) + if _, err = fd.Seek(input.Offset, io.SeekStart); err != nil { + return nil, err + } input.Body = fileReaderWrapper repeatable = true } diff --git a/pkg/multicloud/huawei/obs/conf.go b/pkg/multicloud/huawei/obs/conf.go index 22e3f66368..38e9d4d7ee 100644 --- a/pkg/multicloud/huawei/obs/conf.go +++ b/pkg/multicloud/huawei/obs/conf.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -44,9 +42,11 @@ type urlHolder struct { type config struct { securityProvider *securityProvider urlHolder *urlHolder + pathStyle bool + cname bool + sslVerify bool endpoint string signature SignatureType - pathStyle bool region string connectTimeout int socketTimeout int @@ -56,19 +56,19 @@ type config struct { maxRetryCount int proxyUrl string maxConnsPerHost int - sslVerify bool pemCerts []byte transport *http.Transport ctx context.Context + maxRedirectCount int } func (conf config) String() string { return fmt.Sprintf("[endpoint:%s, signature:%s, pathStyle:%v, region:%s"+ "\nconnectTimeout:%d, socketTimeout:%dheaderTimeout:%d, idleConnTimeout:%d"+ - "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, proxyUrl:%s]", + "\nmaxRetryCount:%d, maxConnsPerHost:%d, sslVerify:%v, proxyUrl:%s, maxRedirectCount:%d]", conf.endpoint, conf.signature, conf.pathStyle, conf.region, conf.connectTimeout, conf.socketTimeout, conf.headerTimeout, conf.idleConnTimeout, - conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.proxyUrl, + conf.maxRetryCount, conf.maxConnsPerHost, conf.sslVerify, conf.proxyUrl, conf.maxRedirectCount, ) } @@ -163,6 +163,18 @@ func WithRequestContext(ctx context.Context) configurer { } } +func WithCustomDomainName(cname bool) configurer { + return func(conf *config) { + conf.cname = cname + } +} + +func WithMaxRedirectCount(maxRedirectCount int) configurer { + return func(conf *config) { + conf.maxRedirectCount = maxRedirectCount + } +} + func (conf *config) initConfigWithDefault() error { conf.securityProvider.ak = strings.TrimSpace(conf.securityProvider.ak) conf.securityProvider.sk = strings.TrimSpace(conf.securityProvider.sk) @@ -249,6 +261,10 @@ func (conf *config) initConfigWithDefault() error { conf.maxConnsPerHost = DEFAULT_MAX_CONN_PER_HOST } + if conf.maxRedirectCount < 0 { + conf.maxRedirectCount = DEFAULT_MAX_REDIRECT_COUNT + } + conf.proxyUrl = strings.TrimSpace(conf.proxyUrl) return nil } @@ -275,8 +291,6 @@ func (conf *config) getTransport() error { return err } conf.transport.Proxy = http.ProxyURL(proxyUrl) - } else { - conf.transport.Proxy = http.ProxyFromEnvironment } tlsConfig := &tls.Config{InsecureSkipVerify: !conf.sslVerify} @@ -303,19 +317,28 @@ func DummyQueryEscape(s string) string { func (conf *config) formatUrls(bucketName, objectKey string, params map[string]string, escape bool) (requestUrl string, canonicalizedUrl string) { urlHolder := conf.urlHolder - if bucketName == "" { + if conf.cname { requestUrl = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) - canonicalizedUrl = "/" - } else { - if conf.pathStyle { - requestUrl = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) - canonicalizedUrl = "/" + bucketName + if conf.signature == "v4" { + canonicalizedUrl = "/" } else { - requestUrl = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) - if conf.signature == "v2" { - canonicalizedUrl = "/" + bucketName + "/" + canonicalizedUrl = "/" + urlHolder.host + "/" + } + } else { + if bucketName == "" { + requestUrl = fmt.Sprintf("%s://%s:%d", urlHolder.scheme, urlHolder.host, urlHolder.port) + canonicalizedUrl = "/" + } else { + if conf.pathStyle { + requestUrl = fmt.Sprintf("%s://%s:%d/%s", urlHolder.scheme, urlHolder.host, urlHolder.port, bucketName) + canonicalizedUrl = "/" + bucketName } else { - canonicalizedUrl = "/" + requestUrl = fmt.Sprintf("%s://%s.%s:%d", urlHolder.scheme, bucketName, urlHolder.host, urlHolder.port) + if conf.signature == "v2" || conf.signature == "OBS" { + canonicalizedUrl = "/" + bucketName + "/" + } else { + canonicalizedUrl = "/" + } } } } @@ -327,7 +350,21 @@ func (conf *config) formatUrls(bucketName, objectKey string, params map[string]s } if objectKey != "" { - encodeObjectKey := escapeFunc(objectKey) + var encodeObjectKey string + if escape { + tempKey := []rune(objectKey) + result := make([]string, 0, len(tempKey)) + for _, value := range tempKey { + if string(value) == "/" { + result = append(result, string(value)) + } else { + result = append(result, url.QueryEscape(string(value))) + } + } + encodeObjectKey = strings.Join(result, "") + } else { + encodeObjectKey = escapeFunc(objectKey) + } requestUrl += "/" + encodeObjectKey if !strings.HasSuffix(canonicalizedUrl, "/") { canonicalizedUrl += "/" diff --git a/pkg/multicloud/huawei/obs/const.go b/pkg/multicloud/huawei/obs/const.go index 0fb03a38d6..9bc8020dd4 100644 --- a/pkg/multicloud/huawei/obs/const.go +++ b/pkg/multicloud/huawei/obs/const.go @@ -1,21 +1,19 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs const ( - obs_sdk_version = "3.0.0" + obs_sdk_version = "3.20.1" USER_AGENT = "obs-sdk-go/" + obs_sdk_version HEADER_PREFIX = "x-amz-" HEADER_PREFIX_META = "x-amz-meta-" @@ -24,6 +22,7 @@ const ( HEADER_DATE_AMZ = "x-amz-date" HEADER_DATE_OBS = "x-obs-date" HEADER_STS_TOKEN_AMZ = "x-amz-security-token" + HEADER_STS_TOKEN_OBS = "x-obs-security-token" HEADER_ACCESSS_KEY_AMZ = "AWSAccessKeyId" PREFIX_META = "meta-" @@ -115,6 +114,11 @@ const ( HEADER_USER_AGENT_CAMEL = "User-Agent" HEADER_ORIGIN_CAMEL = "Origin" HEADER_ACCESS_CONTROL_REQUEST_HEADER_CAMEL = "Access-Control-Request-Headers" + HEADER_CACHE_CONTROL_CAMEL = "Cache-Control" + HEADER_CONTENT_DISPOSITION_CAMEL = "Content-Disposition" + HEADER_CONTENT_ENCODING_CAMEL = "Content-Encoding" + HEADER_CONTENT_LANGUAGE_CAMEL = "Content-Language" + HEADER_EXPIRES_CAMEL = "Expires" PARAM_VERSION_ID = "versionId" PARAM_RESPONSE_CONTENT_TYPE = "response-content-type" @@ -140,6 +144,7 @@ const ( DEFAULT_HEADER_TIMEOUT = 60 DEFAULT_IDLE_CONN_TIMEOUT = 30 DEFAULT_MAX_RETRY_COUNT = 3 + DEFAULT_MAX_REDIRECT_COUNT = 3 DEFAULT_MAX_CONN_PER_HOST = 1000 EMPTY_CONTENT_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD" @@ -200,6 +205,7 @@ var ( "last-modified": true, "content-range": true, "x-reserved": true, + "x-reserved-indicator": true, "access-control-allow-origin": true, "access-control-allow-headers": true, "access-control-max-age": true, @@ -271,104 +277,385 @@ var ( "response-content-encoding": true, "x-image-process": true, "x-oss-process": true, + "x-image-save-bucket": true, + "x-image-save-object": true, + "ignore-sign-in-query": true, } mime_types = map[string]string{ + "001": "application/x-001", + "301": "application/x-301", + "323": "text/h323", "7z": "application/x-7z-compressed", + "906": "application/x-906", + "907": "drawing/907", + "IVF": "video/x-ivf", + "a11": "application/x-a11", "aac": "audio/x-aac", + "acp": "audio/x-mei-aac", "ai": "application/postscript", - "aif": "audio/x-aiff", - "asc": "text/plain", + "aif": "audio/aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "anv": "application/x-anv", + "apk": "application/vnd.android.package-archive", + "asa": "text/asa", "asf": "video/x-ms-asf", + "asp": "text/asp", + "asx": "video/x-ms-asf", "atom": "application/atom+xml", - "avi": "video/x-msvideo", - "bmp": "image/bmp", + "au": "audio/basic", + "avi": "video/avi", + "awf": "application/vnd.adobe.workflow", + "biz": "text/xml", + "bmp": "application/x-bmp", + "bot": "application/x-bot", "bz2": "application/x-bzip2", - "cer": "application/pkix-cert", + "c4t": "application/x-c4t", + "c90": "application/x-c90", + "cal": "application/x-cals", + "cat": "application/vnd.ms-pki.seccat", + "cdf": "application/x-netcdf", + "cdr": "application/x-cdr", + "cel": "application/x-cel", + "cer": "application/x-x509-ca-cert", + "cg4": "application/x-g4", + "cgm": "application/x-cgm", + "cit": "application/x-cit", + "class": "java/*", + "cml": "text/xml", + "cmp": "application/x-cmp", + "cmx": "application/x-cmx", + "cot": "application/x-cot", "crl": "application/pkix-crl", "crt": "application/x-x509-ca-cert", + "csi": "application/x-csi", "css": "text/css", "csv": "text/csv", "cu": "application/cu-seeme", + "cut": "application/x-cut", + "dbf": "application/x-dbf", + "dbm": "application/x-dbm", + "dbx": "application/x-dbx", + "dcd": "text/xml", + "dcx": "application/x-dcx", "deb": "application/x-debian-package", + "der": "application/x-x509-ca-cert", + "dgn": "application/x-dgn", + "dib": "application/x-dib", + "dll": "application/x-msdownload", "doc": "application/msword", "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dot": "application/msword", + "drw": "application/x-drw", + "dtd": "text/xml", "dvi": "application/x-dvi", + "dwf": "application/x-dwf", + "dwg": "application/x-dwg", + "dxb": "application/x-dxb", + "dxf": "application/x-dxf", + "edn": "application/vnd.adobe.edn", + "emf": "application/x-emf", + "eml": "message/rfc822", + "ent": "text/xml", "eot": "application/vnd.ms-fontobject", + "epi": "application/x-epi", "eps": "application/postscript", "epub": "application/epub+zip", + "etd": "application/x-ebx", "etx": "text/x-setext", + "exe": "application/x-msdownload", + "fax": "image/fax", + "fdf": "application/vnd.fdf", + "fif": "application/fractals", "flac": "audio/flac", "flv": "video/x-flv", + "fo": "text/xml", + "frm": "application/x-frm", + "g4": "application/x-g4", + "gbr": "application/x-gbr", "gif": "image/gif", + "gl2": "application/x-gl2", + "gp4": "application/x-gp4", "gz": "application/gzip", + "hgl": "application/x-hgl", + "hmr": "application/x-hmr", + "hpg": "application/x-hpgl", + "hpl": "application/x-hpl", + "hqx": "application/mac-binhex40", + "hrf": "application/x-hrf", + "hta": "application/hta", + "htc": "text/x-component", "htm": "text/html", "html": "text/html", - "ico": "image/x-icon", + "htt": "text/webviewhtml", + "htx": "text/html", + "icb": "application/x-icb", + "ico": "application/x-ico", "ics": "text/calendar", + "iff": "application/x-iff", + "ig4": "application/x-g4", + "igs": "application/x-igs", + "iii": "application/x-iphone", + "img": "application/x-img", "ini": "text/plain", + "ins": "application/x-internet-signup", + "ipa": "application/vnd.iphone", "iso": "application/x-iso9660-image", + "isp": "application/x-internet-signup", "jar": "application/java-archive", + "java": "java/*", + "jfif": "image/jpeg", "jpe": "image/jpeg", "jpeg": "image/jpeg", "jpg": "image/jpeg", - "js": "text/javascript", + "js": "application/x-javascript", "json": "application/json", + "jsp": "text/html", + "la1": "audio/x-liquid-file", + "lar": "application/x-laplayer-reg", "latex": "application/x-latex", + "lavs": "audio/x-liquid-secure", + "lbm": "application/x-lbm", + "lmsff": "audio/x-la-lms", "log": "text/plain", + "ls": "application/x-javascript", + "ltr": "application/x-ltr", + "m1v": "video/x-mpeg", + "m2v": "video/x-mpeg", + "m3u": "audio/mpegurl", "m4a": "audio/mp4", + "m4e": "video/mpeg4", "m4v": "video/mp4", - "mid": "audio/midi", - "midi": "audio/midi", + "mac": "application/x-mac", + "man": "application/x-troff-man", + "math": "text/xml", + "mdb": "application/msaccess", + "mfp": "application/x-shockwave-flash", + "mht": "message/rfc822", + "mhtml": "message/rfc822", + "mi": "application/x-mi", + "mid": "audio/mid", + "midi": "audio/mid", + "mil": "application/x-mil", + "mml": "text/xml", + "mnd": "audio/x-musicnet-download", + "mns": "audio/x-musicnet-stream", + "mocha": "application/x-javascript", "mov": "video/quicktime", - "mp3": "audio/mpeg", - "mp4": "video/mp4", + "movie": "video/x-sgi-movie", + "mp1": "audio/mp1", + "mp2": "audio/mp2", + "mp2v": "video/mpeg", + "mp3": "audio/mp3", + "mp4": "video/mpeg4", "mp4a": "audio/mp4", "mp4v": "video/mp4", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpg": "video/mpeg", + "mpa": "video/x-mpg", + "mpd": "application/vnd.ms-project", + "mpe": "video/x-mpeg", + "mpeg": "video/mpg", + "mpg": "video/mpg", "mpg4": "video/mp4", + "mpga": "audio/rn-mpeg", + "mpp": "application/vnd.ms-project", + "mps": "video/x-mpeg", + "mpt": "application/vnd.ms-project", + "mpv": "video/mpg", + "mpv2": "video/mpeg", + "mpw": "application/vnd.ms-project", + "mpx": "application/vnd.ms-project", + "mtx": "text/xml", + "mxp": "application/x-mmxp", + "net": "image/pnetvue", + "nrf": "application/x-nrf", + "nws": "message/rfc822", + "odc": "text/x-ms-odc", "oga": "audio/ogg", "ogg": "audio/ogg", "ogv": "video/ogg", "ogx": "application/ogg", + "out": "application/x-out", + "p10": "application/pkcs10", + "p12": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "p7c": "application/pkcs7-mime", + "p7m": "application/pkcs7-mime", + "p7r": "application/x-pkcs7-certreqresp", + "p7s": "application/pkcs7-signature", "pbm": "image/x-portable-bitmap", + "pc5": "application/x-pc5", + "pci": "application/x-pci", + "pcl": "application/x-pcl", + "pcx": "application/x-pcx", "pdf": "application/pdf", + "pdx": "application/vnd.adobe.pdx", + "pfx": "application/x-pkcs12", + "pgl": "application/x-pgl", "pgm": "image/x-portable-graymap", + "pic": "application/x-pic", + "pko": "application/vnd.ms-pki.pko", + "pl": "application/x-perl", + "plg": "text/html", + "pls": "audio/scpls", + "plt": "application/x-plt", "png": "image/png", "pnm": "image/x-portable-anymap", - "ppm": "image/x-portable-pixmap", + "pot": "application/vnd.ms-powerpoint", + "ppa": "application/vnd.ms-powerpoint", + "ppm": "application/x-ppm", + "pps": "application/vnd.ms-powerpoint", "ppt": "application/vnd.ms-powerpoint", "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pr": "application/x-pr", + "prf": "application/pics-rules", + "prn": "application/x-prn", + "prt": "application/x-prt", "ps": "application/postscript", + "ptn": "application/x-ptn", + "pwz": "application/vnd.ms-powerpoint", "qt": "video/quicktime", + "r3t": "text/vnd.rn-realtext3d", + "ra": "audio/vnd.rn-realaudio", + "ram": "audio/x-pn-realaudio", "rar": "application/x-rar-compressed", - "ras": "image/x-cmu-raster", + "ras": "application/x-ras", + "rat": "application/rat-file", + "rdf": "text/xml", + "rec": "application/vnd.rn-recording", + "red": "application/x-red", + "rgb": "application/x-rgb", + "rjs": "application/vnd.rn-realsystem-rjs", + "rjt": "application/vnd.rn-realsystem-rjt", + "rlc": "application/x-rlc", + "rle": "application/x-rle", + "rm": "application/vnd.rn-realmedia", + "rmf": "application/vnd.adobe.rmf", + "rmi": "audio/mid", + "rmj": "application/vnd.rn-realsystem-rmj", + "rmm": "audio/x-pn-realaudio", + "rmp": "application/vnd.rn-rn_music_package", + "rms": "application/vnd.rn-realmedia-secure", + "rmvb": "application/vnd.rn-realmedia-vbr", + "rmx": "application/vnd.rn-realsystem-rmx", + "rnx": "application/vnd.rn-realplayer", + "rp": "image/vnd.rn-realpix", + "rpm": "audio/x-pn-realaudio-plugin", + "rsml": "application/vnd.rn-rsml", "rss": "application/rss+xml", - "rtf": "application/rtf", + "rt": "text/vnd.rn-realtext", + "rtf": "application/x-rtf", + "rv": "video/vnd.rn-realvideo", + "sam": "application/x-sam", + "sat": "application/x-sat", + "sdp": "application/sdp", + "sdw": "application/x-sdw", "sgm": "text/sgml", "sgml": "text/sgml", + "sis": "application/vnd.symbian.install", + "sisx": "application/vnd.symbian.install", + "sit": "application/x-stuffit", + "slb": "application/x-slb", + "sld": "application/x-sld", + "slk": "drawing/x-slk", + "smi": "application/smil", + "smil": "application/smil", + "smk": "application/x-smk", + "snd": "audio/basic", + "sol": "text/plain", + "sor": "text/plain", + "spc": "application/x-pkcs7-certificates", + "spl": "application/futuresplash", + "spp": "text/xml", + "ssm": "application/streamingmedia", + "sst": "application/vnd.ms-pki.certstore", + "stl": "application/vnd.ms-pki.stl", + "stm": "text/html", + "sty": "application/x-sty", "svg": "image/svg+xml", "swf": "application/x-shockwave-flash", "tar": "application/x-tar", + "tdf": "application/x-tdf", + "tg4": "application/x-tg4", + "tga": "application/x-tga", "tif": "image/tiff", "tiff": "image/tiff", + "tld": "text/xml", + "top": "drawing/x-top", "torrent": "application/x-bittorrent", + "tsd": "text/xml", "ttf": "application/x-font-ttf", "txt": "text/plain", - "wav": "audio/x-wav", + "uin": "application/x-icq", + "uls": "text/iuls", + "vcf": "text/x-vcard", + "vda": "application/x-vda", + "vdx": "application/vnd.visio", + "vml": "text/xml", + "vpg": "application/x-vpeg005", + "vsd": "application/vnd.visio", + "vss": "application/vnd.visio", + "vst": "application/x-vst", + "vsw": "application/vnd.visio", + "vsx": "application/vnd.visio", + "vtx": "application/vnd.visio", + "vxml": "text/xml", + "wav": "audio/wav", + "wax": "audio/x-ms-wax", + "wb1": "application/x-wb1", + "wb2": "application/x-wb2", + "wb3": "application/x-wb3", + "wbmp": "image/vnd.wap.wbmp", "webm": "video/webm", + "wiz": "application/msword", + "wk3": "application/x-wk3", + "wk4": "application/x-wk4", + "wkq": "application/x-wkq", + "wks": "application/x-wks", + "wm": "video/x-ms-wm", "wma": "audio/x-ms-wma", + "wmd": "application/x-ms-wmd", + "wmf": "application/x-wmf", + "wml": "text/vnd.wap.wml", "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wmz": "application/x-ms-wmz", "woff": "application/x-font-woff", - "wsdl": "application/wsdl+xml", + "wp6": "application/x-wp6", + "wpd": "application/x-wpd", + "wpg": "application/x-wpg", + "wpl": "application/vnd.ms-wpl", + "wq1": "application/x-wq1", + "wr1": "application/x-wr1", + "wri": "application/x-wri", + "wrk": "application/x-wrk", + "ws": "application/x-ws", + "ws2": "application/x-ws", + "wsc": "text/scriptlet", + "wsdl": "text/xml", + "wvx": "video/x-ms-wvx", + "x_b": "application/x-x_b", + "x_t": "application/x-x_t", + "xap": "application/x-silverlight-app", "xbm": "image/x-xbitmap", + "xdp": "application/vnd.adobe.xdp", + "xdr": "text/xml", + "xfd": "application/vnd.adobe.xfd", + "xfdf": "application/vnd.adobe.xfdf", + "xhtml": "text/html", "xls": "application/vnd.ms-excel", "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xml": "application/xml", + "xlw": "application/x-xlw", + "xml": "text/xml", + "xpl": "audio/scpls", "xpm": "image/x-xpixmap", - "xwd": "image/x-xwindowdump", + "xq": "text/xml", + "xql": "text/xml", + "xquery": "text/xml", + "xsd": "text/xml", + "xsl": "text/xml", + "xslt": "text/xml", + "xwd": "application/x-xwd", "yaml": "text/yaml", "yml": "text/yaml", "zip": "application/zip", @@ -407,6 +694,7 @@ const ( SubResourceVersions SubResourceType = "versions" SubResourceUploads SubResourceType = "uploads" SubResourceRestore SubResourceType = "restore" + SubResourceMetadata SubResourceType = "metadata" ) type AclType string @@ -426,9 +714,11 @@ const ( type StorageClassType string const ( - StorageClassStandard StorageClassType = "STANDARD" - StorageClassWarm StorageClassType = "STANDARD_IA" - StorageClassCold StorageClassType = "GLACIER" + StorageClassStandard StorageClassType = "STANDARD" + StorageClassWarm StorageClassType = "WARM" + StorageClassCold StorageClassType = "COLD" + storageClassStandardIA StorageClassType = "STANDARD_IA" + storageClassGlacier StorageClassType = "GLACIER" ) type PermissionType string @@ -489,6 +779,7 @@ type MetadataDirectiveType string const ( CopyMetadata MetadataDirectiveType = "COPY" + ReplaceNew MetadataDirectiveType = "REPLACE_NEW" ReplaceMetadata MetadataDirectiveType = "REPLACE" ) diff --git a/pkg/multicloud/huawei/obs/convert.go b/pkg/multicloud/huawei/obs/convert.go index 9858b349a3..9a468a928d 100644 --- a/pkg/multicloud/huawei/obs/convert.go +++ b/pkg/multicloud/huawei/obs/convert.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -25,17 +23,13 @@ import ( "time" ) -func cleanHeaderPrefix(header http.Header, isObs bool) map[string][]string { +func cleanHeaderPrefix(header http.Header) map[string][]string { responseHeaders := make(map[string][]string) for key, value := range header { if len(value) > 0 { key = strings.ToLower(key) - headerPrefix := HEADER_PREFIX - if isObs { - headerPrefix = HEADER_PREFIX_OBS - } - if strings.HasPrefix(key, headerPrefix) { - key = key[len(headerPrefix):] + if strings.HasPrefix(key, HEADER_PREFIX) || strings.HasPrefix(key, HEADER_PREFIX_OBS) { + key = key[len(HEADER_PREFIX):] } responseHeaders[key] = value } @@ -81,63 +75,85 @@ func ParseStringToStorageClassType(value string) (ret StorageClassType) { return } -func convertGrantToXml(grant Grant, isObs bool) string { - xml := make([]string, 0, 4) - if !isObs { - xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) - } else { - xml = append(xml, fmt.Sprintf("")) +func prepareGrantURI(grant Grant) string { + if grant.Grantee.URI == GroupAllUsers || grant.Grantee.URI == GroupAuthenticatedUsers { + return fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/global/", grant.Grantee.URI) } + if grant.Grantee.URI == GroupLogDelivery { + return fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/s3/", grant.Grantee.URI) + } + return fmt.Sprintf("%s", grant.Grantee.URI) +} + +func convertGrantToXml(grant Grant, isObs bool, isBucket bool) string { + xml := make([]string, 0, 4) + if grant.Grantee.Type == GranteeUser { + if isObs { + xml = append(xml, "") + } else { + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + } if grant.Grantee.ID != "" { granteeID := XmlTranscoding(grant.Grantee.ID) xml = append(xml, fmt.Sprintf("%s", granteeID)) } - if grant.Grantee.DisplayName != "" { + if !isObs && grant.Grantee.DisplayName != "" { granteeDisplayName := XmlTranscoding(grant.Grantee.DisplayName) xml = append(xml, fmt.Sprintf("%s", granteeDisplayName)) } + xml = append(xml, "") } else { if !isObs { - if grant.Grantee.URI == GroupAllUsers || grant.Grantee.URI == GroupAuthenticatedUsers { - xml = append(xml, fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/global/", grant.Grantee.URI)) - } else if grant.Grantee.URI == GroupLogDelivery { - xml = append(xml, fmt.Sprintf("%s%s", "http://acs.amazonaws.com/groups/s3/", grant.Grantee.URI)) - } else { - xml = append(xml, fmt.Sprintf("%s", grant.Grantee.URI)) - } + xml = append(xml, fmt.Sprintf("", grant.Grantee.Type)) + xml = append(xml, prepareGrantURI(grant)) + xml = append(xml, "") } else if grant.Grantee.URI == GroupAllUsers { + xml = append(xml, "") xml = append(xml, fmt.Sprintf("Everyone")) + xml = append(xml, "") + } else { + return strings.Join(xml, "") } } - xml = append(xml, fmt.Sprintf("")) + xml = append(xml, fmt.Sprintf("%s", grant.Permission)) - if isObs { + if isObs && isBucket { xml = append(xml, fmt.Sprintf("%t", grant.Delivered)) } xml = append(xml, fmt.Sprintf("")) return strings.Join(xml, "") } +func hasLoggingTarget(input BucketLoggingStatus) bool { + if input.TargetBucket != "" || input.TargetPrefix != "" || len(input.TargetGrants) > 0 { + return true + } + return false +} + func ConvertLoggingStatusToXml(input BucketLoggingStatus, returnMd5 bool, isObs bool) (data string, md5 string) { grantsLength := len(input.TargetGrants) xml := make([]string, 0, 8+grantsLength) xml = append(xml, "") - if input.Agency != "" { + if isObs && input.Agency != "" { agency := XmlTranscoding(input.Agency) xml = append(xml, fmt.Sprintf("%s", agency)) } - if input.TargetBucket != "" || input.TargetPrefix != "" { + if hasLoggingTarget(input) { xml = append(xml, "") - xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) - targetPrefix := XmlTranscoding(input.TargetPrefix) - xml = append(xml, fmt.Sprintf("%s", targetPrefix)) - + if input.TargetBucket != "" { + xml = append(xml, fmt.Sprintf("%s", input.TargetBucket)) + } + if input.TargetPrefix != "" { + targetPrefix := XmlTranscoding(input.TargetPrefix) + xml = append(xml, fmt.Sprintf("%s", targetPrefix)) + } if grantsLength > 0 { xml = append(xml, "") for _, grant := range input.TargetGrants { - xml = append(xml, convertGrantToXml(grant, isObs)) + xml = append(xml, convertGrantToXml(grant, isObs, false)) } xml = append(xml, "") } @@ -156,13 +172,40 @@ func ConvertAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (dat xml := make([]string, 0, 4+len(input.Grants)) ownerID := XmlTranscoding(input.Owner.ID) xml = append(xml, fmt.Sprintf("%s", ownerID)) - if input.Owner.DisplayName != "" { + if !isObs && input.Owner.DisplayName != "" { ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) } - xml = append(xml, "") + if isObs && input.Delivered != "" { + objectDelivered := XmlTranscoding(input.Delivered) + xml = append(xml, fmt.Sprintf("%s", objectDelivered)) + } else { + xml = append(xml, "") + } for _, grant := range input.Grants { - xml = append(xml, convertGrantToXml(grant, isObs)) + xml = append(xml, convertGrantToXml(grant, isObs, false)) + } + xml = append(xml, "") + data = strings.Join(xml, "") + if returnMd5 { + md5 = Base64Md5([]byte(data)) + } + return +} + +func convertBucketAclToXml(input AccessControlPolicy, returnMd5 bool, isObs bool) (data string, md5 string) { + xml := make([]string, 0, 4+len(input.Grants)) + ownerID := XmlTranscoding(input.Owner.ID) + xml = append(xml, fmt.Sprintf("%s", ownerID)) + if !isObs && input.Owner.DisplayName != "" { + ownerDisplayName := XmlTranscoding(input.Owner.DisplayName) + xml = append(xml, fmt.Sprintf("%s", ownerDisplayName)) + } + + xml = append(xml, "") + + for _, grant := range input.Grants { + xml = append(xml, convertGrantToXml(grant, isObs, true)) } xml = append(xml, "") data = strings.Join(xml, "") @@ -187,6 +230,39 @@ func convertConditionToXml(condition Condition) string { return "" } +func prepareRoutingRule(input BucketWebsiteConfiguration) string { + xml := make([]string, 0, len(input.RoutingRules)*10) + for _, routingRule := range input.RoutingRules { + xml = append(xml, "") + xml = append(xml, "") + if routingRule.Redirect.Protocol != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.Protocol)) + } + if routingRule.Redirect.HostName != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HostName)) + } + if routingRule.Redirect.ReplaceKeyPrefixWith != "" { + replaceKeyPrefixWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyPrefixWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyPrefixWith)) + } + + if routingRule.Redirect.ReplaceKeyWith != "" { + replaceKeyWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyWith) + xml = append(xml, fmt.Sprintf("%s", replaceKeyWith)) + } + if routingRule.Redirect.HttpRedirectCode != "" { + xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HttpRedirectCode)) + } + xml = append(xml, "") + + if ret := convertConditionToXml(routingRule.Condition); ret != "" { + xml = append(xml, ret) + } + xml = append(xml, "") + } + return strings.Join(xml, "") +} + func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd5 bool) (data string, md5 string) { routingRuleLength := len(input.RoutingRules) xml := make([]string, 0, 6+routingRuleLength*10) @@ -199,42 +275,17 @@ func ConvertWebsiteConfigurationToXml(input BucketWebsiteConfiguration, returnMd } xml = append(xml, "") } else { - indexDocumentSuffix := XmlTranscoding(input.IndexDocument.Suffix) - xml = append(xml, fmt.Sprintf("%s", indexDocumentSuffix)) + if input.IndexDocument.Suffix != "" { + indexDocumentSuffix := XmlTranscoding(input.IndexDocument.Suffix) + xml = append(xml, fmt.Sprintf("%s", indexDocumentSuffix)) + } if input.ErrorDocument.Key != "" { errorDocumentKey := XmlTranscoding(input.ErrorDocument.Key) xml = append(xml, fmt.Sprintf("%s", errorDocumentKey)) } if routingRuleLength > 0 { xml = append(xml, "") - for _, routingRule := range input.RoutingRules { - xml = append(xml, "") - xml = append(xml, "") - if routingRule.Redirect.Protocol != "" { - xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.Protocol)) - } - if routingRule.Redirect.HostName != "" { - xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HostName)) - } - if routingRule.Redirect.ReplaceKeyPrefixWith != "" { - replaceKeyPrefixWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyPrefixWith) - xml = append(xml, fmt.Sprintf("%s", replaceKeyPrefixWith)) - } - - if routingRule.Redirect.ReplaceKeyWith != "" { - replaceKeyWith := XmlTranscoding(routingRule.Redirect.ReplaceKeyWith) - xml = append(xml, fmt.Sprintf("%s", replaceKeyWith)) - } - if routingRule.Redirect.HttpRedirectCode != "" { - xml = append(xml, fmt.Sprintf("%s", routingRule.Redirect.HttpRedirectCode)) - } - xml = append(xml, "") - - if ret := convertConditionToXml(routingRule.Condition); ret != "" { - xml = append(xml, ret) - } - xml = append(xml, "") - } + xml = append(xml, prepareRoutingRule(input)) xml = append(xml, "") } } @@ -259,11 +310,11 @@ func convertTransitionsToXml(transitions []Transition, isObs bool) string { } if temp != "" { if !isObs { - storageClass := "STANDARD" - if transition.StorageClass == "WARM" { - storageClass = "STANDARD_IA" - } else if transition.StorageClass == "COLD" { - storageClass = "GLACIER" + storageClass := string(transition.StorageClass) + if transition.StorageClass == StorageClassWarm { + storageClass = string(storageClassStandardIA) + } else if transition.StorageClass == StorageClassCold { + storageClass = string(storageClassGlacier) } xml = append(xml, fmt.Sprintf("%s%s", temp, storageClass)) } else { @@ -291,10 +342,10 @@ func convertNoncurrentVersionTransitionsToXml(noncurrentVersionTransitions []Non if noncurrentVersionTransition.NoncurrentDays > 0 { storageClass := string(noncurrentVersionTransition.StorageClass) if !isObs { - if storageClass == "WARM" { - storageClass = "STANDARD_IA" - } else if storageClass == "COLD" { - storageClass = "GLACIER" + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) } } xml = append(xml, fmt.Sprintf("%d"+ @@ -364,9 +415,8 @@ func converntFilterRulesToXml(filterRules []FilterRule, isObs bool) string { } if !isObs { return fmt.Sprintf("%s", strings.Join(xml, "")) - } else { - return fmt.Sprintf("%s", strings.Join(xml, "")) } + return fmt.Sprintf("%s", strings.Join(xml, "")) } return "" } @@ -404,7 +454,8 @@ func converntConfigureToXml(topicConfiguration TopicConfiguration, xmlElem strin if ret := converntFilterRulesToXml(topicConfiguration.FilterRules, isObs); ret != "" { xml = append(xml, ret) } - xml = append(xml, xmlElem) + tempElem := xmlElem[0:1] + "/" + xmlElem[1:] + xml = append(xml, tempElem) return strings.Join(xml, "") } @@ -470,7 +521,26 @@ func parseSseHeader(responseHeaders map[string][]string) (sseHeader ISseHeader) return } -func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { +func parseCorsHeader(output BaseModel) (AllowOrigin, AllowHeader, AllowMethod, ExposeHeader string, MaxAgeSeconds int) { + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { + AllowOrigin = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { + AllowHeader = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { + MaxAgeSeconds = StringToInt(ret[0], 0) + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { + AllowMethod = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { + ExposeHeader = ret[0] + } + return +} + +func parseUnCommonHeader(output *GetObjectMetadataOutput) { if ret, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { output.VersionId = ret[0] } @@ -484,11 +554,16 @@ func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { output.Restore = ret[0] } if ret, ok := output.ResponseHeaders[HEADER_OBJECT_TYPE]; ok { - output.Restore = ret[0] + output.ObjectType = ret[0] } if ret, ok := output.ResponseHeaders[HEADER_NEXT_APPEND_POSITION]; ok { - output.Restore = ret[0] + output.NextAppendPosition = ret[0] } +} + +func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { + output.AllowOrigin, output.AllowHeader, output.AllowMethod, output.ExposeHeader, output.MaxAgeSeconds = parseCorsHeader(output.BaseModel) + parseUnCommonHeader(output) if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { output.StorageClass = ParseStringToStorageClassType(ret[0]) } @@ -498,21 +573,6 @@ func ParseGetObjectMetadataOutput(output *GetObjectMetadataOutput) { if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { output.ContentType = ret[0] } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { - output.AllowOrigin = ret[0] - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { - output.AllowHeader = ret[0] - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { - output.MaxAgeSeconds = StringToInt(ret[0], 0) - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { - output.AllowMethod = ret[0] - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { - output.ExposeHeader = ret[0] - } output.SseHeader = parseSseHeader(output.ResponseHeaders) if ret, ok := output.ResponseHeaders[HEADER_LASTMODIFIED]; ok { @@ -584,9 +644,10 @@ func ParseCopyPartOutput(output *CopyPartOutput) { } func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { + output.AllowOrigin, output.AllowHeader, output.AllowMethod, output.ExposeHeader, output.MaxAgeSeconds = parseCorsHeader(output.BaseModel) if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { output.StorageClass = ParseStringToStorageClassType(ret[0]) - } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS_OBS]; ok { + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { output.StorageClass = ParseStringToStorageClassType(ret[0]) } if ret, ok := output.ResponseHeaders[HEADER_VERSION_OBS]; ok { @@ -597,26 +658,56 @@ func ParseGetBucketMetadataOutput(output *GetBucketMetadataOutput) { } else if ret, ok := output.ResponseHeaders[HEADER_BUCKET_LOCATION_OBS]; ok { output.Location = ret[0] } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_ORIGIN]; ok { - output.AllowOrigin = ret[0] - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_HEADERS]; ok { - output.AllowHeader = ret[0] - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_MAX_AGE]; ok { - output.MaxAgeSeconds = StringToInt(ret[0], 0) - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_ALLOW_METHODS]; ok { - output.AllowMethod = ret[0] - } - if ret, ok := output.ResponseHeaders[HEADER_ACCESS_CONRTOL_EXPOSE_HEADERS]; ok { - output.ExposeHeader = ret[0] - } if ret, ok := output.ResponseHeaders[HEADER_EPID_HEADERS]; ok { output.Epid = ret[0] } } +func parseContentHeader(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_DISPOSITION]; ok { + output.ContentDisposition = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_ENCODING]; ok { + output.ContentEncoding = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_LANGUAGE]; ok { + output.ContentLanguage = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_CONTENT_TYPE]; ok { + output.ContentType = ret[0] + } +} + +func ParseSetObjectMetadataOutput(output *SetObjectMetadataOutput) { + if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } else if ret, ok := output.ResponseHeaders[HEADER_STORAGE_CLASS2]; ok { + output.StorageClass = ParseStringToStorageClassType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_METADATA_DIRECTIVE]; ok { + output.MetadataDirective = MetadataDirectiveType(ret[0]) + } + if ret, ok := output.ResponseHeaders[HEADER_CACHE_CONTROL]; ok { + output.CacheControl = ret[0] + } + parseContentHeader(output) + if ret, ok := output.ResponseHeaders[HEADER_EXPIRES]; ok { + output.Expires = ret[0] + } + if ret, ok := output.ResponseHeaders[HEADER_WEBSITE_REDIRECT_LOCATION]; ok { + output.WebsiteRedirectLocation = ret[0] + } + output.Metadata = make(map[string]string) + + for key, value := range output.ResponseHeaders { + if strings.HasPrefix(key, PREFIX_META) { + _key := key[len(PREFIX_META):] + output.ResponseHeaders[_key] = value + output.Metadata[_key] = value[0] + delete(output.ResponseHeaders, key) + } + } +} func ParseDeleteObjectOutput(output *DeleteObjectOutput) { if versionId, ok := output.ResponseHeaders[HEADER_VERSION_ID]; ok { output.VersionId = versionId[0] @@ -674,7 +765,12 @@ func ConvertRequestToIoReader(req interface{}) (io.Reader, error) { func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResult bool, isObs bool) (err error) { readCloser, ok := baseModel.(IReadCloser) if !ok { - defer resp.Body.Close() + defer func() { + errMsg := resp.Body.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close response with reason: %v", errMsg) + } + }() body, err := ioutil.ReadAll(resp.Body) if err == nil && len(body) > 0 { if xmlResult { @@ -685,7 +781,7 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu } else { s := reflect.TypeOf(baseModel).Elem() for i := 0; i < s.NumField(); i++ { - if s.Field(i).Tag == "body" { + if s.Field(i).Tag == "json:\"body\"" { reflect.ValueOf(baseModel).Elem().FieldByName(s.Field(i).Name).SetString(string(body)) break } @@ -697,7 +793,7 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu } baseModel.setStatusCode(resp.StatusCode) - responseHeaders := cleanHeaderPrefix(resp.Header, isObs) + responseHeaders := cleanHeaderPrefix(resp.Header) baseModel.setResponseHeaders(responseHeaders) if values, ok := responseHeaders[HEADER_REQUEST_ID]; ok { baseModel.setRequestId(values[0]) @@ -707,7 +803,10 @@ func ParseResponseToBaseModel(resp *http.Response, baseModel IBaseModel, xmlResu func ParseResponseToObsError(resp *http.Response, isObs bool) error { obsError := ObsError{} - ParseResponseToBaseModel(resp, &obsError, true, isObs) + respError := ParseResponseToBaseModel(resp, &obsError, true, isObs) + if respError != nil { + doLog(LEVEL_WARN, "Parse response to BaseModel with error: %v", respError) + } obsError.Status = resp.Status return obsError } diff --git a/pkg/multicloud/huawei/obs/error.go b/pkg/multicloud/huawei/obs/error.go index 81ab8d7148..eff74e6115 100644 --- a/pkg/multicloud/huawei/obs/error.go +++ b/pkg/multicloud/huawei/obs/error.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs diff --git a/pkg/multicloud/huawei/obs/http.go b/pkg/multicloud/huawei/obs/http.go index 35d99e9676..338328f6ff 100644 --- a/pkg/multicloud/huawei/obs/http.go +++ b/pkg/multicloud/huawei/obs/http.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -59,14 +57,14 @@ func (obsClient ObsClient) doActionWithoutBucket(action, method string, input IS } func (obsClient ObsClient) doActionWithBucketV2(action, method, bucketName string, input ISerializable, output IBaseModel) error { - if strings.TrimSpace(bucketName) == "" { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { return errors.New("Bucket is empty") } return obsClient.doAction(action, method, bucketName, "", input, output, false, true) } func (obsClient ObsClient) doActionWithBucket(action, method, bucketName string, input ISerializable, output IBaseModel) error { - if strings.TrimSpace(bucketName) == "" { + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { return errors.New("Bucket is empty") } return obsClient.doAction(action, method, bucketName, "", input, output, true, true) @@ -81,8 +79,8 @@ func (obsClient ObsClient) doActionWithBucketAndKeyUnRepeatable(action, method, } func (obsClient ObsClient) _doActionWithBucketAndKey(action, method, bucketName, objectKey string, input ISerializable, output IBaseModel, repeatable bool) error { - if strings.TrimSpace(bucketName) == "" { - return errors.New("Key is empty") + if strings.TrimSpace(bucketName) == "" && !obsClient.conf.cname { + return errors.New("Bucket is empty") } if strings.TrimSpace(objectKey) == "" { return errors.New("Key is empty") @@ -97,8 +95,10 @@ func (obsClient ObsClient) doAction(action, method, bucketName, objectKey string doLog(LEVEL_INFO, "Enter method %s...", action) start := GetCurrentTimestamp() - params, headers, data := input.trans(obsClient.conf.signature == SignatureObs) - + params, headers, data, err := input.trans(obsClient.conf.signature == SignatureObs) + if err != nil { + return err + } if params == nil { params = make(map[string]string) } @@ -246,6 +246,7 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m var redirectUrl string var requestUrl string maxRetryCount := obsClient.conf.maxRetryCount + maxRedirectCount := obsClient.conf.maxRedirectCount var _data io.Reader if data != nil { @@ -265,15 +266,24 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } } - for i := 0; i <= maxRetryCount; i++ { + var lastRequest *http.Request + redirectFlag := false + for i, redirectCount := 0, 0; i <= maxRetryCount; i++ { if redirectUrl != "" { - parsedRedirectUrl, err := url.Parse(redirectUrl) - if err != nil { - return nil, err - } - requestUrl, _ = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectUrl.Host) - if parsedRequestUrl, _ := url.Parse(requestUrl); parsedRequestUrl.RawQuery != "" && parsedRedirectUrl.RawQuery == "" { - redirectUrl += "?" + parsedRequestUrl.RawQuery + if !redirectFlag { + parsedRedirectUrl, err := url.Parse(redirectUrl) + if err != nil { + return nil, err + } + requestUrl, err = obsClient.doAuth(method, bucketName, objectKey, params, headers, parsedRedirectUrl.Host) + if err != nil { + return nil, err + } + if parsedRequestUrl, err := url.Parse(requestUrl); err != nil { + return nil, err + } else if parsedRequestUrl.RawQuery != "" && parsedRedirectUrl.RawQuery == "" { + redirectUrl += "?" + parsedRequestUrl.RawQuery + } } requestUrl = redirectUrl } else { @@ -285,12 +295,12 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } req, err := http.NewRequest(method, requestUrl, _data) - if obsClient.conf.ctx != nil { - req = req.WithContext(obsClient.conf.ctx) - } if err != nil { return nil, err } + if obsClient.conf.ctx != nil { + req = req.WithContext(obsClient.conf.ctx) + } doLog(LEVEL_DEBUG, "Do request with url [%s] and method [%s]", requestUrl, method) if isDebugLogEnabled() { @@ -312,8 +322,15 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } } + lastRequest = req + req.Header[HEADER_USER_AGENT_CAMEL] = []string{USER_AGENT} + if lastRequest != nil { + req.Host = lastRequest.Host + req.ContentLength = lastRequest.ContentLength + } + start := GetCurrentTimestamp() resp, err = obsClient.httpClient.Do(req) if isInfoLogEnabled() { @@ -325,6 +342,9 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m msg = err respError = err resp = nil + if !repeatable { + break + } } else { doLog(LEVEL_DEBUG, "Response headers: %v", resp.Header) if resp.StatusCode < 300 { @@ -334,11 +354,17 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m resp = nil break } else if resp.StatusCode >= 300 && resp.StatusCode < 400 { - if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" { + if location := resp.Header.Get(HEADER_LOCATION_CAMEL); location != "" && redirectCount < maxRedirectCount { redirectUrl = location doLog(LEVEL_WARN, "Redirect request to %s", redirectUrl) msg = resp.Status maxRetryCount++ + redirectCount++ + if resp.StatusCode == 302 && method == HTTP_GET { + redirectFlag = true + } else { + redirectFlag = false + } } else { respError = ParseResponseToObsError(resp, obsClient.conf.signature == SignatureObs) resp = nil @@ -350,7 +376,10 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } if i != maxRetryCount { if resp != nil { - resp.Body.Close() + _err := resp.Body.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close resp body with reason: %v", _err) + } resp = nil } if _, ok := headers[HEADER_AUTH_CAMEL]; ok { @@ -358,23 +387,40 @@ func (obsClient ObsClient) doHttp(method, bucketName, objectKey string, params m } doLog(LEVEL_WARN, "Failed to send request with reason:%v, will try again", msg) if r, ok := _data.(*strings.Reader); ok { - r.Seek(0, 0) + _, err := r.Seek(0, 0) + if err != nil { + return nil, err + } } else if r, ok := _data.(*bytes.Reader); ok { - r.Seek(0, 0) + _, err := r.Seek(0, 0) + if err != nil { + return nil, err + } } else if r, ok := _data.(*fileReaderWrapper); ok { fd, err := os.Open(r.filePath) if err != nil { return nil, err } - defer fd.Close() + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close with reason: %v", errMsg) + } + }() fileReaderWrapper := &fileReaderWrapper{filePath: r.filePath} fileReaderWrapper.mark = r.mark fileReaderWrapper.reader = fd fileReaderWrapper.totalCount = r.totalCount _data = fileReaderWrapper - fd.Seek(r.mark, 0) + _, err = fd.Seek(r.mark, 0) + if err != nil { + return nil, err + } } else if r, ok := _data.(*readerWrapper); ok { - r.seek(0, 0) + _, err := r.seek(0, 0) + if err != nil { + return nil, err + } } time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second))) } else { @@ -403,18 +449,38 @@ func getConnDelegate(conn net.Conn, socketTimeout int, finalTimeout int) *connDe } func (delegate *connDelegate) Read(b []byte) (n int, err error) { - delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + setReadDeadlineErr := delegate.SetReadDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } + n, err = delegate.conn.Read(b) - delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + setReadDeadlineErr = delegate.SetReadDeadline(time.Now().Add(delegate.finalTimeout)) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } return n, err } func (delegate *connDelegate) Write(b []byte) (n int, err error) { - delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + setWriteDeadlineErr := delegate.SetWriteDeadline(time.Now().Add(delegate.socketTimeout)) + flag := isDebugLogEnabled() + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + n, err = delegate.conn.Write(b) finalTimeout := time.Now().Add(delegate.finalTimeout) - delegate.SetWriteDeadline(finalTimeout) - delegate.SetReadDeadline(finalTimeout) + setWriteDeadlineErr = delegate.SetWriteDeadline(finalTimeout) + if setWriteDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set write deadline with reason: %v, but it's ok", setWriteDeadlineErr) + } + setReadDeadlineErr := delegate.SetReadDeadline(finalTimeout) + if setReadDeadlineErr != nil && flag { + doLog(LEVEL_DEBUG, "Failed to set read deadline with reason: %v, but it's ok", setReadDeadlineErr) + } return n, err } diff --git a/pkg/multicloud/huawei/obs/log.go b/pkg/multicloud/huawei/obs/log.go index 43a8ef1899..a467cc0e33 100644 --- a/pkg/multicloud/huawei/obs/log.go +++ b/pkg/multicloud/huawei/obs/log.go @@ -1,21 +1,18 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs import ( - "errors" "fmt" "log" "os" @@ -86,19 +83,31 @@ func (lw *loggerWrapper) doInit() { func (lw *loggerWrapper) rotate() { stat, err := lw.fd.Stat() if err != nil { - lw.fd.Close() + _err := lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } panic(err) } if stat.Size() >= logConf.maxLogSize { - lw.fd.Sync() - lw.fd.Close() + _err := lw.fd.Sync() + if _err != nil { + panic(err) + } + _err = lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } if lw.index > logConf.backups { lw.index = 1 } - os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) - lw.index += 1 + _err = os.Rename(lw.fullPath, lw.fullPath+"."+IntToString(lw.index)) + if _err != nil { + panic(err) + } + lw.index++ - fd, err := os.OpenFile(lw.fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + fd, err := os.OpenFile(lw.fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { panic(err) } @@ -112,7 +121,10 @@ func (lw *loggerWrapper) doFlush() { for _, m := range lw.queue { lw.logger.Println(m) } - lw.fd.Sync() + err := lw.fd.Sync() + if err != nil { + panic(err) + } } func (lw *loggerWrapper) doClose() { @@ -127,7 +139,10 @@ func (lw *loggerWrapper) doWrite() { msg, ok := <-lw.ch if !ok { lw.doFlush() - lw.fd.Close() + _err := lw.fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } break } if len(lw.queue) >= lw.cacheCount { @@ -148,7 +163,7 @@ func (lw *loggerWrapper) Printf(format string, v ...interface{}) { var consoleLogger *log.Logger var fileLogger *loggerWrapper -var lock *sync.RWMutex = new(sync.RWMutex) +var lock = new(sync.RWMutex) func isDebugLogEnabled() bool { return logConf.level <= LEVEL_DEBUG @@ -198,12 +213,12 @@ func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, leve stat, err := os.Stat(_fullPath) if err == nil && stat.IsDir() { - return errors.New(fmt.Sprintf("logFullPath:[%s] is a directory", _fullPath)) - } else if err := os.MkdirAll(filepath.Dir(_fullPath), os.ModePerm); err != nil { + return fmt.Errorf("logFullPath:[%s] is a directory", _fullPath) + } else if err = os.MkdirAll(filepath.Dir(_fullPath), os.ModePerm); err != nil { return err } - fd, err := os.OpenFile(_fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + fd, err := os.OpenFile(_fullPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { return err } @@ -211,7 +226,10 @@ func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, leve if stat == nil { stat, err = os.Stat(_fullPath) if err != nil { - fd.Close() + _err := fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } return err } } @@ -230,7 +248,10 @@ func InitLogWithCacheCnt(logFullPath string, maxLogSize int64, backups int, leve } if err = filepath.Walk(filepath.Dir(_fullPath), walkFunc); err != nil { - fd.Close() + _err := fd.Close() + if _err != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", _err) + } return err } diff --git a/pkg/multicloud/huawei/obs/model.go b/pkg/multicloud/huawei/obs/model.go index 405faefc38..04596657bb 100644 --- a/pkg/multicloud/huawei/obs/model.go +++ b/pkg/multicloud/huawei/obs/model.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -292,9 +290,10 @@ type grantObs struct { } type AccessControlPolicy struct { - XMLName xml.Name `xml:"AccessControlPolicy"` - Owner Owner `xml:"Owner"` - Grants []Grant `xml:"AccessControlList>Grant"` + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + Grants []Grant `xml:"AccessControlList>Grant"` + Delivered string `xml:"Delivered,omitempty"` } type accessControlPolicyObs struct { @@ -326,7 +325,7 @@ type SetBucketPolicyInput struct { type GetBucketPolicyOutput struct { BaseModel - Policy string + Policy string `json:"body"` } type CorsRule struct { @@ -428,6 +427,36 @@ type GetBucketMetadataInput struct { RequestHeader string } +type SetObjectMetadataInput struct { + Bucket string + Key string + VersionId string + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + +type SetObjectMetadataOutput struct { + BaseModel + MetadataDirective MetadataDirectiveType + CacheControl string + ContentDisposition string + ContentEncoding string + ContentLanguage string + ContentType string + Expires string + WebsiteRedirectLocation string + StorageClass StorageClassType + Metadata map[string]string +} + type GetBucketMetadataOutput struct { BaseModel StorageClass StorageClassType @@ -786,7 +815,7 @@ type CopyObjectInput struct { ContentType string Expires string MetadataDirective MetadataDirectiveType - successActionRedirect string + SuccessActionRedirect string } type CopyObjectOutput struct { diff --git a/pkg/multicloud/huawei/obs/temporary.go b/pkg/multicloud/huawei/obs/temporary.go index 4505cf11e3..fbb0e94e26 100644 --- a/pkg/multicloud/huawei/obs/temporary.go +++ b/pkg/multicloud/huawei/obs/temporary.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -59,6 +57,16 @@ func (obsClient ObsClient) CreateSignedUrl(input *CreateSignedUrlInput) (output return } +func (obsClient ObsClient) isSecurityToken(params map[string]string) { + if obsClient.conf.securityProvider.securityToken != "" { + if obsClient.conf.signature == SignatureObs { + params[HEADER_STS_TOKEN_OBS] = obsClient.conf.securityProvider.securityToken + } else { + params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + } + } +} + func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBasedSignatureInput) (output *CreateBrowserBasedSignatureOutput, err error) { if input == nil { return nil, errors.New("CreateBrowserBasedSignatureInput is nil") @@ -80,14 +88,14 @@ func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBased } expiration := date.Add(time.Second * time.Duration(input.Expires)).Format(ISO8601_DATE_FORMAT) - params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX - params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential - params[PARAM_DATE_AMZ_CAMEL] = longDate - - if obsClient.conf.securityProvider.securityToken != "" { - params[HEADER_STS_TOKEN_AMZ] = obsClient.conf.securityProvider.securityToken + if obsClient.conf.signature == SignatureV4 { + params[PARAM_ALGORITHM_AMZ_CAMEL] = V4_HASH_PREFIX + params[PARAM_CREDENTIAL_AMZ_CAMEL] = credential + params[PARAM_DATE_AMZ_CAMEL] = longDate } + obsClient.isSecurityToken(params) + matchAnyBucket := true matchAnyKey := true count := 5 @@ -124,7 +132,12 @@ func (obsClient ObsClient) CreateBrowserBasedSignature(input *CreateBrowserBased originPolicy := strings.Join(originPolicySlice, "") policy := Base64Encode([]byte(originPolicy)) - signature := getSignature(policy, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) + var signature string + if obsClient.conf.signature == SignatureV4 { + signature = getSignature(policy, obsClient.conf.securityProvider.sk, obsClient.conf.region, shortDate) + } else { + signature = Base64Encode(HmacSha1([]byte(obsClient.conf.securityProvider.sk), []byte(policy))) + } output = &CreateBrowserBasedSignatureOutput{ OriginPolicy: originPolicy, @@ -568,14 +581,21 @@ func (obsClient ObsClient) PutFileWithSignedUrl(signedUrl string, actualSignedRe var data io.Reader sourceFile = strings.TrimSpace(sourceFile) if sourceFile != "" { - fd, err := os.Open(sourceFile) - if err != nil { + fd, _err := os.Open(sourceFile) + if _err != nil { + err = _err return nil, err } - defer fd.Close() + defer func() { + errMsg := fd.Close() + if errMsg != nil { + doLog(LEVEL_WARN, "Failed to close file with reason: %v", errMsg) + } + }() - stat, err := fd.Stat() - if err != nil { + stat, _err := fd.Stat() + if _err != nil { + err = _err return nil, err } fileReaderWrapper := &fileReaderWrapper{filePath: sourceFile} diff --git a/pkg/multicloud/huawei/obs/trait.go b/pkg/multicloud/huawei/obs/trait.go index a921c9c2c6..f47f46f4e1 100644 --- a/pkg/multicloud/huawei/obs/trait.go +++ b/pkg/multicloud/huawei/obs/trait.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -57,7 +55,7 @@ type IBaseModel interface { } type ISerializable interface { - trans(isObs bool) (map[string]string, map[string][]string, interface{}) + trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) } type DefaultSerializable struct { @@ -66,8 +64,8 @@ type DefaultSerializable struct { data interface{} } -func (input DefaultSerializable) trans(isObs bool) (map[string]string, map[string][]string, interface{}) { - return input.params, input.headers, input.data +func (input DefaultSerializable) trans(isObs bool) (map[string]string, map[string][]string, interface{}, error) { + return input.params, input.headers, input.data, nil } var defaultSerializable = &DefaultSerializable{} @@ -76,9 +74,9 @@ func newSubResourceSerial(subResource SubResourceType) *DefaultSerializable { return &DefaultSerializable{map[string]string{string(subResource): ""}, nil, nil} } -func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}) { +func trans(subResource SubResourceType, input interface{}) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(subResource): ""} - data, _ = ConvertRequestToIoReader(input) + data, err = ConvertRequestToIoReader(input) return } @@ -94,7 +92,7 @@ func (baseModel *BaseModel) setResponseHeaders(responseHeaders map[string][]stri baseModel.ResponseHeaders = responseHeaders } -func (input ListBucketsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListBucketsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) if input.QueryLocation && !isObs { setHeaders(headers, HEADER_LOCATION_AMZ, []string{"true"}, isObs) @@ -102,63 +100,81 @@ func (input ListBucketsInput) trans(isObs bool) (params map[string]string, heade return } -func (input CreateBucketInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input CreateBucketInput) prepareGrantHeaders(headers map[string][]string, isObs bool) { + if grantReadId := input.GrantReadId; grantReadId != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{grantReadId}, isObs) + } + if grantWriteId := input.GrantWriteId; grantWriteId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_OBS, []string{grantWriteId}, isObs) + } + if grantReadAcpId := input.GrantReadAcpId; grantReadAcpId != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{grantReadAcpId}, isObs) + } + if grantWriteAcpId := input.GrantWriteAcpId; grantWriteAcpId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{grantWriteAcpId}, isObs) + } + if grantFullControlId := input.GrantFullControlId; grantFullControlId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{grantFullControlId}, isObs) + } + if grantReadDeliveredId := input.GrantReadDeliveredId; grantReadDeliveredId != "" { + setHeaders(headers, HEADER_GRANT_READ_DELIVERED_OBS, []string{grantReadDeliveredId}, true) + } + if grantFullControlDeliveredId := input.GrantFullControlDeliveredId; grantFullControlDeliveredId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS, []string{grantFullControlDeliveredId}, true) + } +} + +func (input CreateBucketInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) if acl := string(input.ACL); acl != "" { setHeaders(headers, HEADER_ACL, []string{acl}, isObs) } if storageClass := string(input.StorageClass); storageClass != "" { if !isObs { - if storageClass == "WARM" { - storageClass = "STANDARD_IA" - } else if storageClass == "COLD" { - storageClass = "GLACIER" + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) } } setHeadersNext(headers, HEADER_STORAGE_CLASS_OBS, HEADER_STORAGE_CLASS, []string{storageClass}, isObs) - if epid := string(input.Epid); epid != "" { + if epid := input.Epid; epid != "" { setHeaders(headers, HEADER_EPID_HEADERS, []string{epid}, isObs) } } - if grantReadId := string(input.GrantReadId); grantReadId != "" { - setHeaders(headers, HEADER_GRANT_READ_OBS, []string{grantReadId}, isObs) - } - if grantWriteId := string(input.GrantWriteId); grantWriteId != "" { - setHeaders(headers, HEADER_GRANT_WRITE_OBS, []string{grantWriteId}, isObs) - } - if grantReadAcpId := string(input.GrantReadAcpId); grantReadAcpId != "" { - setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{grantReadAcpId}, isObs) - } - if grantWriteAcpId := string(input.GrantWriteAcpId); grantWriteAcpId != "" { - setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{grantWriteAcpId}, isObs) - } - if grantFullControlId := string(input.GrantFullControlId); grantFullControlId != "" { - setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{grantFullControlId}, isObs) - } - if grantReadDeliveredId := string(input.GrantReadDeliveredId); grantReadDeliveredId != "" { - setHeaders(headers, HEADER_GRANT_READ_DELIVERED_OBS, []string{grantReadDeliveredId}, true) - } - if grantFullControlDeliveredId := string(input.GrantFullControlDeliveredId); grantFullControlDeliveredId != "" { - setHeaders(headers, HEADER_GRANT_FULL_CONTROL_DELIVERED_OBS, []string{grantFullControlDeliveredId}, true) - } + input.prepareGrantHeaders(headers, isObs) if location := strings.TrimSpace(input.Location); location != "" { input.Location = location - data, _ = ConvertRequestToIoReader(input) + + xml := make([]string, 0, 3) + xml = append(xml, "") + if isObs { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } else { + xml = append(xml, fmt.Sprintf("%s", input.Location)) + } + xml = append(xml, "") + + data = strings.Join(xml, "") } return } -func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { xml := make([]string, 0, 1) if !isObs { storageClass := "STANDARD" - if input.StorageClass == "WARM" { - storageClass = "STANDARD_IA" - } else if input.StorageClass == "COLD" { - storageClass = "GLACIER" + if input.StorageClass == StorageClassWarm { + storageClass = string(storageClassStandardIA) + } else if input.StorageClass == StorageClassCold { + storageClass = string(storageClassGlacier) } - xml = append(xml, fmt.Sprintf("%s", storageClass)) + params = map[string]string{string(SubResourceStoragePolicy): ""} + xml = append(xml, fmt.Sprintf("%s", storageClass)) } else { + if input.StorageClass != StorageClassWarm && input.StorageClass != StorageClassCold { + input.StorageClass = StorageClassStandard + } params = map[string]string{string(SubResourceStorageClass): ""} xml = append(xml, fmt.Sprintf("%s", input.StorageClass)) } @@ -166,7 +182,7 @@ func (input SetBucketStoragePolicyInput) trans(isObs bool) (params map[string]st return } -func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = make(map[string]string) if input.Prefix != "" { params["prefix"] = input.Prefix @@ -187,16 +203,22 @@ func (input ListObjsInput) trans(isObs bool) (params map[string]string, headers return } -func (input ListObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ListObjsInput.trans(isObs) +func (input ListObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } if input.Marker != "" { params["marker"] = input.Marker } return } -func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ListObjsInput.trans(isObs) +func (input ListVersionsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ListObjsInput.trans(isObs) + if err != nil { + return + } params[string(SubResourceVersions)] = "" if input.KeyMarker != "" { params["key-marker"] = input.KeyMarker @@ -207,7 +229,7 @@ func (input ListVersionsInput) trans(isObs bool) (params map[string]string, head return } -func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceUploads): ""} if input.Prefix != "" { params["prefix"] = input.Prefix @@ -227,46 +249,49 @@ func (input ListMultipartUploadsInput) trans(isObs bool) (params map[string]stri return } -func (input SetBucketQuotaInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketQuotaInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { return trans(SubResourceQuota, input) } -func (input SetBucketAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceAcl): ""} headers = make(map[string][]string) if acl := string(input.ACL); acl != "" { setHeaders(headers, HEADER_ACL, []string{acl}, isObs) } else { - data, _ = ConvertAclToXml(input.AccessControlPolicy, false, isObs) + data, _ = convertBucketAclToXml(input.AccessControlPolicy, false, isObs) } return } -func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketPolicyInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourcePolicy): ""} data = strings.NewReader(input.Policy) return } -func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketCorsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceCors): ""} - data, md5, _ := ConvertRequestToIoReaderV2(input) - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetBucketVersioningInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketVersioningInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { return trans(SubResourceVersioning, input) } -func (input SetBucketWebsiteConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketWebsiteConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceWebsite): ""} data, _ = ConvertWebsiteConfigurationToXml(input.BucketWebsiteConfiguration, false) return } -func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) if origin := strings.TrimSpace(input.Origin); origin != "" { headers[HEADER_ORIGIN_CAMEL] = []string{origin} @@ -277,33 +302,36 @@ func (input GetBucketMetadataInput) trans(isObs bool) (params map[string]string, return } -func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketLoggingConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceLogging): ""} data, _ = ConvertLoggingStatusToXml(input.BucketLoggingStatus, false, isObs) return } -func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketLifecycleConfigurationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceLifecycle): ""} data, md5 := ConvertLifecyleConfigurationToXml(input.BucketLifecyleConfiguration, true, isObs) - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketTaggingInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceTagging): ""} - data, md5, _ := ConvertRequestToIoReaderV2(input) - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetBucketNotificationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetBucketNotificationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceNotification): ""} data, _ = ConvertNotificationToXml(input.BucketNotification, false, isObs) return } -func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = make(map[string]string) if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -311,14 +339,17 @@ func (input DeleteObjectInput) trans(isObs bool) (params map[string]string, head return } -func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input DeleteObjectsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceDelete): ""} - data, md5, _ := ConvertRequestToIoReaderV2(input) - headers = map[string][]string{HEADER_MD5_CAMEL: {md5}} + data, md5, err := ConvertRequestToIoReaderV2(input) + if err != nil { + return + } + headers = map[string][]string{HEADER_MD5_CAMEL: []string{md5}} return } -func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceAcl): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -332,7 +363,7 @@ func (input SetObjectAclInput) trans(isObs bool) (params map[string]string, head return } -func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceAcl): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -340,13 +371,13 @@ func (input GetObjectAclInput) trans(isObs bool) (params map[string]string, head return } -func (input RestoreObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input RestoreObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{string(SubResourceRestore): ""} if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId } if !isObs { - data, _ = ConvertRequestToIoReader(input) + data, err = ConvertRequestToIoReader(input) } else { data = ConverntObsRestoreToXml(input) } @@ -359,9 +390,8 @@ func (header SseKmsHeader) GetEncryption() string { } if !header.isObs { return DEFAULT_SSE_KMS_ENCRYPTION - } else { - return DEFAULT_SSE_KMS_ENCRYPTION_OBS } + return DEFAULT_SSE_KMS_ENCRYPTION_OBS } func (header SseKmsHeader) GetKey() string { @@ -404,7 +434,7 @@ func setSseHeader(headers map[string][]string, sseHeader ISseHeader, sseCOnly bo } } -func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = make(map[string]string) if input.VersionId != "" { params[PARAM_VERSION_ID] = input.VersionId @@ -422,8 +452,69 @@ func (input GetObjectMetadataInput) trans(isObs bool) (params map[string]string, return } -func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.GetObjectMetadataInput.trans(isObs) +func (input SetObjectMetadataInput) prepareContentHeaders(headers map[string][]string) { + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION_CAMEL] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING_CAMEL] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE_CAMEL] = []string{input.ContentLanguage} + } + + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } +} + +func (input SetObjectMetadataInput) prepareStorageClass(headers map[string][]string, isObs bool) { + if storageClass := string(input.StorageClass); storageClass != "" { + if !isObs { + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) + } + } + setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) + } +} + +func (input SetObjectMetadataInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params = make(map[string]string) + params = map[string]string{string(SubResourceMetadata): ""} + if input.VersionId != "" { + params[PARAM_VERSION_ID] = input.VersionId + } + headers = make(map[string][]string) + + if directive := string(input.MetadataDirective); directive != "" { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(input.MetadataDirective)}, isObs) + } else { + setHeaders(headers, HEADER_METADATA_DIRECTIVE, []string{string(ReplaceNew)}, isObs) + } + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL_CAMEL] = []string{input.CacheControl} + } + input.prepareContentHeaders(headers) + if input.Expires != "" { + headers[HEADER_EXPIRES_CAMEL] = []string{input.Expires} + } + if input.WebsiteRedirectLocation != "" { + setHeaders(headers, HEADER_WEBSITE_REDIRECT_LOCATION, []string{input.WebsiteRedirectLocation}, isObs) + } + input.prepareStorageClass(headers, isObs) + if input.Metadata != nil { + for key, value := range input.Metadata { + key = strings.TrimSpace(key) + setHeadersNext(headers, HEADER_PREFIX_META_OBS+key, HEADER_PREFIX_META+key, []string{value}, isObs) + } + } + return +} + +func (input GetObjectInput) prepareResponseParams(params map[string]string) { if input.ResponseCacheControl != "" { params[PARAM_RESPONSE_CACHE_CONTROL] = input.ResponseCacheControl } @@ -442,6 +533,14 @@ func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers if input.ResponseExpires != "" { params[PARAM_RESPONSE_EXPIRES] = input.ResponseExpires } +} + +func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.GetObjectMetadataInput.trans(isObs) + if err != nil { + return + } + input.prepareResponseParams(params) if input.ImageProcess != "" { params[PARAM_IMAGE_PROCESS] = input.ImageProcess } @@ -464,30 +563,34 @@ func (input GetObjectInput) trans(isObs bool) (params map[string]string, headers return } -func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input ObjectOperationInput) prepareGrantHeaders(headers map[string][]string) { + if GrantReadId := input.GrantReadId; GrantReadId != "" { + setHeaders(headers, HEADER_GRANT_READ_OBS, []string{GrantReadId}, true) + } + if GrantReadAcpId := input.GrantReadAcpId; GrantReadAcpId != "" { + setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{GrantReadAcpId}, true) + } + if GrantWriteAcpId := input.GrantWriteAcpId; GrantWriteAcpId != "" { + setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{GrantWriteAcpId}, true) + } + if GrantFullControlId := input.GrantFullControlId; GrantFullControlId != "" { + setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{GrantFullControlId}, true) + } +} + +func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { headers = make(map[string][]string) params = make(map[string]string) if acl := string(input.ACL); acl != "" { setHeaders(headers, HEADER_ACL, []string{acl}, isObs) } - if GrantReadId := string(input.GrantReadId); GrantReadId != "" { - setHeaders(headers, HEADER_GRANT_READ_OBS, []string{GrantReadId}, true) - } - if GrantReadAcpId := string(input.GrantReadAcpId); GrantReadAcpId != "" { - setHeaders(headers, HEADER_GRANT_READ_ACP_OBS, []string{GrantReadAcpId}, true) - } - if GrantWriteAcpId := string(input.GrantWriteAcpId); GrantWriteAcpId != "" { - setHeaders(headers, HEADER_GRANT_WRITE_ACP_OBS, []string{GrantWriteAcpId}, true) - } - if GrantFullControlId := string(input.GrantFullControlId); GrantFullControlId != "" { - setHeaders(headers, HEADER_GRANT_FULL_CONTROL_OBS, []string{GrantFullControlId}, true) - } + input.prepareGrantHeaders(headers) if storageClass := string(input.StorageClass); storageClass != "" { if !isObs { - if storageClass == "WARM" { - storageClass = "STANDARD_IA" - } else if storageClass == "COLD" { - storageClass = "GLACIER" + if storageClass == string(StorageClassWarm) { + storageClass = string(storageClassStandardIA) + } else if storageClass == string(StorageClassCold) { + storageClass = string(storageClassGlacier) } } setHeaders(headers, HEADER_STORAGE_CLASS2, []string{storageClass}, isObs) @@ -498,7 +601,7 @@ func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, h } setSseHeader(headers, input.SseHeader, false, isObs) if input.Expires != 0 { - setHeaders(headers, HEADER_EXPIRES_OBS, []string{Int64ToString(input.Expires)}, true) + setHeaders(headers, HEADER_EXPIRES, []string{Int64ToString(input.Expires)}, true) } if input.Metadata != nil { for key, value := range input.Metadata { @@ -509,8 +612,11 @@ func (input ObjectOperationInput) trans(isObs bool) (params map[string]string, h return } -func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ObjectOperationInput.trans(isObs) +func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } if input.ContentMD5 != "" { headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} @@ -526,16 +632,58 @@ func (input PutObjectBasicInput) trans(isObs bool) (params map[string]string, he return } -func (input PutObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.PutObjectBasicInput.trans(isObs) +func (input PutObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.PutObjectBasicInput.trans(isObs) + if err != nil { + return + } if input.Body != nil { data = input.Body } return } -func (input CopyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ObjectOperationInput.trans(isObs) +func (input CopyObjectInput) prepareReplaceHeaders(headers map[string][]string) { + if input.CacheControl != "" { + headers[HEADER_CACHE_CONTROL] = []string{input.CacheControl} + } + if input.ContentDisposition != "" { + headers[HEADER_CONTENT_DISPOSITION] = []string{input.ContentDisposition} + } + if input.ContentEncoding != "" { + headers[HEADER_CONTENT_ENCODING] = []string{input.ContentEncoding} + } + if input.ContentLanguage != "" { + headers[HEADER_CONTENT_LANGUAGE] = []string{input.ContentLanguage} + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE] = []string{input.ContentType} + } + if input.Expires != "" { + headers[HEADER_EXPIRES] = []string{input.Expires} + } +} + +func (input CopyObjectInput) prepareCopySourceHeaders(headers map[string][]string, isObs bool) { + if input.CopySourceIfMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MATCH, []string{input.CopySourceIfMatch}, isObs) + } + if input.CopySourceIfNoneMatch != "" { + setHeaders(headers, HEADER_COPY_SOURCE_IF_NONE_MATCH, []string{input.CopySourceIfNoneMatch}, isObs) + } + if !input.CopySourceIfModifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_MODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}, isObs) + } + if !input.CopySourceIfUnmodifiedSince.IsZero() { + setHeaders(headers, HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}, isObs) + } +} + +func (input CopyObjectInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } var copySource string if input.CopySourceVersionId != "" { @@ -550,38 +698,10 @@ func (input CopyObjectInput) trans(isObs bool) (params map[string]string, header } if input.MetadataDirective == ReplaceMetadata { - if input.CacheControl != "" { - headers[HEADER_CACHE_CONTROL] = []string{input.CacheControl} - } - if input.ContentDisposition != "" { - headers[HEADER_CONTENT_DISPOSITION] = []string{input.ContentDisposition} - } - if input.ContentEncoding != "" { - headers[HEADER_CONTENT_ENCODING] = []string{input.ContentEncoding} - } - if input.ContentLanguage != "" { - headers[HEADER_CONTENT_LANGUAGE] = []string{input.ContentLanguage} - } - if input.ContentType != "" { - headers[HEADER_CONTENT_TYPE] = []string{input.ContentType} - } - if input.Expires != "" { - headers[HEADER_EXPIRES] = []string{input.Expires} - } + input.prepareReplaceHeaders(headers) } - if input.CopySourceIfMatch != "" { - setHeaders(headers, HEADER_COPY_SOURCE_IF_MATCH, []string{input.CopySourceIfMatch}, isObs) - } - if input.CopySourceIfNoneMatch != "" { - setHeaders(headers, HEADER_COPY_SOURCE_IF_NONE_MATCH, []string{input.CopySourceIfNoneMatch}, isObs) - } - if !input.CopySourceIfModifiedSince.IsZero() { - setHeaders(headers, HEADER_COPY_SOURCE_IF_MODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfModifiedSince)}, isObs) - } - if !input.CopySourceIfUnmodifiedSince.IsZero() { - setHeaders(headers, HEADER_COPY_SOURCE_IF_UNMODIFIED_SINCE, []string{FormatUtcToRfc1123(input.CopySourceIfUnmodifiedSince)}, isObs) - } + input.prepareCopySourceHeaders(headers, isObs) if input.SourceSseHeader != nil { if sseCHeader, ok := input.SourceSseHeader.(SseCHeader); ok { setHeaders(headers, HEADER_SSEC_COPY_SOURCE_ENCRYPTION, []string{sseCHeader.GetEncryption()}, isObs) @@ -589,40 +709,49 @@ func (input CopyObjectInput) trans(isObs bool) (params map[string]string, header setHeaders(headers, HEADER_SSEC_COPY_SOURCE_KEY_MD5, []string{sseCHeader.GetKeyMD5()}, isObs) } } - if input.successActionRedirect != "" { - headers[HEADER_SUCCESS_ACTION_REDIRECT] = []string{input.successActionRedirect} + if input.SuccessActionRedirect != "" { + headers[HEADER_SUCCESS_ACTION_REDIRECT] = []string{input.SuccessActionRedirect} } return } -func (input AbortMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input AbortMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId} return } -func (input InitiateMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { - params, headers, data = input.ObjectOperationInput.trans(isObs) +func (input InitiateMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { + params, headers, data, err = input.ObjectOperationInput.trans(isObs) + if err != nil { + return + } + if input.ContentType != "" { + headers[HEADER_CONTENT_TYPE_CAML] = []string{input.ContentType} + } params[string(SubResourceUploads)] = "" return } -func (input UploadPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input UploadPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} headers = make(map[string][]string) setSseHeader(headers, input.SseHeader, true, isObs) + if input.ContentMD5 != "" { + headers[HEADER_MD5_CAMEL] = []string{input.ContentMD5} + } if input.Body != nil { data = input.Body } return } -func (input CompleteMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input CompleteMultipartUploadInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId} data, _ = ConvertCompleteMultipartUploadInputToXml(input, false) return } -func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId} if input.MaxParts > 0 { params["max-parts"] = IntToString(input.MaxParts) @@ -633,7 +762,7 @@ func (input ListPartsInput) trans(isObs bool) (params map[string]string, headers return } -func (input CopyPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}) { +func (input CopyPartInput) trans(isObs bool) (params map[string]string, headers map[string][]string, data interface{}, err error) { params = map[string]string{"uploadId": input.UploadId, "partNumber": IntToString(input.PartNumber)} headers = make(map[string][]string, 1) var copySource string @@ -698,13 +827,13 @@ func (rw *readerWrapper) Read(p []byte) (n int, err error) { if rw.totalCount > 0 { n, err = rw.reader.Read(p) readedOnce := int64(n) - if remainCount := rw.totalCount - rw.readedCount; remainCount > readedOnce { + remainCount := rw.totalCount - rw.readedCount + if remainCount > readedOnce { rw.readedCount += readedOnce return n, err - } else { - rw.readedCount += remainCount - return int(remainCount), io.EOF } + rw.readedCount += remainCount + return int(remainCount), io.EOF } return rw.reader.Read(p) } diff --git a/pkg/multicloud/huawei/obs/util.go b/pkg/multicloud/huawei/obs/util.go index ce53ad8118..d868831c53 100644 --- a/pkg/multicloud/huawei/obs/util.go +++ b/pkg/multicloud/huawei/obs/util.go @@ -1,16 +1,14 @@ -// Copyright 2019 Yunion +// Copyright 2019 Huawei Technologies Co.,Ltd. +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 // -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. package obs @@ -85,19 +83,28 @@ func FormatUtcToRfc1123(t time.Time) string { func Md5(value []byte) []byte { m := md5.New() - m.Write(value) + _, err := m.Write(value) + if err != nil { + doLog(LEVEL_WARN, "MD5 failed to write with reason: %v", err) + } return m.Sum(nil) } func HmacSha1(key, value []byte) []byte { mac := hmac.New(sha1.New, key) - mac.Write(value) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha1 failed to write with reason: %v", err) + } return mac.Sum(nil) } func HmacSha256(key, value []byte) []byte { mac := hmac.New(sha256.New, key) - mac.Write(value) + _, err := mac.Write(value) + if err != nil { + doLog(LEVEL_WARN, "HmacSha256 failed to write with reason: %v", err) + } return mac.Sum(nil) } @@ -119,7 +126,10 @@ func Base64Md5(value []byte) string { func Sha256Hash(value []byte) []byte { hash := sha256.New() - hash.Write(value) + _, err := hash.Write(value) + if err != nil { + doLog(LEVEL_WARN, "Sha256Hash failed to write with reason: %v", err) + } return hash.Sum(nil) } @@ -153,6 +163,17 @@ func UrlDecode(value string) (string, error) { return "", err } +func UrlDecodeWithoutError(value string) string { + ret, err := UrlDecode(value) + if err == nil { + return ret + } + if isErrorLogEnabled() { + doLog(LEVEL_ERROR, "Url decode error: %v", err) + } + return "" +} + func IsIP(value string) bool { return ipRegex.MatchString(value) } @@ -243,6 +264,59 @@ func getIsObs(isTemporary bool, querys []string, headers map[string][]string) bo return isObs } +func isPathStyle(headers map[string][]string, bucketName string) bool { + if receviedHost, ok := headers[HEADER_HOST]; ok && len(receviedHost) > 0 && !strings.HasPrefix(receviedHost[0], bucketName+".") { + return true + } + return false +} + +func GetV2Authorization(ak, sk, method, bucketName, objectKey, queryUrl string, headers map[string][]string) (ret map[string]string) { + + if strings.HasPrefix(queryUrl, "?") { + queryUrl = queryUrl[1:] + } + + method = strings.ToUpper(method) + + querys := strings.Split(queryUrl, "&") + querysResult := make([]string, 0) + for _, value := range querys { + if value != "=" && len(value) != 0 { + querysResult = append(querysResult, value) + } + } + params := make(map[string]string) + + for _, value := range querysResult { + kv := strings.Split(value, "=") + length := len(kv) + if length == 1 { + key := UrlDecodeWithoutError(kv[0]) + params[key] = "" + } else if length >= 2 { + key := UrlDecodeWithoutError(kv[0]) + vals := make([]string, 0, length-1) + for i := 1; i < length; i++ { + val := UrlDecodeWithoutError(kv[i]) + vals = append(vals, val) + } + params[key] = strings.Join(vals, "=") + } + } + headers = copyHeaders(headers) + pathStyle := isPathStyle(headers, bucketName) + conf := &config{securityProvider: &securityProvider{ak: ak, sk: sk}, + urlHolder: &urlHolder{scheme: "https", host: "dummy", port: 443}, + pathStyle: pathStyle} + conf.signature = SignatureObs + _, canonicalizedURL := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedURL, headers, true) + v2HashPrefix := OBS_HASH_PREFIX + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + return +} + func GetAuthorization(ak, sk, method, bucketName, objectKey, queryUrl string, headers map[string][]string) (ret map[string]string) { if strings.HasPrefix(queryUrl, "?") { @@ -264,13 +338,13 @@ func GetAuthorization(ak, sk, method, bucketName, objectKey, queryUrl string, he kv := strings.Split(value, "=") length := len(kv) if length == 1 { - key, _ := UrlDecode(kv[0]) + key := UrlDecodeWithoutError(kv[0]) params[key] = "" } else if length >= 2 { - key, _ := UrlDecode(kv[0]) + key := UrlDecodeWithoutError(kv[0]) vals := make([]string, 0, length-1) for i := 1; i < length; i++ { - val, _ := UrlDecode(kv[i]) + val := UrlDecodeWithoutError(kv[i]) vals = append(vals, val) } params[key] = strings.Join(vals, "=") @@ -302,31 +376,38 @@ func GetAuthorization(ak, sk, method, bucketName, objectKey, queryUrl string, he if isTemporary { return getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature, conf, params, headers, isObs) - } else { - signature, region, signedHeaders := parseHeaders(headers) - if signature == "v4" { - conf.signature = SignatureV4 - requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) - parsedRequestUrl, _ := url.Parse(requestUrl) - headerKeys := strings.Split(signedHeaders, ";") - _headers := make(map[string][]string, len(headerKeys)) - for _, headerKey := range headerKeys { - _headers[headerKey] = headers[headerKey] - } - ret = v4Auth(ak, sk, region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, _headers) - ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) - } else if signature == "v2" { - conf.signature = SignatureV2 - _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) - ret = v2Auth(ak, sk, method, canonicalizedUrl, headers, isObs) - v2HashPrefix := V2_HASH_PREFIX - if isObs { - v2HashPrefix = OBS_HASH_PREFIX - } - ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) - } - return } + signature, region, signedHeaders := parseHeaders(headers) + if signature == "v4" { + conf.signature = SignatureV4 + requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + parsedRequestUrl, _err := url.Parse(requestUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl with reason: %v", _err) + return nil + } + headerKeys := strings.Split(signedHeaders, ";") + _headers := make(map[string][]string, len(headerKeys)) + for _, headerKey := range headerKeys { + _headers[headerKey] = headers[headerKey] + } + ret = v4Auth(ak, sk, region, method, canonicalizedUrl, parsedRequestUrl.RawQuery, _headers) + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s Credential=%s,SignedHeaders=%s,Signature=%s", V4_HASH_PREFIX, ret["Credential"], ret["SignedHeaders"], ret["Signature"]) + } else if signature == "v2" { + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } + _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) + ret = v2Auth(ak, sk, method, canonicalizedUrl, headers, isObs) + v2HashPrefix := V2_HASH_PREFIX + if isObs { + v2HashPrefix = OBS_HASH_PREFIX + } + ret[HEADER_AUTH_CAMEL] = fmt.Sprintf("%s %s:%s", v2HashPrefix, ak, ret["Signature"]) + } + return } @@ -347,7 +428,7 @@ func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature credential = params[strings.ToLower(PARAM_CREDENTIAL_AMZ_CAMEL)] } - _credential, _ := UrlDecode(credential) + _credential := UrlDecodeWithoutError(credential) regions := regionRegex.FindStringSubmatch(_credential) var region string @@ -386,11 +467,19 @@ func getTemporaryAuthorization(ak, sk, method, bucketName, objectKey, signature ret[PARAM_SIGNEDHEADERS_AMZ_CAMEL] = signedHeaders requestUrl, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) - parsedRequestUrl, _ := url.Parse(requestUrl) + parsedRequestUrl, _err := url.Parse(requestUrl) + if _err != nil { + doLog(LEVEL_WARN, "Failed to parse requestUrl with reason: %v", _err) + return nil + } stringToSign := getV4StringToSign(method, canonicalizedUrl, parsedRequestUrl.RawQuery, scope, longDate, UNSIGNED_PAYLOAD, strings.Split(signedHeaders, ";"), headers) ret[PARAM_SIGNATURE_AMZ_CAMEL] = UrlEncode(getSignature(stringToSign, sk, region, shortDate), false) } else if signature == "v2" { - conf.signature = SignatureV2 + if isObs { + conf.signature = SignatureObs + } else { + conf.signature = SignatureV2 + } _, canonicalizedUrl := conf.formatUrls(bucketName, objectKey, params, false) expires, ok := params["Expires"] if !ok { diff --git a/pkg/multicloud/huawei/shell/iam.go b/pkg/multicloud/huawei/shell/iam.go new file mode 100644 index 0000000000..1ce771a196 --- /dev/null +++ b/pkg/multicloud/huawei/shell/iam.go @@ -0,0 +1,21 @@ +package shell + +import ( + "fmt" + + "yunion.io/x/onecloud/pkg/multicloud/huawei" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type OwnerShowOptions struct { + } + shellutils.R(&OwnerShowOptions{}, "owner-show", "Get aksk owner id", func(cli *huawei.SRegion, args *OwnerShowOptions) error { + result, err := cli.GetClient().GetOwnerId() + if err != nil { + return err + } + fmt.Println(result) + return nil + }) +}