From 8a9ec04f9f46c1c007baa753fbb6e07e4c3fc779 Mon Sep 17 00:00:00 2001 From: lvyangyang Date: Thu, 3 Dec 2020 11:31:51 +0800 Subject: [PATCH] =?UTF-8?q?feat(region):=20=E5=A4=9A=E4=BA=91=20bucket=20?= =?UTF-8?q?=20website.cors,referer=20=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/cloudprovider/objectstore.go | 86 ++++++++++- pkg/compute/models/buckets.go | 6 +- pkg/multicloud/aliyun/aliyun.go | 9 ++ pkg/multicloud/aliyun/bucket.go | 213 ++++++++++++++++++++++++++++ pkg/multicloud/aliyun/cdn.go | 104 ++++++++++++++ pkg/multicloud/aliyun/shell/cdn.go | 46 ++++++ pkg/multicloud/aws/bucket.go | 164 +++++++++++++++++++++ pkg/multicloud/aws/region.go | 8 ++ pkg/multicloud/bucket_base.go | 4 +- pkg/multicloud/huawei/bucket.go | 116 +++++++++++++++ pkg/multicloud/objectstore/shell.go | 4 +- pkg/multicloud/qcloud/bucket.go | 101 ++----------- 12 files changed, 764 insertions(+), 97 deletions(-) create mode 100644 pkg/multicloud/aliyun/cdn.go create mode 100644 pkg/multicloud/aliyun/shell/cdn.go diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index a403fd0497..664d69b7cf 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -250,7 +250,7 @@ type ICloudBucket interface { SetCORS(rules []SBucketCORSRule) error GetCORSRules() ([]SBucketCORSRule, error) - DeleteCORS(id []string) ([]SBucketCORSRule, error) + DeleteCORS() error SetReferer(conf SBucketRefererConf) error GetReferer() (SBucketRefererConf, error) @@ -738,3 +738,87 @@ func FetchMetaFromHttpHeader(metaPrefix string, headers http.Header) http.Header } return meta } + +func SetBucketCORS(ibucket ICloudBucket, rules []SBucketCORSRule) error { + if len(rules) == 0 { + return nil + } + + oldRules, err := ibucket.GetCORSRules() + if err != nil { + return errors.Wrap(err, "ibucket.GetCORSRules()") + } + + newSet := []SBucketCORSRule{} + updateSet := map[int]SBucketCORSRule{} + for i := range rules { + index, err := strconv.Atoi(rules[i].Id) + if err == nil && index < len(oldRules) { + updateSet[index] = rules[i] + } else { + newSet = append(newSet, rules[i]) + } + } + + updatedRules := []SBucketCORSRule{} + for i := range oldRules { + if _, ok := updateSet[i]; !ok { + updatedRules = append(updatedRules, oldRules[i]) + } else { + updatedRules = append(updatedRules, updateSet[i]) + } + } + updatedRules = append(updatedRules, newSet...) + + err = ibucket.SetCORS(updatedRules) + if err != nil { + return errors.Wrap(err, "ibucket.SetCORS(updatedRules)") + } + return nil +} + +func DeleteBucketCORS(ibucket ICloudBucket, id []string) ([]SBucketCORSRule, error) { + if len(id) == 0 { + return nil, nil + } + deletedRules := []SBucketCORSRule{} + + oldRules, err := ibucket.GetCORSRules() + if err != nil { + return nil, errors.Wrap(err, "ibucket.GetCORSRules()") + } + + excludeMap := map[int]bool{} + for i := range id { + index, err := strconv.Atoi(id[i]) + if err == nil && index < len(oldRules) { + excludeMap[index] = true + } + } + if len(excludeMap) == 0 { + return nil, nil + } + + newRules := []SBucketCORSRule{} + for i := range oldRules { + if _, ok := excludeMap[i]; !ok { + newRules = append(newRules, oldRules[i]) + } else { + deletedRules = append(deletedRules, oldRules[i]) + } + } + + if len(newRules) == 0 { + err = ibucket.DeleteCORS() + if err != nil { + return nil, errors.Wrapf(err, "ibucket.DeleteCORS()") + } + } else { + err = ibucket.SetCORS(newRules) + if err != nil { + return nil, errors.Wrapf(err, "ibucket.SetBucketCORS(newRules)") + } + } + + return deletedRules, nil +} diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 400a5d5d0e..1472e7d7ad 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -1377,9 +1377,9 @@ func (bucket *SBucket) PerformSetCors( Id: input.Data[i].Id, }) } - err = iBucket.SetCORS(rules) + err = cloudprovider.SetBucketCORS(iBucket, rules) if err != nil { - return nil, httperrors.NewInternalServerError("iBucket.SetCORS error %s", err) + return nil, httperrors.NewInternalServerError("cloudprovider.SetBucketCORS error %s", err) } db.OpsLog.LogEvent(bucket, db.ACT_SET_CORS, rules, userCred) logclient.AddActionLogWithContext(ctx, bucket, logclient.ACT_SET_CORS, rules, userCred, true) @@ -1404,7 +1404,7 @@ func (bucket *SBucket) PerformDeleteCors( if err != nil { return nil, errors.Wrap(err, "GetIBucket") } - result, err := iBucket.DeleteCORS(input.Id) + result, err := cloudprovider.DeleteBucketCORS(iBucket, input.Id) if err != nil { return nil, httperrors.NewInternalServerError("iBucket.DeleteCORS error %s", err) } diff --git a/pkg/multicloud/aliyun/aliyun.go b/pkg/multicloud/aliyun/aliyun.go index 4657882c93..364a1b2322 100644 --- a/pkg/multicloud/aliyun/aliyun.go +++ b/pkg/multicloud/aliyun/aliyun.go @@ -60,6 +60,7 @@ const ( ALIYUN_PVTZ_API_VERSION = "2018-01-01" ALIYUN_ALIDNS_API_VERSION = "2015-01-09" ALIYUN_CBN_API_VERSION = "2017-09-12" + ALIYUN_CDN_API_VERSION = "2018-05-10" ) var ( @@ -290,6 +291,14 @@ func (self *SAliyunClient) cbnRequest(apiName string, params map[string]string) return jsonRequest(cli, "cbn.aliyuncs.com", ALIYUN_CBN_API_VERSION, apiName, params, self.debug) } +func (self *SAliyunClient) cdnRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getDefaultClient() + if err != nil { + return nil, err + } + return jsonRequest(cli, "cdn.aliyuncs.com", ALIYUN_CDN_API_VERSION, apiName, params, self.debug) +} + func (self *SAliyunClient) fetchRegions() error { body, err := self.ecsRequest("DescribeRegions", map[string]string{"AcceptLanguage": "zh-CN"}) if err != nil { diff --git a/pkg/multicloud/aliyun/bucket.go b/pkg/multicloud/aliyun/bucket.go index 8d3a304b89..285baea9eb 100644 --- a/pkg/multicloud/aliyun/bucket.go +++ b/pkg/multicloud/aliyun/bucket.go @@ -19,13 +19,17 @@ import ( "fmt" "io" "net/http" + "strconv" + "strings" "time" "github.com/aliyun/aliyun-oss-go-sdk/oss" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" ) @@ -502,3 +506,212 @@ func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, par } return part.ETag, nil } + +func (b *SBucket) SetWebsite(websitConf cloudprovider.SBucketWebsiteConf) error { + if len(websitConf.Index) == 0 { + return errors.Wrap(cloudprovider.ErrNotSupported, "missing Index") + } + if len(websitConf.ErrorDocument) == 0 { + return errors.Wrap(cloudprovider.ErrNotSupported, "missing ErrorDocument") + } + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + + err = osscli.SetBucketWebsite(b.Name, websitConf.Index, websitConf.ErrorDocument) + if err != nil { + return errors.Wrapf(err, " osscli.SetBucketWebsite(%s,%s,%s)", b.Name, websitConf.Index, websitConf.ErrorDocument) + } + return nil +} + +func (b *SBucket) GetWebsiteConf() (cloudprovider.SBucketWebsiteConf, error) { + result := cloudprovider.SBucketWebsiteConf{} + osscli, err := b.region.GetOssClient() + if err != nil { + return result, errors.Wrap(err, "GetOssClient") + } + websiteResult, err := osscli.GetBucketWebsite(b.Name) + if err != nil { + if strings.Contains(err.Error(), "NoSuchWebsiteConfiguration") { + return cloudprovider.SBucketWebsiteConf{}, nil + } + return result, errors.Wrapf(err, "osscli.GetBucketWebsite(%s)", b.Name) + } + result.Index = websiteResult.IndexDocument.Suffix + result.ErrorDocument = websiteResult.ErrorDocument.Key + return result, nil +} + +func (b *SBucket) DeleteWebSiteConf() error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + log.Infof("to be delete") + err = osscli.DeleteBucketWebsite(b.Name) + if err != nil { + return errors.Wrapf(err, "osscli.DeleteBucketWebsite(%s)", b.Name) + } + log.Infof("deleted") + return nil +} + +func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { + if len(rules) == 0 { + return nil + } + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + input := []oss.CORSRule{} + for i := range rules { + input = append(input, oss.CORSRule{ + AllowedOrigin: rules[i].AllowedOrigins, + AllowedMethod: rules[i].AllowedMethods, + AllowedHeader: rules[i].AllowedHeaders, + MaxAgeSeconds: rules[i].MaxAgeSeconds, + ExposeHeader: rules[i].ExposeHeaders, + }) + } + + err = osscli.SetBucketCORS(b.Name, input) + if err != nil { + return errors.Wrapf(err, "osscli.SetBucketCORS(%s,%s)", b.Name, jsonutils.Marshal(input).String()) + } + return nil +} + +func (b *SBucket) GetCORSRules() ([]cloudprovider.SBucketCORSRule, error) { + osscli, err := b.region.GetOssClient() + if err != nil { + return nil, errors.Wrap(err, "GetOssClient") + } + conf, err := osscli.GetBucketCORS(b.Name) + if err != nil { + if !strings.Contains(err.Error(), "NoSuchCORSConfiguration") { + return nil, errors.Wrapf(err, "osscli.GetBucketCORS(%s)", b.Name) + } + } + result := []cloudprovider.SBucketCORSRule{} + for i := range conf.CORSRules { + result = append(result, cloudprovider.SBucketCORSRule{ + AllowedOrigins: conf.CORSRules[i].AllowedOrigin, + AllowedMethods: conf.CORSRules[i].AllowedMethod, + AllowedHeaders: conf.CORSRules[i].AllowedHeader, + MaxAgeSeconds: conf.CORSRules[i].MaxAgeSeconds, + ExposeHeaders: conf.CORSRules[i].ExposeHeader, + Id: strconv.Itoa(i), + }) + } + return result, nil +} + +func (b *SBucket) DeleteCORS() error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + + err = osscli.DeleteBucketCORS(b.Name) + if err != nil { + return errors.Wrapf(err, "osscli.DeleteBucketCORS(%s)", b.Name) + } + + return nil +} + +func (b *SBucket) SetReferer(conf cloudprovider.SBucketRefererConf) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + err = osscli.SetBucketReferer(b.Name, conf.WhiteList, conf.AllowEmptyRefer) + if err != nil { + return errors.Wrapf(err, "osscli.SetBucketReferer(%s,%s,%d)", b.Name, conf.WhiteList, conf.AllowEmptyRefer) + } + return nil +} + +func (b *SBucket) GetReferer() (cloudprovider.SBucketRefererConf, error) { + result := cloudprovider.SBucketRefererConf{} + osscli, err := b.region.GetOssClient() + if err != nil { + return result, errors.Wrap(err, "GetOssClient") + } + refererResult, err := osscli.GetBucketReferer(b.Name) + if err != nil { + return result, errors.Wrapf(err, "osscli.GetBucketReferer(%s)", b.Name) + } + result = cloudprovider.SBucketRefererConf{ + WhiteList: refererResult.RefererList, + AllowEmptyRefer: refererResult.AllowEmptyReferer, + } + return result, nil +} + +func toAPICdnArea(area string) string { + switch area { + case "domestic": + return api.CDN_DOMAIN_AREA_MAINLAND + case "overseas": + return api.CDN_DOMAIN_AREA_OVERSEAS + case "global": + return api.CDN_DOMAIN_AREA_GLOBAL + default: + return "" + } +} + +func toAPICdnStatus(status string) string { + switch status { + case "online": + return api.CDN_DOMAIN_STATUS_ONLINE + case "offline": + return api.CDN_DOMAIN_STATUS_OFFLINE + case "configuring", "checking", "stopping", "deleting": + return api.CDN_DOMAIN_STATUS_PROCESSING + case "check_failed", "configure_failed": + return api.CDN_DOMAIN_STATUS_REJECTED + default: + return "" + } +} + +func (b *SBucket) GetCdnDomains() ([]cloudprovider.SCdnDomain, error) { + bucketExtUrl := fmt.Sprintf("%s.%s", b.Name, b.region.getOSSExternalDomain()) + cdnDomains, err := b.region.client.DescribeDomainsBySource(bucketExtUrl) + if err != nil { + return nil, errors.Wrapf(err, " b.region.client.DescribeDomainsBySource(%s)", bucketExtUrl) + } + result := []cloudprovider.SCdnDomain{} + for i := range cdnDomains.DomainsData { + if cdnDomains.DomainsData[i].Source == bucketExtUrl { + for j := range cdnDomains.DomainsData[i].Domains.DomainNames { + area := "" + cdnDomianDescribes, err := b.region.client.DescribeUserDomains(cdnDomains.DomainsData[i].Domains.DomainNames[j]) + if err != nil { + return nil, errors.Wrapf(err, "b.region.client.DescribeUserDomains(%s)", cdnDomains.DomainsData[i].Domains.DomainNames[j]) + } + for k := range cdnDomianDescribes.PageData { + if cdnDomianDescribes.PageData[k].DomainName == cdnDomains.DomainsData[i].Domains.DomainNames[j] { + area = cdnDomianDescribes.PageData[k].Coverage + break + } + } + + result = append(result, cloudprovider.SCdnDomain{ + Domain: cdnDomains.DomainsData[i].Domains.DomainNames[j], + Status: toAPICdnStatus(cdnDomains.DomainsData[i].DomainInfos.DomainInfo[j].Status), + Cname: cdnDomains.DomainsData[i].DomainInfos.DomainInfo[j].DomainCname, + Area: toAPICdnArea(area), + Origin: bucketExtUrl, + OriginType: api.CDN_DOMAIN_ORIGIN_TYPE_BUCKET, + }) + } + } + } + return result, nil +} diff --git a/pkg/multicloud/aliyun/cdn.go b/pkg/multicloud/aliyun/cdn.go new file mode 100644 index 0000000000..7f45c26cc9 --- /dev/null +++ b/pkg/multicloud/aliyun/cdn.go @@ -0,0 +1,104 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package aliyun + +import ( + "time" + + "github.com/pkg/errors" +) + +type SCdnDomainNames struct { + DomainNames []string `json:"domainNames"` +} +type SDomainInfo struct { + DomainCname string `json:"DomainCname"` + Status string `json:"Status"` + CreateTime time.Time `json:"CreateTime"` + UpdateTime time.Time `json:"UpdateTime"` + DomainName string `json:"DomainName"` +} +type SCdnDomainInfos struct { + DomainInfo []SDomainInfo `json:"domainInfo"` +} +type SCdnDomainsData struct { + Source string `json:"Source"` + Domains SCdnDomainNames `json:"Domains"` + DomainInfos SCdnDomainInfos `json:"DomainInfos"` +} +type SCdnDomainsList struct { + DomainsData []SCdnDomainsData `json:"DomainsData"` +} + +type SCdnSource struct { + Port int `json:"Port"` + Weight string `json:"Weight"` + Type string `json:"Type"` + Content string `json:"Content"` + Priority string `json:"Priority"` +} +type SCdnSources struct { + Source []SCdnSource `json:"Source"` +} +type SCdnPageData struct { + Cname string `json:"Cname"` + Description string `json:"Description"` + CdnType string `json:"CdnType"` + ResourceGroupID string `json:"ResourceGroupId"` + DomainStatus string `json:"DomainStatus"` + SslProtocol string `json:"SslProtocol"` + DomainName string `json:"DomainName"` + Coverage string `json:"Coverage"` + Sources SCdnSource `json:"Sources"` + GmtModified string `json:"GmtModified"` + Sandbox string `json:"Sandbox"` + GmtCreated time.Time `json:"GmtCreated"` +} +type SCdnDomains struct { + PageData []SCdnPageData `json:"PageData"` +} + +func (client *SAliyunClient) DescribeDomainsBySource(origin string) (SCdnDomainsList, error) { + sproducts := SCdnDomainsList{} + params := map[string]string{} + params["Action"] = "DescribeDomainsBySource" + params["Sources"] = origin + resp, err := client.cdnRequest("DescribeDomainsBySource", params) + if err != nil { + return sproducts, errors.Wrap(err, "DescribeDomainsBySource") + } + err = resp.Unmarshal(&sproducts, "DomainsList") + if err != nil { + return sproducts, errors.Wrap(err, "resp.Unmarshal") + } + return sproducts, nil +} + +func (client *SAliyunClient) DescribeUserDomains(domain string) (SCdnDomains, error) { + sproducts := SCdnDomains{} + params := map[string]string{} + params["Action"] = "DescribeUserDomains" + params["DomainName"] = domain + params["DomainSearchType"] = "full_match" + resp, err := client.cdnRequest("DescribeUserDomains", params) + if err != nil { + return sproducts, errors.Wrap(err, "DescribeUserDomains") + } + err = resp.Unmarshal(&sproducts, "Domains") + if err != nil { + return sproducts, errors.Wrap(err, "resp.Unmarshal") + } + return sproducts, nil +} diff --git a/pkg/multicloud/aliyun/shell/cdn.go b/pkg/multicloud/aliyun/shell/cdn.go new file mode 100644 index 0000000000..6f70567a0f --- /dev/null +++ b/pkg/multicloud/aliyun/shell/cdn.go @@ -0,0 +1,46 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// PageSizeations under the License. + +package shell + +import ( + "yunion.io/x/onecloud/pkg/multicloud/aliyun" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type CdnDomainList struct { + Origin string + } + shellutils.R(&CdnDomainList{}, "cdn-domain-list", "List cdn domain", func(cli *aliyun.SRegion, args *CdnDomainList) error { + domainlist, e := cli.GetClient().DescribeDomainsBySource(args.Origin) + if e != nil { + return e + } + printList(domainlist.DomainsData, len(domainlist.DomainsData), 1, len(domainlist.DomainsData), []string{}) + return nil + }) + + type CdnDomainShow struct { + DOMAIN string + } + shellutils.R(&CdnDomainShow{}, "cdn-domain-show", "show cdn domain", func(cli *aliyun.SRegion, args *CdnDomainShow) error { + domainlist, e := cli.GetClient().DescribeUserDomains(args.DOMAIN) + if e != nil { + return e + } + printList(domainlist.PageData, len(domainlist.PageData), 1, len(domainlist.PageData), []string{}) + return nil + }) +} diff --git a/pkg/multicloud/aws/bucket.go b/pkg/multicloud/aws/bucket.go index 450776ad77..42935c1102 100644 --- a/pkg/multicloud/aws/bucket.go +++ b/pkg/multicloud/aws/bucket.go @@ -20,11 +20,14 @@ import ( "io" "net/http" "net/url" + "strconv" + "strings" "time" "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/s3" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/s3cli" @@ -145,6 +148,10 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { } } +func (b *SBucket) GetWebsiteUrl() string { + return fmt.Sprintf("http://%s.%s", b.Name, b.region.getS3WebsiteEndpoint()) +} + func (b *SBucket) GetStats() cloudprovider.SBucketStats { stats, _ := cloudprovider.GetIBucketStats(b) return stats @@ -514,3 +521,160 @@ func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, par } return *output.CopyPartResult.ETag, nil } + +func (b *SBucket) SetWebsite(websitConf cloudprovider.SBucketWebsiteConf) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + s3WebConf := s3.WebsiteConfiguration{} + s3WebConf.SetIndexDocument(&s3.IndexDocument{Suffix: &websitConf.Index}) + s3WebConf.SetErrorDocument(&s3.ErrorDocument{Key: &websitConf.ErrorDocument}) + input := s3.PutBucketWebsiteInput{} + input.SetBucket(b.Name) + input.SetWebsiteConfiguration(&s3WebConf) + _, err = s3cli.PutBucketWebsite(&input) + if err != nil { + return errors.Wrapf(err, "s3cli.PutBucketWebsite(%s)", jsonutils.Marshal(input).String()) + } + return nil +} + +func (b *SBucket) GetWebsiteConf() (cloudprovider.SBucketWebsiteConf, error) { + result := cloudprovider.SBucketWebsiteConf{} + s3cli, err := b.region.GetS3Client() + if err != nil { + return result, errors.Wrap(err, "GetS3Client") + } + input := s3.GetBucketWebsiteInput{} + input.SetBucket(b.Name) + webconfResult, err := s3cli.GetBucketWebsite(&input) + if err != nil { + return result, errors.Wrapf(err, "s3cli.GetBucketWebsite(%s)", b.Name) + } + + if webconfResult.IndexDocument != nil && webconfResult.IndexDocument.Suffix != nil { + result.Index = *webconfResult.IndexDocument.Suffix + } + if webconfResult.ErrorDocument != nil && webconfResult.ErrorDocument.Key != nil { + result.ErrorDocument = *webconfResult.ErrorDocument.Key + } + result.Url = b.GetWebsiteUrl() + return result, nil +} + +func (b *SBucket) DeleteWebSiteConf() error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + input := s3.DeleteBucketWebsiteInput{} + input.SetBucket(b.Name) + _, err = s3cli.DeleteBucketWebsite(&input) + if err != nil { + return errors.Wrapf(err, "s3cli.DeleteBucketWebsite(%s)", b.Name) + } + return nil +} + +func InputToAwsApiSliceString(input []string) []*string { + result := []*string{} + for i := range input { + result = append(result, &input[i]) + } + return result +} + +func InputToAwsApiInt64(input int64) *int64 { + return &input +} + +func AwsApiSliceStringToOutput(input []*string) []string { + result := []string{} + for i := range input { + if input[i] != nil { + result = append(result, *input[i]) + } else { + result = append(result, "") + } + } + return result +} + +func AwsApiInt64ToOutput(input *int64) int64 { + if input == nil { + return 0 + } + return *input +} + +func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + opts := []*s3.CORSRule{} + for i := range rules { + opts = append(opts, &s3.CORSRule{ + AllowedOrigins: InputToAwsApiSliceString(rules[i].AllowedOrigins), + AllowedMethods: InputToAwsApiSliceString(rules[i].AllowedMethods), + AllowedHeaders: InputToAwsApiSliceString(rules[i].AllowedHeaders), + MaxAgeSeconds: InputToAwsApiInt64(int64(rules[i].MaxAgeSeconds)), + ExposeHeaders: InputToAwsApiSliceString(rules[i].ExposeHeaders), + }) + } + + input := s3.PutBucketCorsInput{} + input.SetBucket(b.Name) + input.SetCORSConfiguration(&s3.CORSConfiguration{CORSRules: opts}) + _, err = s3cli.PutBucketCors(&input) + if err != nil { + return errors.Wrapf(err, "s3cli.PutBucketCors(%s)", input) + } + return nil +} + +func (b *SBucket) GetCORSRules() ([]cloudprovider.SBucketCORSRule, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return nil, errors.Wrap(err, "GetS3Client") + } + input := s3.GetBucketCorsInput{} + input.SetBucket(b.Name) + conf, err := s3cli.GetBucketCors(&input) + if err != nil { + if !strings.Contains(err.Error(), "NoSuchCORSConfiguration") { + return nil, errors.Wrapf(err, "s3cli.GetBucketCors(%s)", b.Name) + } + } + if conf == nil { + return nil, nil + } + result := []cloudprovider.SBucketCORSRule{} + for i := range conf.CORSRules { + result = append(result, cloudprovider.SBucketCORSRule{ + AllowedOrigins: AwsApiSliceStringToOutput(conf.CORSRules[i].AllowedOrigins), + AllowedMethods: AwsApiSliceStringToOutput(conf.CORSRules[i].AllowedMethods), + AllowedHeaders: AwsApiSliceStringToOutput(conf.CORSRules[i].AllowedHeaders), + MaxAgeSeconds: int(AwsApiInt64ToOutput(conf.CORSRules[i].MaxAgeSeconds)), + ExposeHeaders: AwsApiSliceStringToOutput(conf.CORSRules[i].ExposeHeaders), + Id: strconv.Itoa(i), + }) + } + return result, nil +} + +func (b *SBucket) DeleteCORS() error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + + input := s3.DeleteBucketCorsInput{} + input.SetBucket(b.Name) + _, err = s3cli.DeleteBucketCors(&input) + if err != nil { + return errors.Wrapf(err, "s3cli.DeleteBucketCors(%s)", b.Name) + } + return nil +} diff --git a/pkg/multicloud/aws/region.go b/pkg/multicloud/aws/region.go index 43c96f7351..17775f4c52 100644 --- a/pkg/multicloud/aws/region.go +++ b/pkg/multicloud/aws/region.go @@ -1026,6 +1026,14 @@ func (region *SRegion) getS3Endpoint() string { return "" } +func (region *SRegion) getS3WebsiteEndpoint() string { + base := region.getBaseEndpoint() + if len(base) > 0 { + return "s3-website." + base + } + return "" +} + func (region *SRegion) getEc2Endpoint() string { return region.RegionEndpoint } diff --git a/pkg/multicloud/bucket_base.go b/pkg/multicloud/bucket_base.go index a755149ea1..13bc014f82 100644 --- a/pkg/multicloud/bucket_base.go +++ b/pkg/multicloud/bucket_base.go @@ -94,8 +94,8 @@ func (b *SBaseBucket) GetCORSRules() ([]cloudprovider.SBucketCORSRule, error) { return nil, cloudprovider.ErrNotImplemented } -func (b *SBaseBucket) DeleteCORS(id []string) ([]cloudprovider.SBucketCORSRule, error) { - return nil, cloudprovider.ErrNotImplemented +func (b *SBaseBucket) DeleteCORS() error { + return cloudprovider.ErrNotImplemented } func (b *SBaseBucket) SetReferer(conf cloudprovider.SBucketRefererConf) error { diff --git a/pkg/multicloud/huawei/bucket.go b/pkg/multicloud/huawei/bucket.go index 5b80349141..0f5e509a67 100644 --- a/pkg/multicloud/huawei/bucket.go +++ b/pkg/multicloud/huawei/bucket.go @@ -19,6 +19,8 @@ import ( "fmt" "io" "net/http" + "strconv" + "strings" "time" "yunion.io/x/log" @@ -557,3 +559,117 @@ func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, par } return output.ETag, nil } + +func (b *SBucket) SetWebsite(websitConf cloudprovider.SBucketWebsiteConf) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + obsWebConf := obs.SetBucketWebsiteConfigurationInput{} + obsWebConf.Bucket = b.Name + obsWebConf.BucketWebsiteConfiguration = obs.BucketWebsiteConfiguration{ + IndexDocument: obs.IndexDocument{Suffix: websitConf.Index}, + ErrorDocument: obs.ErrorDocument{Key: websitConf.ErrorDocument}, + } + _, err = obscli.SetBucketWebsiteConfiguration(&obsWebConf) + if err != nil { + return errors.Wrap(err, "obscli.SetBucketWebsiteConfiguration(&obsWebConf)") + } + return nil +} + +func (b *SBucket) GetWebsiteConf() (cloudprovider.SBucketWebsiteConf, error) { + result := cloudprovider.SBucketWebsiteConf{} + obscli, err := b.region.getOBSClient() + if err != nil { + return result, errors.Wrap(err, "GetOBSClient") + } + out, err := obscli.GetBucketWebsiteConfiguration(b.Name) + if out == nil { + return result, nil + } + result.Index = out.IndexDocument.Suffix + result.ErrorDocument = out.ErrorDocument.Key + result.Url = fmt.Sprintf("https://%s.obs-website.%s.myhuaweicloud.com", b.Name, b.region.GetId()) + return result, nil +} + +func (b *SBucket) DeleteWebSiteConf() error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + _, err = obscli.DeleteBucketWebsiteConfiguration(b.Name) + if err != nil { + return errors.Wrapf(err, "obscli.DeleteBucketWebsiteConfiguration(%s)", b.Name) + } + return nil +} + +func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + opts := []obs.CorsRule{} + for i := range rules { + opts = append(opts, obs.CorsRule{ + AllowedOrigin: rules[i].AllowedOrigins, + AllowedMethod: rules[i].AllowedMethods, + AllowedHeader: rules[i].AllowedHeaders, + MaxAgeSeconds: rules[i].MaxAgeSeconds, + ExposeHeader: rules[i].ExposeHeaders, + }) + } + + input := obs.SetBucketCorsInput{} + input.Bucket = b.Name + input.BucketCors.CorsRules = opts + _, err = obscli.SetBucketCors(&input) + if err != nil { + return errors.Wrapf(err, "obscli.SetBucketCors(%s)", input) + } + return nil +} + +func (b *SBucket) GetCORSRules() ([]cloudprovider.SBucketCORSRule, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + conf, err := obscli.GetBucketCors(b.Name) + if err != nil { + if !strings.Contains(err.Error(), "NoSuchCORSConfiguration") { + return nil, errors.Wrapf(err, "obscli.GetBucketCors(%s)", b.Name) + } + } + if conf == nil { + return nil, nil + } + result := []cloudprovider.SBucketCORSRule{} + for i := range conf.CorsRules { + result = append(result, cloudprovider.SBucketCORSRule{ + AllowedOrigins: conf.CorsRules[i].AllowedOrigin, + AllowedMethods: conf.CorsRules[i].AllowedMethod, + AllowedHeaders: conf.CorsRules[i].AllowedHeader, + MaxAgeSeconds: conf.CorsRules[i].MaxAgeSeconds, + ExposeHeaders: conf.CorsRules[i].ExposeHeader, + Id: strconv.Itoa(i), + }) + } + return result, nil +} + +func (b *SBucket) DeleteCORS() error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + + _, err = obscli.DeleteBucketCors(b.Name) + if err != nil { + return errors.Wrapf(err, "obscli.DeleteBucketCors(%s)", b.Name) + } + return nil +} diff --git a/pkg/multicloud/objectstore/shell.go b/pkg/multicloud/objectstore/shell.go index d30eefc79b..7e1a63532e 100644 --- a/pkg/multicloud/objectstore/shell.go +++ b/pkg/multicloud/objectstore/shell.go @@ -466,7 +466,7 @@ func S3Shell() { ExposeHeaders: args.ExposeHeaders, Id: args.Id, } - err = bucket.SetCORS([]cloudprovider.SBucketCORSRule{rule}) + err = cloudprovider.SetBucketCORS(bucket, []cloudprovider.SBucketCORSRule{rule}) if err != nil { return err } @@ -499,7 +499,7 @@ func S3Shell() { if err != nil { return err } - result, err := bucket.DeleteCORS(args.Ids) + result, err := cloudprovider.DeleteBucketCORS(bucket, args.Ids) if err != nil { return err } diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index df03715f0b..8f9cc29221 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -625,21 +625,16 @@ func (b *SBucket) DeleteWebSiteConf() error { } func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { - for i := range rules { - if len(rules[i].AllowedOrigins) == 0 { - return errors.Wrap(cloudprovider.ErrNotSupported, "missing AllowedOrigins") - } - if len(rules[i].AllowedMethods) == 0 { - return errors.Wrap(cloudprovider.ErrNotSupported, "missing AllowedMethods") - } + if len(rules) == 0 { + return nil } coscli, err := b.region.GetCosClient(b) if err != nil { return errors.Wrap(err, "b.region.GetCosClient") } - opts := cos.BucketPutCORSOptions{} + input := cos.BucketPutCORSOptions{} for i := range rules { - opts.Rules = append(opts.Rules, cos.BucketCORSRule{ + input.Rules = append(input.Rules, cos.BucketCORSRule{ AllowedOrigins: rules[i].AllowedOrigins, AllowedMethods: rules[i].AllowedMethods, AllowedHeaders: rules[i].AllowedHeaders, @@ -649,36 +644,7 @@ func (b *SBucket) SetCORS(rules []cloudprovider.SBucketCORSRule) error { }) } - newSet := []cos.BucketCORSRule{} - updateSet := map[int]cos.BucketCORSRule{} - - oldConf, _, err := coscli.Bucket.GetCORS(context.Background()) - if err != nil { - if !strings.Contains(err.Error(), "NoSuchCORSConfiguration") { - return errors.Wrap(err, "b.region.GetCORS") - } - } - - for i := range opts.Rules { - index, err := strconv.Atoi(opts.Rules[i].ID) - if err == nil && index < len(oldConf.Rules) { - updateSet[index] = opts.Rules[i] - } else { - newSet = append(newSet, opts.Rules[i]) - } - } - updatedOpts := cos.BucketPutCORSOptions{} - for i := range oldConf.Rules { - if _, ok := updateSet[i]; !ok { - updatedOpts.Rules = append(updatedOpts.Rules, oldConf.Rules[i]) - } else { - updatedOpts.Rules = append(updatedOpts.Rules, updateSet[i]) - } - } - - updatedOpts.Rules = append(updatedOpts.Rules, newSet...) - - _, err = coscli.Bucket.PutCORS(context.Background(), &updatedOpts) + _, err = coscli.Bucket.PutCORS(context.Background(), &input) if err != nil { return errors.Wrap(err, "coscli.Bucket.PutCORS") } @@ -711,60 +677,16 @@ func (b *SBucket) GetCORSRules() ([]cloudprovider.SBucketCORSRule, error) { return result, nil } -func (b *SBucket) DeleteCORS(id []string) ([]cloudprovider.SBucketCORSRule, error) { - deletedRules := []cloudprovider.SBucketCORSRule{} +func (b *SBucket) DeleteCORS() error { coscli, err := b.region.GetCosClient(b) if err != nil { - return nil, errors.Wrap(err, "b.region.GetCosClient") + return errors.Wrap(err, "b.region.GetCosClient") } - - existedRules := []cos.BucketCORSRule{} - if len(id) > 0 { - conf, _, err := coscli.Bucket.GetCORS(context.Background()) - if err != nil { - if strings.Contains(err.Error(), "NoSuchCORSConfiguration") { - return nil, nil - } - return nil, errors.Wrap(err, "b.region.GetCORS") - } - existedRules = conf.Rules + _, err = coscli.Bucket.DeleteCORS(context.Background()) + if err != nil { + return errors.Wrap(err, "coscli.Bucket.DeleteCORS") } - - excludeMap := map[int]bool{} - for i := range id { - index, err := strconv.Atoi(id[i]) - if err == nil { - excludeMap[index] = true - } - } - newRules := []cos.BucketCORSRule{} - for i := range existedRules { - if _, ok := excludeMap[i]; !ok { - newRules = append(newRules, existedRules[i]) - } else { - deletedRules = append(deletedRules, cloudprovider.SBucketCORSRule{ - AllowedOrigins: existedRules[i].AllowedOrigins, - AllowedMethods: existedRules[i].AllowedMethods, - AllowedHeaders: existedRules[i].AllowedHeaders, - MaxAgeSeconds: existedRules[i].MaxAgeSeconds, - ExposeHeaders: existedRules[i].ExposeHeaders, - }) - } - } - if len(newRules) < len(existedRules) { - if len(newRules) == 0 { - _, err = coscli.Bucket.DeleteCORS(context.Background()) - if err != nil { - return nil, errors.Wrap(err, "coscli.Bucket.DeleteCORS") - } - return deletedRules, nil - } - _, err = coscli.Bucket.PutCORS(context.Background(), &cos.BucketPutCORSOptions{Rules: newRules}) - if err != nil { - return nil, errors.Wrap(err, "coscli.Bucket.PutCORS") - } - } - return deletedRules, nil + return nil } func (b *SBucket) SetReferer(conf cloudprovider.SBucketRefererConf) error { @@ -807,6 +729,7 @@ func (b *SBucket) GetReferer() (cloudprovider.SBucketRefererConf, error) { result.AllowEmptyRefer = true } if referResult.Status == "Disabled" { + result.AllowEmptyRefer = true return result, nil } result.WhiteList = referResult.DomainList