From 4fa6fe457aa77d79c88c63d217b21bbfdce1f45d Mon Sep 17 00:00:00 2001 From: ioito Date: Tue, 19 Apr 2022 11:20:26 +0800 Subject: [PATCH] fix(region): apsara oss to asapi --- cmd/apsaracli/main.go | 16 +- pkg/cloudevent/models/cloudproviders.go | 1 - pkg/cloudid/models/cloudaccount.go | 1 - pkg/cloudprovider/cloudprovider.go | 3 - pkg/cloudprovider/endpoints.go | 14 -- pkg/compute/models/cloudaccounts.go | 3 - pkg/mcclient/options/cloudaccounts.go | 1 - pkg/multicloud/apsara/apsara.go | 140 +++----------- pkg/multicloud/apsara/bucket.go | 35 ++-- pkg/multicloud/apsara/disk.go | 2 - pkg/multicloud/apsara/image.go | 6 +- pkg/multicloud/apsara/objects.go | 8 +- pkg/multicloud/apsara/project.go | 9 +- pkg/multicloud/apsara/provider/provider.go | 5 - pkg/multicloud/apsara/region.go | 140 +++++++------- pkg/multicloud/apsara/shell/bucket.go | 19 +- pkg/multicloud/apsara/shell/image.go | 17 +- pkg/multicloud/apsara/shell/oss.go | 202 --------------------- pkg/multicloud/apsara/storagecache.go | 72 +------- pkg/multicloud/apsara/vpc.go | 24 +++ 20 files changed, 188 insertions(+), 530 deletions(-) delete mode 100644 pkg/multicloud/apsara/shell/oss.go diff --git a/cmd/apsaracli/main.go b/cmd/apsaracli/main.go index be3e6d0bd4..3faca57e93 100644 --- a/cmd/apsaracli/main.go +++ b/cmd/apsaracli/main.go @@ -31,14 +31,13 @@ import ( ) type BaseOptions struct { - Debug bool `help:"debug mode"` - AccessKey string `help:"Access key" default:"$APSARA_ACCESS_KEY" metavar:"APSARA_ACCESS_KEY"` - Secret string `help:"Secret" default:"$APSARA_SECRET" metavar:"APSARA_SECRET"` - cloudprovider.SApsaraEndpoints `help:"Endpoints for Apsara"` - Endpoint string `help:"Apsara endpoint" default:"$APSARA_ENDPOINT" metavar:"APSARA_ENDPOINT"` - RegionId string `help:"RegionId" default:"$APSARA_REGION" metavar:"APSARA_REGION"` - DEFAULT_REGION string `help:"Default region" default:"$APSARA_DEFAULT_REGION"` - SUBCOMMAND string `help:"apsaracli subcommand" subcommand:"true"` + Debug bool `help:"debug mode"` + AccessKey string `help:"Access key" default:"$APSARA_ACCESS_KEY" metavar:"APSARA_ACCESS_KEY"` + Secret string `help:"Secret" default:"$APSARA_SECRET" metavar:"APSARA_SECRET"` + Endpoint string `help:"Apsara endpoint" default:"$APSARA_ENDPOINT" metavar:"APSARA_ENDPOINT"` + RegionId string `help:"RegionId" default:"$APSARA_REGION" metavar:"APSARA_REGION"` + DEFAULT_REGION string `help:"Default region" default:"$APSARA_DEFAULT_REGION"` + SUBCOMMAND string `help:"apsaracli subcommand" subcommand:"true"` } func getSubcommandParser() (*structarg.ArgumentParser, error) { @@ -94,7 +93,6 @@ func newClient(options *BaseOptions) (*apsara.SRegion, error) { options.AccessKey, options.Secret, options.Endpoint, - options.SApsaraEndpoints, ).Debug(options.Debug). CloudproviderConfig( cloudprovider.ProviderConfig{ diff --git a/pkg/cloudevent/models/cloudproviders.go b/pkg/cloudevent/models/cloudproviders.go index 500f61a19c..582d4e9088 100644 --- a/pkg/cloudevent/models/cloudproviders.go +++ b/pkg/cloudevent/models/cloudproviders.go @@ -319,7 +319,6 @@ type SCloudproviderDelegate struct { Brand string Options struct { - cloudprovider.SApsaraEndpoints cloudprovider.SHCSOEndpoints } diff --git a/pkg/cloudid/models/cloudaccount.go b/pkg/cloudid/models/cloudaccount.go index 373455c105..64b2e3e887 100644 --- a/pkg/cloudid/models/cloudaccount.go +++ b/pkg/cloudid/models/cloudaccount.go @@ -488,7 +488,6 @@ type SCloudDelegate struct { Options struct { cloudprovider.SHCSOEndpoints - cloudprovider.SApsaraEndpoints } ProxySetting proxyapi.SProxySetting } diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index abf212c959..2e8baeb4a0 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -94,9 +94,6 @@ type SCloudaccountCredential struct { // Google服务账号秘钥 (gcp) GCPPrivateKey string `json:"gcp_private_key"` - // 阿里云专有云Endpoints - *SApsaraEndpoints - // 默认区域Id, Apara及HCSO需要此参数 // example: cn-north-2 // required: true diff --git a/pkg/cloudprovider/endpoints.go b/pkg/cloudprovider/endpoints.go index e07d6d6e31..ce4762bafe 100644 --- a/pkg/cloudprovider/endpoints.go +++ b/pkg/cloudprovider/endpoints.go @@ -21,20 +21,6 @@ import ( "yunion.io/x/pkg/utils" ) -type SApsaraEndpoints struct { - EcsEndpoint string `default:"$APSARA_ECS_ENDPOINT" metavar:"APSARA_ECS_ENDPOINT"` - RdsEndpoint string `default:"$APSARA_RDS_ENDPOINT"` - VpcEndpoint string `default:"$APSARA_VPC_ENDPOINT"` - KvsEndpoint string `default:"$APSARA_KVS_ENDPOINT"` - SlbEndpoint string `default:"$APSARA_SLB_ENDPOINT"` - OssEndpoint string `default:"$APSARA_OSS_ENDPOINT"` - StsEndpoint string `default:"$APSARA_STS_ENDPOINT"` - ActionTrailEndpoint string `default:"$APSARA_ACTION_TRAIL_ENDPOINT"` - RamEndpoint string `default:"$APSARA_RAM_ENDPOINT"` - MetricsEndpoint string `default:"$APSRRA_METRICS_ENDPOINT"` - ResourcemanagerEndpoint string `default:"$APSARA_RESOURCEMANAGER_ENDPOINT"` -} - // SHCSOEndpoints 华为私有云endpoints配置 /* endpoint获取方式优先级： diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 57bc066988..354bba8b2f 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -448,9 +448,6 @@ func (manager *SCloudaccountManager) validateCreateData( } var endpointOptions jsonutils.JSONObject - if input.SCloudaccountCredential.SApsaraEndpoints != nil { - endpointOptions = jsonutils.Marshal(input.SCloudaccountCredential.SApsaraEndpoints) - } if input.SCloudaccountCredential.SHCSOEndpoints != nil { endpointOptions = jsonutils.Marshal(input.SCloudaccountCredential.SHCSOEndpoints) diff --git a/pkg/mcclient/options/cloudaccounts.go b/pkg/mcclient/options/cloudaccounts.go index a49389b65c..e76260a463 100644 --- a/pkg/mcclient/options/cloudaccounts.go +++ b/pkg/mcclient/options/cloudaccounts.go @@ -913,7 +913,6 @@ func (opts *SVMwareCloudAccountPrepareNetsOptions) Params() (jsonutils.JSONObjec type SApsaraCloudAccountCreateOptions struct { SCloudAccountCreateBaseOptions - cloudprovider.SApsaraEndpoints Endpoint string SAccessKeyCredential } diff --git a/pkg/multicloud/apsara/apsara.go b/pkg/multicloud/apsara/apsara.go index 3a7ac9eba7..80593454ae 100644 --- a/pkg/multicloud/apsara/apsara.go +++ b/pkg/multicloud/apsara/apsara.go @@ -76,15 +76,12 @@ type ApsaraClientConfig struct { accessKey string accessSecret string debug bool - - endpoints cloudprovider.SApsaraEndpoints } -func NewApsaraClientConfig(accessKey, accessSecret string, endpoint string, endpoints cloudprovider.SApsaraEndpoints) *ApsaraClientConfig { +func NewApsaraClientConfig(accessKey, accessSecret string, endpoint string) *ApsaraClientConfig { cfg := &ApsaraClientConfig{ accessKey: accessKey, accessSecret: accessSecret, - endpoints: endpoints, } return cfg } @@ -110,7 +107,6 @@ type SApsaraClient struct { ownerName string iregions []cloudprovider.ICloudRegion - iBuckets []cloudprovider.ICloudBucket } func NewApsaraClient(cfg *ApsaraClientConfig) (*SApsaraClient, error) { @@ -122,49 +118,10 @@ func NewApsaraClient(cfg *ApsaraClientConfig) (*SApsaraClient, error) { if err != nil { return nil, errors.Wrap(err, "fetchRegions") } - if len(client.endpoints.OssEndpoint) > 0 { - err = client.fetchBuckets() - if err != nil { - return nil, errors.Wrapf(err, "fetchBuckets") - } - if client.debug { - log.Debugf("ClientID: %s ClientName: %s", client.ownerId, client.ownerName) - } - } return &client, nil } func (self *SApsaraClient) getDomain(product string) string { - switch product { - case APSARA_PRODUCT_ECS: - if len(self.endpoints.EcsEndpoint) > 0 { - return self.endpoints.EcsEndpoint - } - case APSARA_PRODUCT_RAM: - if len(self.endpoints.RamEndpoint) > 0 { - return self.endpoints.RamEndpoint - } - case APSARA_PRODUCT_RDS: - if len(self.endpoints.RdsEndpoint) > 0 { - return self.endpoints.RdsEndpoint - } - case APSARA_PRODUCT_SLB: - if len(self.endpoints.SlbEndpoint) > 0 { - return self.endpoints.SlbEndpoint - } - case APSARA_PRODUCT_STS: - if len(self.endpoints.StsEndpoint) > 0 { - return self.endpoints.StsEndpoint - } - case APSARA_PRODUCT_VPC: - if len(self.endpoints.VpcEndpoint) > 0 { - return self.endpoints.VpcEndpoint - } - case APSARA_PRODUCT_KVSTORE: - if len(self.endpoints.KvsEndpoint) > 0 { - return self.endpoints.KvsEndpoint - } - } return self.cpcfg.URL } @@ -233,6 +190,8 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str req.Domain = domain req.Version = version req.ApiName = apiName + req.Scheme = "http" + req.Method = "POST" id := "" if params != nil { for k, v := range params { @@ -242,7 +201,6 @@ func _jsonRequest(client *sdk.Client, domain string, version string, apiName str } } } - req.Scheme = "http" req.GetHeaders()["User-Agent"] = "vendor/yunion-OneCloud@" + v.Get().GitVersion if strings.HasPrefix(apiName, "Describe") && len(id) > 0 { req.GetHeaders()["x-acs-instanceId"] = id @@ -348,6 +306,29 @@ func (self *SApsaraClient) ecsRequest(apiName string, params map[string]string) return productRequest(cli, APSARA_PRODUCT_ECS, domain, APSARA_API_VERSION, apiName, params, self.debug) } +func (self *SApsaraClient) ossRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + cli, err := self.getDefaultClient("") + if err != nil { + return nil, err + } + //pm := map[string]string{} + //for k, v := range params { + // if k != "RegionId" { + // pm[k] = v + // delete(params, k) + // } + //} + //if len(pm) > 0 { + // params["Params"] = jsonutils.Marshal(pm).String() + //} + if _, ok := params["RegionId"]; !ok { + params["RegionId"] = self.cpcfg.DefaultRegion + } + params["ProductName"] = "oss" + params["OpenApiAction"] = apiName + return productRequest(cli, "OneRouter", self.cpcfg.URL, "2018-12-12", "DoOpenApi", params, self.debug) +} + func (self *SApsaraClient) trialRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { cli, err := self.getDefaultClient("") if err != nil { @@ -381,7 +362,7 @@ func (self *SApsaraClient) fetchRegions() error { } // https://help.apsara.com/document_detail/31837.html?spm=a2c4g.11186623.2.6.XqEgD1 -func (client *SApsaraClient) getOssClient(regionId string) (*oss.Client, error) { +func (client *SApsaraClient) getOssClient(endpoint string) (*oss.Client, error) { // NOTE // // oss package as of version 20181116160301-c6838fdc33ed does not @@ -395,7 +376,7 @@ func (client *SApsaraClient) getOssClient(regionId string) (*oss.Client, error) transport, _ := httpClient.Transport.(*http.Transport) httpClient.Transport = cloudprovider.GetCheckTransport(transport, func(req *http.Request) (func(resp *http.Response), error) { if client.cpcfg.ReadOnly { - if req.Method == "GET" { + if req.Method == "GET" || req.Method == "HEAD" { return nil, nil } return nil, errors.Wrapf(cloudprovider.ErrAccountReadOnly, "%s %s", req.Method, req.URL.Path) @@ -405,76 +386,13 @@ func (client *SApsaraClient) getOssClient(regionId string) (*oss.Client, error) cliOpts := []oss.ClientOption{ oss.HTTPClient(httpClient), } - cli, err := oss.New(client.endpoints.OssEndpoint, client.accessKey, client.accessSecret, cliOpts...) + cli, err := oss.New(endpoint, client.accessKey, client.accessSecret, cliOpts...) if err != nil { return nil, errors.Wrap(err, "oss.New") } return cli, nil } -func (self *SApsaraClient) getRegionByRegionId(id string) (cloudprovider.ICloudRegion, error) { - for i := 0; i < len(self.iregions); i += 1 { - if self.iregions[i].GetId() == id { - return self.iregions[i], nil - } - } - return nil, cloudprovider.ErrNotFound -} - -func (self *SApsaraClient) invalidateIBuckets() { - self.iBuckets = nil -} - -func (self *SApsaraClient) getIBuckets() ([]cloudprovider.ICloudBucket, error) { - if len(self.endpoints.OssEndpoint) == 0 { - return nil, fmt.Errorf("empty oss endpoint") - } - if self.iBuckets == nil { - err := self.fetchBuckets() - if err != nil { - return nil, errors.Wrap(err, "fetchBuckets") - } - } - return self.iBuckets, nil -} - -func (self *SApsaraClient) fetchBuckets() error { - osscli, err := self.getOssClient("") - if err != nil { - return errors.Wrap(err, "self.getOssClient") - } - result, err := osscli.ListBuckets() - if err != nil { - return errors.Wrap(err, "oss.ListBuckets") - } - - self.ownerId = result.Owner.ID - self.ownerName = result.Owner.DisplayName - - ret := make([]cloudprovider.ICloudBucket, 0) - for _, bInfo := range result.Buckets { - regionId := bInfo.Location - if strings.HasPrefix(regionId, "oss-") { - regionId = regionId[4:] - } - region, err := self.getRegionByRegionId(regionId) - if err != nil { - log.Errorf("cannot find bucket %s region %s", bInfo.Name, regionId) - continue - } - b := SBucket{ - region: region.(*SRegion), - Name: bInfo.Name, - Location: bInfo.Location, - CreationDate: bInfo.CreationDate, - StorageClass: bInfo.StorageClass, - } - ret = append(ret, &b) - } - self.iBuckets = ret - return nil -} - func (self *SApsaraClient) GetRegions() []SRegion { regions := make([]SRegion, len(self.iregions)) for i := 0; i < len(regions); i += 1 { diff --git a/pkg/multicloud/apsara/bucket.go b/pkg/multicloud/apsara/bucket.go index 79bed48267..f934099f48 100644 --- a/pkg/multicloud/apsara/bucket.go +++ b/pkg/multicloud/apsara/bucket.go @@ -41,6 +41,8 @@ type SBucket struct { CreationDate time.Time StorageClass string + ExtranetEndpoint string + IntranetEndpoint string DepartmentInfo } @@ -52,9 +54,13 @@ func (b *SBucket) GetName() string { return b.Name } +func (self *SBucket) GetOssClient() (*oss.Client, error) { + return self.region.GetOssClient() +} + func (b *SBucket) GetAcl() cloudprovider.TBucketACLType { acl := cloudprovider.ACLPrivate - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { log.Errorf("b.region.GetOssClient fail %s", err) return acl @@ -87,7 +93,7 @@ func (b *SBucket) GetStorageClass() string { func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { return []cloudprovider.SBucketAccessUrl{ { - Url: fmt.Sprintf("%s.%s", b.Name, b.region.client.endpoints.OssEndpoint), + Url: fmt.Sprintf("%s.%s", b.Name, b.ExtranetEndpoint), Description: "ExtranetEndpoint", Primary: true, }, @@ -103,9 +109,8 @@ func (b *SBucket) GetStats() cloudprovider.SBucketStats { } func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { - log.Errorf("b.region.GetOssClient fail %s", err) return errors.Wrap(err, "b.region.GetOssClient") } acl, err := str2Acl(string(aclStr)) @@ -121,7 +126,7 @@ func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error { func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { result := cloudprovider.SListObjectResult{} - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return result, errors.Wrap(err, "GetOssClient") } @@ -200,7 +205,7 @@ func metaOpts(opts []oss.Option, meta http.Header) []oss.Option { } func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return errors.Wrap(err, "GetOssClient") } @@ -234,7 +239,7 @@ func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, si } func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) (string, error) { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return "", errors.Wrap(err, "GetOssClient") } @@ -269,7 +274,7 @@ func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl } func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64, offset, totalSize int64) (string, error) { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return "", errors.Wrap(err, "GetOssClient") } @@ -293,7 +298,7 @@ func (b *SBucket) UploadPart(ctx context.Context, key string, uploadId string, p } func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return errors.Wrap(err, "GetOssClient") } @@ -324,7 +329,7 @@ func (b *SBucket) CompleteMultipartUpload(ctx context.Context, key string, uploa } func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return errors.Wrap(err, "GetOssClient") } @@ -345,7 +350,7 @@ func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId } func (b *SBucket) DeleteObject(ctx context.Context, key string) error { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return errors.Wrap(err, "GetOssClient") } @@ -364,7 +369,7 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s if method != "GET" && method != "PUT" && method != "DELETE" { return "", errors.Error("unsupported method") } - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return "", errors.Wrap(err, "GetOssClient") } @@ -380,7 +385,7 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s } func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return errors.Wrap(err, "GetOssClient") } @@ -415,7 +420,7 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src } func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return nil, errors.Wrap(err, "GetOssClient") } @@ -435,7 +440,7 @@ func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprov } func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { - osscli, err := b.region.GetOssClient() + osscli, err := b.GetOssClient() if err != nil { return "", errors.Wrap(err, "GetOssClient") } diff --git a/pkg/multicloud/apsara/disk.go b/pkg/multicloud/apsara/disk.go index d635bda787..e3e3a655d5 100644 --- a/pkg/multicloud/apsara/disk.go +++ b/pkg/multicloud/apsara/disk.go @@ -17,7 +17,6 @@ package apsara import ( "context" "fmt" - "runtime/debug" "time" "yunion.io/x/jsonutils" @@ -273,7 +272,6 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz } func (self *SRegion) getDisk(diskId string) (*SDisk, error) { - debug.PrintStack() disks, total, err := self.GetDisks("", "", "", []string{diskId}, 0, 1) if err != nil { return nil, err diff --git a/pkg/multicloud/apsara/image.go b/pkg/multicloud/apsara/image.go index f2b6b4c7be..7d06769aa2 100644 --- a/pkg/multicloud/apsara/image.go +++ b/pkg/multicloud/apsara/image.go @@ -20,8 +20,6 @@ import ( "strings" "time" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -213,11 +211,11 @@ type ImageExportTask struct { TaskId string } -func (self *SRegion) ExportImage(imageId string, bucket *oss.Bucket) (*ImageExportTask, error) { +func (self *SRegion) ExportImage(imageId string, bucketName string) (*ImageExportTask, error) { params := make(map[string]string) params["RegionId"] = self.RegionId params["ImageId"] = imageId - params["OssBucket"] = bucket.BucketName + params["OssBucket"] = bucketName params["OssPrefix"] = fmt.Sprintf("%sexport", strings.Replace(imageId, "-", "", -1)) if body, err := self.ecsRequest("ExportImage", params); err != nil { diff --git a/pkg/multicloud/apsara/objects.go b/pkg/multicloud/apsara/objects.go index 9c488b1ff4..5a15ddf1f2 100644 --- a/pkg/multicloud/apsara/objects.go +++ b/pkg/multicloud/apsara/objects.go @@ -42,7 +42,7 @@ func (o *SObject) GetIBucket() cloudprovider.ICloudBucket { func (o *SObject) GetAcl() cloudprovider.TBucketACLType { acl := cloudprovider.ACLPrivate - osscli, err := o.bucket.region.GetOssClient() + osscli, err := o.bucket.GetOssClient() if err != nil { log.Errorf("o.bucket.region.GetOssClient error %s", err) return acl @@ -69,7 +69,7 @@ func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error { if err != nil { return errors.Wrap(err, "str2Acl") } - osscli, err := o.bucket.region.GetOssClient() + osscli, err := o.bucket.GetOssClient() if err != nil { return errors.Wrap(err, "o.bucket.region.GetOssClient") } @@ -88,9 +88,9 @@ func (o *SObject) GetMeta() http.Header { if o.Meta != nil { return o.Meta } - osscli, err := o.bucket.region.GetOssClient() + osscli, err := o.bucket.GetOssClient() if err != nil { - log.Errorf("o.bucket.region.GetOssClient error %s", err) + log.Errorf("o.bucket.GetOssClient error %s", err) return nil } bucket, err := osscli.Bucket(o.bucket.Name) diff --git a/pkg/multicloud/apsara/project.go b/pkg/multicloud/apsara/project.go index 0b1f163142..05f5274fd2 100644 --- a/pkg/multicloud/apsara/project.go +++ b/pkg/multicloud/apsara/project.go @@ -26,10 +26,11 @@ import ( ) type DepartmentInfo struct { - Department string - DepartmentName string - ResourceGroup string - ResourceGroupId string + Department string + DepartmentName string + ResourceGroup string + ResourceGroupName string + ResourceGroupId string } func (self DepartmentInfo) GetProjectId() string { diff --git a/pkg/multicloud/apsara/provider/provider.go b/pkg/multicloud/apsara/provider/provider.go index 1f8d439103..fbdd9a40eb 100644 --- a/pkg/multicloud/apsara/provider/provider.go +++ b/pkg/multicloud/apsara/provider/provider.go @@ -83,16 +83,11 @@ func (self *SApsaraProviderFactory) ValidateUpdateCloudaccountCredential(ctx con } func (self *SApsaraProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { - endpoints := cloudprovider.SApsaraEndpoints{} - if cfg.Options != nil { - cfg.Options.Unmarshal(&endpoints) - } client, err := apsara.NewApsaraClient( apsara.NewApsaraClientConfig( cfg.Account, cfg.Secret, cfg.URL, - endpoints, ).CloudproviderConfig(cfg), ) if err != nil { diff --git a/pkg/multicloud/apsara/region.go b/pkg/multicloud/apsara/region.go index cd0004f70f..e8c7942c55 100644 --- a/pkg/multicloud/apsara/region.go +++ b/pkg/multicloud/apsara/region.go @@ -38,7 +38,6 @@ type SRegion struct { client *SApsaraClient sdkClient *sdk.Client - ossClient *oss.Client RegionId string LocalName string @@ -58,6 +57,8 @@ type SRegion struct { latitude float64 longitude float64 fetchLocation bool + + ossEndpoint string } func (self *SRegion) GetILoadBalancerBackendGroups() ([]cloudprovider.ICloudLoadbalancerBackendGroup, error) { @@ -79,20 +80,27 @@ func (self *SRegion) getSdkClient() (*sdk.Client, error) { return self.sdkClient, nil } +func (self *SRegion) GetOssClient() (*oss.Client, error) { + if len(self.ossEndpoint) == 0 { + _, err := self.GetBuckets() + if err != nil { + return nil, err + } + if len(self.ossEndpoint) == 0 { + return nil, fmt.Errorf("no available buckets") + } + } + return self.client.getOssClient(self.ossEndpoint) +} + func (self *SRegion) productRequest(client *sdk.Client, product, domain, apiVersion, apiName string, params map[string]string, debug bool) (jsonutils.JSONObject, error) { params["Product"] = product return jsonRequest(client, domain, apiVersion, apiName, params, debug) } -func (self *SRegion) GetOssClient() (*oss.Client, error) { - if self.ossClient == nil { - cli, err := self.client.getOssClient(self.RegionId) - if err != nil { - return nil, errors.Wrap(err, "self.client.getOssClient") - } - self.ossClient = cli - } - return self.ossClient, nil +func (self *SRegion) ossRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { + params["RegionId"] = self.RegionId + return self.client.ossRequest(apiName, params) } func (self *SRegion) ecsRequest(apiName string, params map[string]string) (jsonutils.JSONObject, error) { @@ -933,22 +941,37 @@ func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAc return iAcl, region.AddAccessControlListEntry(aclId, acl.Entrys) } -func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { - iBuckets, err := region.client.getIBuckets() +func (self *SRegion) GetBuckets() ([]SBucket, error) { + resp, err := self.ossRequest("GetService", map[string]string{}) if err != nil { - return nil, errors.Wrap(err, "getIBuckets") + return nil, err } - ret := make([]cloudprovider.ICloudBucket, 0) - for i := range iBuckets { - loc := iBuckets[i].GetLocation() - // remove oss- prefix - if strings.HasPrefix(loc, "oss-") { - loc = loc[4:] + if !resp.Contains("Data") { + if self.client.cpcfg.UpdatePermission != nil { + self.client.cpcfg.UpdatePermission("oss", "ListBuckets") } - if loc != region.GetId() { - continue - } - ret = append(ret, iBuckets[i]) + return []SBucket{}, nil + } + ret := []SBucket{} + err = resp.Unmarshal(&ret, "Data", "ListAllMyBucketsResult", "Buckets", "Bucket") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + if len(ret) > 0 { + self.ossEndpoint = ret[0].IntranetEndpoint + } + return ret, nil +} + +func (self *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) { + buckets, err := self.GetBuckets() + if err != nil { + return nil, err + } + ret := []cloudprovider.ICloudBucket{} + for i := range buckets { + buckets[i].region = self + ret = append(ret, &buckets[i]) } return ret, nil } @@ -981,8 +1004,8 @@ func str2Acl(aclStr string) (oss.ACLType, error) { return acl, nil } -func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { - osscli, err := region.GetOssClient() +func (self *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error { + osscli, err := self.GetOssClient() if err != nil { return errors.Wrap(err, "region.GetOssClient") } @@ -1001,12 +1024,7 @@ func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr } opts = append(opts, oss.ACL(acl)) } - err = osscli.CreateBucket(name, opts...) - if err != nil { - return errors.Wrap(err, "oss.CreateBucket") - } - region.client.invalidateIBuckets() - return nil + return osscli.CreateBucket(name, opts...) } func ossErrorCode(err error) int { @@ -1019,52 +1037,48 @@ func ossErrorCode(err error) int { return -1 } -func (region *SRegion) DeleteIBucket(name string) error { - osscli, err := region.GetOssClient() +func (self *SRegion) DeleteIBucket(name string) error { + cli, err := self.GetOssClient() if err != nil { - return errors.Wrap(err, "region.GetOssClient") + return err } - err = osscli.DeleteBucket(name) - if err != nil { - if ossErrorCode(err) == 404 { - return nil - } - return errors.Wrap(err, "DeleteBucket") - } - region.client.invalidateIBuckets() - return nil + return cli.DeleteBucket(name) } -func (region *SRegion) IBucketExist(name string) (bool, error) { - osscli, err := region.GetOssClient() +func (self *SRegion) GetBucket(name string) (*SBucket, error) { + cli, err := self.GetOssClient() if err != nil { - return false, errors.Wrap(err, "region.GetOssClient") + return nil, err } - exist, err := osscli.IsBucketExist(name) + bucket, err := cli.GetBucketInfo(name) if err != nil { - return false, errors.Wrap(err, "IsBucketExist") + return nil, err } - return exist, nil -} -func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { - osscli, err := region.GetOssClient() - if err != nil { - return nil, errors.Wrap(err, "region.GetOssClient") - } - bi, err := osscli.GetBucketInfo(name) - if err != nil { - return nil, errors.Wrap(err, "Bucket") - } - bInfo := bi.BucketInfo - b := SBucket{ - region: region, + bInfo := bucket.BucketInfo + return &SBucket{ + region: self, Name: bInfo.Name, Location: bInfo.Location, CreationDate: bInfo.CreationDate, StorageClass: bInfo.StorageClass, + }, nil +} + +func (self *SRegion) IBucketExist(name string) (bool, error) { + _, err := self.GetBucket(name) + if err != nil { + return false, err } - return &b, nil + return true, nil +} + +func (self *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) { + bucket, err := self.GetBucket(name) + if err != nil { + return nil, err + } + return bucket, nil } func (region *SRegion) GetIBucketByName(name string) (cloudprovider.ICloudBucket, error) { diff --git a/pkg/multicloud/apsara/shell/bucket.go b/pkg/multicloud/apsara/shell/bucket.go index f44f1c4081..389827e628 100644 --- a/pkg/multicloud/apsara/shell/bucket.go +++ b/pkg/multicloud/apsara/shell/bucket.go @@ -14,8 +14,25 @@ package shell -import "yunion.io/x/onecloud/pkg/multicloud/objectstore" +import ( + "yunion.io/x/onecloud/pkg/multicloud/apsara" + "yunion.io/x/onecloud/pkg/multicloud/objectstore" + "yunion.io/x/onecloud/pkg/util/shellutils" +) func init() { objectstore.S3Shell() + + type BucketOptions struct { + BUCKET string + } + shellutils.R(&BucketOptions{}, "bucket-show", "Show bucket", func(cli *apsara.SRegion, args *BucketOptions) error { + bucket, e := cli.GetBucket(args.BUCKET) + if e != nil { + return e + } + printObject(bucket) + return nil + }) + } diff --git a/pkg/multicloud/apsara/shell/image.go b/pkg/multicloud/apsara/shell/image.go index 52b8f997fd..178f8f3761 100644 --- a/pkg/multicloud/apsara/shell/image.go +++ b/pkg/multicloud/apsara/shell/image.go @@ -78,22 +78,7 @@ func init() { } shellutils.R(&ImageExportOptions{}, "image-export", "Export image", func(cli *apsara.SRegion, args *ImageExportOptions) error { - oss, err := cli.GetOssClient() - if err != nil { - return err - } - exist, err := oss.IsBucketExist(args.BUCKET) - if err != nil { - return err - } - if !exist { - return fmt.Errorf("not exist bucket %s", args.BUCKET) - } - bucket, err := oss.Bucket(args.BUCKET) - if err != nil { - return err - } - task, err := cli.ExportImage(args.ID, bucket) + task, err := cli.ExportImage(args.ID, args.BUCKET) if err != nil { return err } diff --git a/pkg/multicloud/apsara/shell/oss.go b/pkg/multicloud/apsara/shell/oss.go deleted file mode 100644 index ef8d0a3f69..0000000000 --- a/pkg/multicloud/apsara/shell/oss.go +++ /dev/null @@ -1,202 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package shell - -import ( - "fmt" - "os" - "path/filepath" - - osslib "github.com/aliyun/aliyun-oss-go-sdk/oss" - - "yunion.io/x/onecloud/pkg/multicloud/apsara" - "yunion.io/x/onecloud/pkg/util/fileutils2" - "yunion.io/x/onecloud/pkg/util/shellutils" -) - -type progressListener struct { -} - -func (this *progressListener) ProgressChanged(event *osslib.ProgressEvent) { - switch event.EventType { - case osslib.TransferStartedEvent: - fmt.Printf("\n") - case osslib.TransferDataEvent: - fmt.Printf("Progess: %f%%\r", (float64(event.ConsumedBytes) * 100.0 / float64(event.TotalBytes))) - case osslib.TransferCompletedEvent: - fmt.Printf("Transfer complete!\n") - case osslib.TransferFailedEvent: - fmt.Printf("Transfer failed!\n") - default: - fmt.Printf("Unknonw event type %d\n", event.EventType) - } -} - -func str2AclType(aclStr string) osslib.ACLType { - switch aclStr { - case "public-rw": - return osslib.ACLPublicReadWrite - case "public-read": - return osslib.ACLPublicRead - default: - return osslib.ACLPrivate - } -} - -func init() { - type OssListOptions struct { - } - shellutils.R(&OssListOptions{}, "oss-list", "List OSS buckets", func(cli *apsara.SRegion, args *OssListOptions) error { - buckets, err := cli.GetIBuckets() - if err != nil { - return err - } - printList(buckets, len(buckets), 0, 50, nil) - return nil - }) - - type OssListBucketOptions struct { - BUCKET string `help:"bucket name"` - } - - shellutils.R(&OssListBucketOptions{}, "oss-list-bucket", "List content of a OSS bucket", func(cli *apsara.SRegion, args *OssListBucketOptions) error { - oss, err := cli.GetOssClient() - if err != nil { - return err - } - bucket, err := oss.Bucket(args.BUCKET) - if err != nil { - return err - } - result, err := bucket.ListObjects() - if err != nil { - return err - } - printList(result.Objects, len(result.Objects), 0, len(result.Objects), nil) - return nil - }) - - type OssCreateBucketOptions struct { - BUCKET string `help:"bucket name"` - StorageClass string `help:"storage class" choices:"Standard|IA|Archive"` - - Acl string `help:"ACL" choices:"private|public-read|public-read-write"` - } - shellutils.R(&OssCreateBucketOptions{}, "oss-create-bucket", "Create a OSS bucket", func(cli *apsara.SRegion, args *OssCreateBucketOptions) error { - err := cli.CreateIBucket(args.BUCKET, args.StorageClass, args.Acl) - if err != nil { - return err - } - return nil - }) - - type OssDeleteBucketOptions struct { - BUCKET string `help:"bucket name"` - } - shellutils.R(&OssDeleteBucketOptions{}, "oss-delete-bucket", "Delete a OSS bucket", func(cli *apsara.SRegion, args *OssDeleteBucketOptions) error { - err := cli.DeleteIBucket(args.BUCKET) - if err != nil { - return err - } - return nil - }) - - type OssUploadOptions struct { - BUCKET string `help:"bucket name"` - KEY string `help:"Object key"` - FILE string `help:"Local file path"` - Progress bool `help:"show progress"` - Acl string `help:"Object ACL" choices:"private|public-read|public-rw"` - } - shellutils.R(&OssUploadOptions{}, "oss-upload", "Upload a file to a OSS bucket", func(cli *apsara.SRegion, args *OssUploadOptions) error { - oss, err := cli.GetOssClient() - if err != nil { - return err - } - bucket, err := oss.Bucket(args.BUCKET) - if err != nil { - return err - } - - options := make([]osslib.Option, 0) - if args.Progress { - listener := progressListener{} - options = append(options, osslib.Progress(&listener)) - } - if len(args.Acl) > 0 { - options = append(options, osslib.ObjectACL(str2AclType(args.Acl))) - } - if fileutils2.IsFile(args.FILE) { - err = bucket.UploadFile(args.KEY, args.FILE, 4*1024*1024, options...) - return err - } else if fileutils2.IsDir(args.FILE) { - return filepath.Walk(args.FILE, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.Mode().IsRegular() { - rel, _ := filepath.Rel(args.FILE, path) - src := path - dst := filepath.Join(args.KEY, rel) - fmt.Println("upload", src, "to", dst) - uploadErr := bucket.UploadFile(dst, src, - 4*1024*1024, options...) - if uploadErr != nil { - return uploadErr - } - } - return nil - }) - } else { - return fmt.Errorf("Unsupported file type %s", args.FILE) - } - }) - - type OssObjectAclOptions struct { - BUCKET string `help:"bucket name"` - KEY string `help:"object key"` - ACL string `help:"ACL" choices:"private|public-read|public-rw"` - } - shellutils.R(&OssObjectAclOptions{}, "oss-set-acl", "Set acl for a object", func(cli *apsara.SRegion, args *OssObjectAclOptions) error { - oss, err := cli.GetOssClient() - if err != nil { - return err - } - bucket, err := oss.Bucket(args.BUCKET) - if err != nil { - return err - } - err = bucket.SetObjectACL(args.KEY, str2AclType(args.ACL)) - return err - }) - - type OssDeleteOptions struct { - BUCKET string `help:"bucket name"` - KEY string `help:"Object key"` - } - - shellutils.R(&OssDeleteOptions{}, "oss-delete", "Delete a file from a OSS bucket", func(cli *apsara.SRegion, args *OssDeleteOptions) error { - oss, err := cli.GetOssClient() - if err != nil { - return err - } - bucket, err := oss.Bucket(args.BUCKET) - if err != nil { - return err - } - err = bucket.DeleteObject(args.KEY) - return err - }) -} diff --git a/pkg/multicloud/apsara/storagecache.go b/pkg/multicloud/apsara/storagecache.go index 80369f49f4..ed5921d3bf 100644 --- a/pkg/multicloud/apsara/storagecache.go +++ b/pkg/multicloud/apsara/storagecache.go @@ -17,8 +17,6 @@ package apsara import ( "context" "fmt" - "io/ioutil" - "os" "strings" "time" @@ -260,35 +258,6 @@ func (self *SStoragecache) CreateIImage(snapshoutId, imageName, osType, imageDes } } -func (self *SRegion) CheckBucket(bucketName string) (*oss.Bucket, error) { - return self.checkBucket(bucketName) -} - -func (self *SRegion) checkBucket(bucketName string) (*oss.Bucket, error) { - oss, err := self.GetOssClient() - if err != nil { - log.Errorf("GetOssClient err %s", err) - return nil, err - } - if exist, err := oss.IsBucketExist(bucketName); err != nil { - log.Errorf("IsBucketExist err %s", err) - return nil, err - } else if !exist { - log.Debugf("Bucket %s not exists, to create ...", bucketName) - if err := oss.CreateBucket(bucketName); err != nil { - log.Errorf("Create bucket error %s", err) - return nil, err - } - } - log.Debugf("Bucket %s exists", bucketName) - if bucket, err := oss.Bucket(bucketName); err != nil { - log.Errorf("Bucket error %s %s", bucketName, err) - return nil, err - } else { - return bucket, nil - } -} - func (self *SRegion) CreateImage(snapshoutId, imageName, imageDesc string) (string, error) { return self.createIImage(snapshoutId, imageName, imageDesc) } @@ -301,10 +270,6 @@ func (self *SRegion) createIImage(snapshoutId, imageName, imageDesc string) (str params["ImageName"] = imageName params["Description"] = imageDesc - if _, err := self.checkBucket(params["OssBucket"]); err != nil { - return "", err - } - if body, err := self.ecsRequest("CreateImage", params); err != nil { log.Errorf("CreateImage fail %s", err) return "", err @@ -342,42 +307,7 @@ func (listener *OssProgressListener) ProgressChanged(event *oss.ProgressEvent) { } func (self *SStoragecache) downloadImage(userCred mcclient.TokenCredential, imageId string, extId string, path string) (jsonutils.JSONObject, error) { - err := self.region.GetClient().EnableImageExport() - if err != nil { - log.Errorf("fail to enable export privileges: %s", err) - return nil, err - } - - tmpImageFile, err := ioutil.TempFile(path, extId) - if err != nil { - return nil, err - } - defer tmpImageFile.Close() - defer os.Remove(tmpImageFile.Name()) - bucketName := strings.ToLower(fmt.Sprintf("imgcache-%s", self.region.GetId())) - if bucket, err := self.region.checkBucket(bucketName); err != nil { - return nil, err - } else if _, err := self.region.GetImage(extId); err != nil { - return nil, err - } else if task, err := self.region.ExportImage(extId, bucket); err != nil { - return nil, err - } else if err := self.region.waitTaskStatus(ExportImageTask, task.TaskId, TaskStatusFinished, 15*time.Second, 3600*time.Second); err != nil { - return nil, err - } else if imageList, err := bucket.ListObjects(oss.Prefix(fmt.Sprintf("%sexport", strings.Replace(extId, "-", "", -1)))); err != nil { - return nil, err - } else if len(imageList.Objects) != 1 { - return nil, fmt.Errorf("exported image not find") - } else if err := bucket.DownloadFile(imageList.Objects[0].Key, tmpImageFile.Name(), 12*1024*1024, oss.Routines(3), oss.Progress(&OssProgressListener{})); err != nil { - return nil, err - } else { - s := auth.GetAdminSession(context.Background(), options.Options.Region, "") - params := jsonutils.Marshal(map[string]string{"image_id": imageId, "disk-format": "raw"}) - if result, err := modules.Images.Upload(s, params, tmpImageFile, imageList.Objects[0].Size); err != nil { - return nil, err - } else { - return result, nil - } - } + return nil, cloudprovider.ErrNotImplemented } func (region *SRegion) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) { diff --git a/pkg/multicloud/apsara/vpc.go b/pkg/multicloud/apsara/vpc.go index f1011a595f..86c70bcb1e 100644 --- a/pkg/multicloud/apsara/vpc.go +++ b/pkg/multicloud/apsara/vpc.go @@ -62,6 +62,8 @@ type SVpc struct { VSwitchIds SVSwitchIds VpcId string VpcName string + + DepartmentInfo } func (self *SVpc) GetId() string { @@ -107,6 +109,28 @@ func (self *SVpc) GetRegion() cloudprovider.ICloudRegion { return self.region } +func (self *SVpc) GetSysTags() map[string]string { + tags := self.ApsaraTags.GetSysTags() + if len(self.ResourceGroup) > 0 { + tags["ResourceGroup"] = self.ResourceGroup + } + if len(self.ResourceGroupId) > 0 { + tags["ResourceGroupId"] = self.ResourceGroupId + } + if len(self.Department) > 0 { + tags["Department"] = self.Department + } + if len(self.DepartmentName) > 0 { + tags["DepartmentName"] = self.DepartmentName + } + if len(self.ResourceGroupName) > 0 { + groupName := strings.TrimPrefix(self.ResourceGroupName, "ResourceSet(") + groupName = strings.TrimSuffix(groupName, ")") + tags["ResourceGroupName"] = groupName + } + return tags +} + func (self *SVpc) addWire(wire *SWire) { if self.iwires == nil { self.iwires = make([]cloudprovider.ICloudWire, 0)