From b26249811abf7c62f07a4ec754ded8cad569c8b1 Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 11 Dec 2020 10:34:37 +0800 Subject: [PATCH 1/2] fix(region): server login info& keypair import input parameter trimspace --- pkg/compute/models/keypairs.go | 2 ++ pkg/mcclient/modules/mod_servers.go | 1 + 2 files changed, 3 insertions(+) diff --git a/pkg/compute/models/keypairs.go b/pkg/compute/models/keypairs.go index 2edc7a94d5..621a7ac6d2 100644 --- a/pkg/compute/models/keypairs.go +++ b/pkg/compute/models/keypairs.go @@ -16,6 +16,7 @@ package models import ( "context" + "strings" "golang.org/x/crypto/ssh" @@ -148,6 +149,7 @@ func (self *SKeypair) GetLinkedGuestsCount() (int, error) { } func (manager *SKeypairManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.KeypairCreateInput) (api.KeypairCreateInput, error) { + input.PublicKey = strings.TrimSpace(input.PublicKey) if len(input.PublicKey) == 0 { if len(input.Scheme) == 0 { input.Scheme = api.KEYPAIRE_SCHEME_RSA diff --git a/pkg/mcclient/modules/mod_servers.go b/pkg/mcclient/modules/mod_servers.go index 876705b6e7..b95db08964 100644 --- a/pkg/mcclient/modules/mod_servers.go +++ b/pkg/mcclient/modules/mod_servers.go @@ -64,6 +64,7 @@ func (this *ServerManager) GetLoginInfo(s *mcclient.ClientSession, id string, pa } if params != nil && !gotypes.IsNil(params) { privateKey, _ := params.GetString("private_key") + privateKey = strings.TrimSpace(privateKey) if len(privateKey) > 0 { passwd, e = seclib2.DecryptBase64(privateKey, loginKey) if e != nil { From c862f06996b0adbccb8a6ca90c42fd13efa619a9 Mon Sep 17 00:00:00 2001 From: TangBin Date: Fri, 11 Dec 2020 17:24:24 +0800 Subject: [PATCH 2/2] feat(region): support sync skus by cloudregion --- cmd/climc/shell/compute/dbinstance_skus.go | 16 +++ cmd/climc/shell/compute/elasticcache.go | 18 +++ cmd/climc/shell/compute/skus.go | 16 +++ pkg/apis/compute/guests.go | 2 + pkg/apis/compute/sku_const.go | 14 +++ pkg/apis/compute/vpcs_ovn.go | 2 + pkg/cloudprovider/resources.go | 9 ++ pkg/compute/models/cloudsync.go | 4 +- pkg/compute/models/dbinstance_skus.go | 16 +++ pkg/compute/models/elasticcache_skus.go | 16 +++ pkg/compute/models/elasticipresource.go | 23 ++++ pkg/compute/models/elasticips.go | 10 ++ pkg/compute/models/guest_actions.go | 13 ++- pkg/compute/models/guest_queries.go | 28 +++-- pkg/compute/models/skuresource.go | 109 ++++++++++++++++++ pkg/compute/models/skus.go | 16 +++ pkg/compute/models/skus_tools.go | 25 +++- pkg/compute/models/vpcs.go | 44 ++++++- pkg/compute/regiondrivers/managedvirtual.go | 13 +++ .../tasks/cloud_region_sync_skus_task.go | 61 ++++++++++ pkg/mcclient/options/skus.go | 35 ++++++ pkg/multicloud/aws/internetgateway.go | 51 ++++++++ pkg/multicloud/aws/region.go | 17 +++ pkg/multicloud/aws/vpc.go | 59 ++++++++++ pkg/multicloud/region_base.go | 4 + pkg/multicloud/vpc_base.go | 13 +++ 26 files changed, 611 insertions(+), 23 deletions(-) create mode 100644 pkg/compute/models/elasticipresource.go create mode 100644 pkg/compute/models/skuresource.go create mode 100644 pkg/compute/tasks/cloud_region_sync_skus_task.go create mode 100644 pkg/mcclient/options/skus.go create mode 100644 pkg/multicloud/aws/internetgateway.go diff --git a/cmd/climc/shell/compute/dbinstance_skus.go b/cmd/climc/shell/compute/dbinstance_skus.go index 4febdbb481..96eeb3f0ec 100644 --- a/cmd/climc/shell/compute/dbinstance_skus.go +++ b/cmd/climc/shell/compute/dbinstance_skus.go @@ -16,6 +16,7 @@ package compute import ( "yunion.io/x/onecloud/cmd/climc/shell" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" ) @@ -24,4 +25,19 @@ func init() { cmd := shell.NewResourceCmd(&modules.DBInstanceSkus).WithKeyword("dbinstance-sku") cmd.List(&options.DBInstanceSkuListOption{}) cmd.Show(&options.DBInstanceSkuIdOption{}) + cmd.PerformClass("sync-skus", &options.SkuSyncOptions{}) + + R(&options.SkuTaskQueryOptions{}, "dbinstance-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error { + params, err := args.Params() + if err != nil { + return err + } + + result, err := modules.DBInstanceSkus.Get(s, "sync-tasks", params) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/cmd/climc/shell/compute/elasticcache.go b/cmd/climc/shell/compute/elasticcache.go index 851a42db1f..de4b290f90 100644 --- a/cmd/climc/shell/compute/elasticcache.go +++ b/cmd/climc/shell/compute/elasticcache.go @@ -19,12 +19,16 @@ import ( "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/cmd/climc/shell" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" ) func init() { + cmd := shell.NewResourceCmd(&modules.ElasticcacheSkus).WithKeyword("elastic-cache-sku") + cmd.PerformClass("sync-skus", &options.SkuSyncOptions{}) + R(&options.BaseListOptions{}, "elastic-cache-list", "List elastisc cache instance", func(s *mcclient.ClientSession, opts *options.BaseListOptions) error { params, err := options.ListStructToParams(opts) if err != nil { @@ -398,6 +402,20 @@ func init() { return nil }) + R(&options.SkuTaskQueryOptions{}, "elastic-cache-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error { + params, err := args.Params() + if err != nil { + return err + } + + result, err := modules.ElasticcacheSkus.Get(s, "sync-tasks", params) + if err != nil { + return err + } + printObject(result) + return nil + }) + R(&options.ResourceMetadataOptions{}, "elastic-cache-add-tag", "Set tag of a server", func(s *mcclient.ClientSession, opts *options.ResourceMetadataOptions) error { params, err := opts.Params() if err != nil { diff --git a/cmd/climc/shell/compute/skus.go b/cmd/climc/shell/compute/skus.go index 9da6265a30..bbdb8aeb4f 100644 --- a/cmd/climc/shell/compute/skus.go +++ b/cmd/climc/shell/compute/skus.go @@ -16,6 +16,7 @@ package compute import ( "yunion.io/x/onecloud/cmd/climc/shell" + "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/mcclient/options" ) @@ -31,4 +32,19 @@ func init() { cmd.Update(&options.ServerSkusUpdateOptions{}) cmd.PerformWithKeyword("cache", "cache-sku", &options.ServerSkusCacheOptions{}) cmd.ClassShow(&options.ServerSkusListOptions{}) + cmd.PerformClass("sync-skus", &options.SkuSyncOptions{}) + + R(&options.SkuTaskQueryOptions{}, "server-sku-sync-task-show", "Show details of skus sync tasks", func(s *mcclient.ClientSession, args *options.SkuTaskQueryOptions) error { + params, err := args.Params() + if err != nil { + return err + } + + result, err := modules.ServerSkus.Get(s, "sync-tasks", params) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/pkg/apis/compute/guests.go b/pkg/apis/compute/guests.go index 2a92c2c24d..fe6841836a 100644 --- a/pkg/apis/compute/guests.go +++ b/pkg/apis/compute/guests.go @@ -217,6 +217,8 @@ type ServerDetails struct { Vpc string `json:"vpc"` // 归属VPC ID VpcId string `json:"vpc_id"` + // Vpc外网访问模式 + VpcExternalAccessMode string `json:"vpc_external_access_mode"` // 关联安全组列表 Secgroups []apis.StandaloneShortDesc `json:"secgroups"` diff --git a/pkg/apis/compute/sku_const.go b/pkg/apis/compute/sku_const.go index 267c94b8e3..7c26f3c8b2 100644 --- a/pkg/apis/compute/sku_const.go +++ b/pkg/apis/compute/sku_const.go @@ -155,3 +155,17 @@ type DBInstanceSkuListInput struct { Zone2 []string `json:"zone2"` Zone3 []string `json:"zone3"` } + +type SkuSyncInput struct { + // 云平台名称 + // example: Google + Provider string `json:"provider,omitempty"` + + // 区域ID + CloudregionIds []string `json:"cloudregion_ids"` +} + +type SkuTaskQueryInput struct { + // 异步任务ID + TaskIds []string `json:"task_ids"` +} diff --git a/pkg/apis/compute/vpcs_ovn.go b/pkg/apis/compute/vpcs_ovn.go index abbc0d6033..cac44e9814 100644 --- a/pkg/apis/compute/vpcs_ovn.go +++ b/pkg/apis/compute/vpcs_ovn.go @@ -22,6 +22,7 @@ const ( VPC_EXTERNAL_ACCESS_MODE_DISTGW = "distgw" // distgw only VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW = "eip-distgw" // eip when available, distgw otherwise VPC_EXTERNAL_ACCESS_MODE_EIP = "eip" // eip only + VPC_EXTERNAL_ACCESS_MODE_NONE = "none" // no external access ) var ( @@ -29,6 +30,7 @@ var ( VPC_EXTERNAL_ACCESS_MODE_DISTGW, VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW, VPC_EXTERNAL_ACCESS_MODE_EIP, + VPC_EXTERNAL_ACCESS_MODE_NONE, } ) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 650f482abd..f5de668010 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -77,6 +77,7 @@ type ICloudRegion interface { CreateISecurityGroup(conf *SecurityGroupCreateInput) (ICloudSecurityGroup, error) CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error) + CreateInternetGateway() (ICloudInternetGateway, error) CreateEIP(eip *SEip) (ICloudEIP, error) GetISnapshots() ([]ICloudSnapshot, error) @@ -496,6 +497,10 @@ type ICloudVpc interface { // GetGlobalId() // 若vpc属于globalvpc,此函数返回格式必须是 'region.GetGlobalId()/vpc.GetGlobalId()' ICloudResource + IsSupportSetExternalAccess() bool // 是否支持Attach互联网网关. + GetExternalAccessMode() string + AttachInternetGateway(igwId string) error + GetRegion() ICloudRegion GetIsDefault() bool GetCidrBlock() string @@ -521,6 +526,10 @@ type ICloudVpc interface { ProposeJoinICloudInterVpcNetwork(opts *SVpcJointInterVpcNetworkOption) error } +type ICloudInternetGateway interface { + ICloudResource +} + type ICloudWire interface { ICloudResource GetIVpc() ICloudVpc diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index f482bc7383..807a0b58f1 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -146,8 +146,8 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca if cnt == 0 { // 提前同步instance type.如果同步失败可能导致vm 内存显示为0 - if err = SyncServerSkusByRegion(ctx, userCred, localRegion); err != nil { - msg := fmt.Sprintf("Get Skus for region %s failed %s", localRegion.GetName(), err) + if ret := SyncServerSkusByRegion(ctx, userCred, localRegion, nil); ret.IsError() { + msg := fmt.Sprintf("Get Skus for region %s failed %s", localRegion.GetName(), ret.Result()) log.Errorln(msg) // 暂时不终止同步 // logSyncFailed(provider, task, msg) diff --git a/pkg/compute/models/dbinstance_skus.go b/pkg/compute/models/dbinstance_skus.go index 15a830a92d..a7088e035f 100644 --- a/pkg/compute/models/dbinstance_skus.go +++ b/pkg/compute/models/dbinstance_skus.go @@ -653,3 +653,19 @@ func (self *SDBInstanceSku) GetZoneInfo() (cloudprovider.SZoneInfo, error) { zoneInfo.Zone3, _ = cloudZoneId(self.Zone3) return zoneInfo, nil } + +func (manager *SDBInstanceSkuManager) AllowSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowPerform(userCred, manager, "sync-skus") +} + +func (manager *SDBInstanceSkuManager) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SkuSyncInput) (jsonutils.JSONObject, error) { + return PerformActionSyncSkus(ctx, userCred, manager.Keyword(), input) +} + +func (manager *SDBInstanceSkuManager) AllowGetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowGetSpec(userCred, manager, "sync-tasks") +} + +func (manager *SDBInstanceSkuManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) { + return GetPropertySkusSyncTasks(ctx, userCred, query) +} diff --git a/pkg/compute/models/elasticcache_skus.go b/pkg/compute/models/elasticcache_skus.go index 63beb06af2..a5b7e28010 100644 --- a/pkg/compute/models/elasticcache_skus.go +++ b/pkg/compute/models/elasticcache_skus.go @@ -681,3 +681,19 @@ func (manager *SElasticcacheSkuManager) ListItemExportKeys(ctx context.Context, return q, nil } + +func (manager *SElasticcacheSkuManager) AllowSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowPerform(userCred, manager, "sync-skus") +} + +func (manager *SElasticcacheSkuManager) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SkuSyncInput) (jsonutils.JSONObject, error) { + return PerformActionSyncSkus(ctx, userCred, manager.Keyword(), input) +} + +func (manager *SElasticcacheSkuManager) AllowGetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowGetSpec(userCred, manager, "sync-tasks") +} + +func (manager *SElasticcacheSkuManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) { + return GetPropertySkusSyncTasks(ctx, userCred, query) +} diff --git a/pkg/compute/models/elasticipresource.go b/pkg/compute/models/elasticipresource.go new file mode 100644 index 0000000000..d54c895524 --- /dev/null +++ b/pkg/compute/models/elasticipresource.go @@ -0,0 +1,23 @@ +package models + +import "yunion.io/x/onecloud/pkg/httperrors" + +type IGetVpc interface { + GetName() string + GetVpc() (*SVpc, error) +} + +func ValidateAssociateEip(obj IGetVpc) error { + vpc, err := obj.GetVpc() + if err != nil { + return httperrors.NewGeneralError(err) + } + + if vpc != nil { + if !vpc.IsSupportAssociateEip() { + return httperrors.NewNotSupportedError("resource %s in vpc %s external access mode %s is not support accociate eip", obj.GetName(), vpc.GetName(), vpc.ExternalAccessMode) + } + } + + return nil +} diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index ecd8c9364c..bd2ef7a70c 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -983,6 +983,11 @@ func (self *SElasticip) PerformAssociate(ctx context.Context, userCred mcclient. return nil, httperrors.NewInvalidStatusError("cannot associate server in status %s", server.Status) } + err = ValidateAssociateEip(server) + if err != nil { + return nil, err + } + if len(self.NetworkId) > 0 { gns, err := server.GetNetworks("") if err != nil { @@ -1330,6 +1335,11 @@ func (manager *SElasticipManager) NewEipForVMOnHost(ctx context.Context, userCre } func (eip *SElasticip) AllocateAndAssociateVM(ctx context.Context, userCred mcclient.TokenCredential, vm *SGuest, parentTaskId string) error { + err := ValidateAssociateEip(vm) + if err != nil { + return err + } + params := jsonutils.NewDict() params.Add(jsonutils.NewString(vm.ExternalId), "instance_external_id") params.Add(jsonutils.NewString(vm.Id), "instance_id") diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index a1bc16371c..63cbb40ffe 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -2900,6 +2900,11 @@ func (self *SGuest) PerformAssociateEip(ctx context.Context, userCred mcclient.T return nil, httperrors.NewInvalidStatusError("cannot associate eip in status %s", self.Status) } + err := ValidateAssociateEip(self) + if err != nil { + return nil, err + } + eip, err := self.GetEipOrPublicIp() if err != nil { log.Errorf("Fail to get Eip %s", err) @@ -3025,6 +3030,12 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke bgpType string autoDellocate bool ) + + err := ValidateAssociateEip(self) + if err != nil { + return nil, err + } + chargeType, _ = data.GetString("charge_type") if chargeType == "" { chargeType = regionDriver.GetEipDefaultChargeType() @@ -3039,7 +3050,7 @@ func (self *SGuest) PerformCreateEip(ctx context.Context, userCred mcclient.Toke bgpType, _ = data.GetString("bgp_type") autoDellocate, _ = data.Bool("auto_dellocate") - err := self.GetDriver().ValidateCreateEip(ctx, userCred, data) + err = self.GetDriver().ValidateCreateEip(ctx, userCred, data) if err != nil { return nil, err } diff --git a/pkg/compute/models/guest_queries.go b/pkg/compute/models/guest_queries.go index 80313c1623..0a44f2571a 100644 --- a/pkg/compute/models/guest_queries.go +++ b/pkg/compute/models/guest_queries.go @@ -108,6 +108,10 @@ func (manager *SGuestManager) FetchCustomizeColumns( if len(fields) == 0 || fields.Contains("vpc_id") { rows[i].VpcId = strings.Join(gvpc.VpcId, ",") } + + if len(fields) == 0 || fields.Contains("external_access_mode") { + rows[i].VpcExternalAccessMode = strings.Join(gvpc.ExternalAccessMode, ",") + } } } } @@ -362,9 +366,10 @@ func (self *SGuest) GetRealIPs() []string { } type sGuestVpcsInfo struct { - GuestId string - Vpc []string - VpcId []string + GuestId string + Vpc []string + VpcId []string + ExternalAccessMode []string } func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo { @@ -373,7 +378,7 @@ func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo { networks := NetworkManager.Query().SubQuery() guestnetworks := GuestnetworkManager.Query().SubQuery() - q := vpcs.Query(guestnetworks.Field("guest_id"), vpcs.Field("id"), vpcs.Field("name")) + q := vpcs.Query(guestnetworks.Field("guest_id"), vpcs.Field("id"), vpcs.Field("name"), vpcs.Field("external_access_mode")) q = q.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) q = q.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) q = q.Join(guestnetworks, sqlchemy.Equals(networks.Field("id"), guestnetworks.Field("network_id"))) @@ -381,9 +386,10 @@ func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo { q = q.Distinct() type sGuestVpcInfo struct { - GuestId string - Id string - Name string + GuestId string + Id string + Name string + ExternalAccessMode string } gvpcs := make([]sGuestVpcInfo, 0) err := q.All(&gvpcs) @@ -396,13 +402,15 @@ func fetchGuestVpcs(guestIds []string) map[string]sGuestVpcsInfo { gvpc, ok := ret[gvpcs[i].GuestId] if !ok { gvpc = sGuestVpcsInfo{ - GuestId: gvpcs[i].GuestId, - Vpc: make([]string, 0), - VpcId: make([]string, 0), + GuestId: gvpcs[i].GuestId, + Vpc: make([]string, 0), + VpcId: make([]string, 0), + ExternalAccessMode: make([]string, 0), } } gvpc.VpcId = append(gvpc.VpcId, gvpcs[i].Id) gvpc.Vpc = append(gvpc.Vpc, gvpcs[i].Name) + gvpc.ExternalAccessMode = append(gvpc.ExternalAccessMode, gvpcs[i].ExternalAccessMode) ret[gvpcs[i].GuestId] = gvpc } diff --git a/pkg/compute/models/skuresource.go b/pkg/compute/models/skuresource.go new file mode 100644 index 0000000000..be73961bde --- /dev/null +++ b/pkg/compute/models/skuresource.go @@ -0,0 +1,109 @@ +package models + +import ( + "context" + "database/sql" + "strings" + + "github.com/pkg/errors" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/utils" + "yunion.io/x/sqlchemy" + + apis "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func PerformActionSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, resourceKey string, input apis.SkuSyncInput) (jsonutils.JSONObject, error) { + if !utils.IsInStringArray(resourceKey, []string{ServerSkuManager.Keyword(), ElasticcacheSkuManager.Keyword(), DBInstanceSkuManager.Keyword()}) { + return nil, httperrors.NewUnsupportOperationError("resource %s is not support sync skus", resourceKey) + } + + if len(input.Provider) == 0 && len(input.CloudregionIds) == 0 { + return nil, httperrors.NewMissingParameterError("must specific one of `provider` or `cloudregionids`") + } + + input.Provider = strings.TrimSpace(input.Provider) + if len(input.Provider) > 0 { + if !utils.IsInStringArray(input.Provider, cloudprovider.GetPublicProviders()) { + return nil, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider) + } + } + + // cloudregions to sync + q := CloudregionManager.Query() + if len(input.Provider) > 0 { + q = q.Equals("provider", input.Provider) + } else { + q = q.In("provider", cloudprovider.GetPublicProviders()) + } + + if len(input.CloudregionIds) > 0 { + q = q.Filter(sqlchemy.OR(sqlchemy.Equals(q.Field("id"), input.CloudregionIds), sqlchemy.Equals(q.Field("name"), input.CloudregionIds))) + } + + regions := []SCloudregion{} + err := db.FetchModelObjects(CloudregionManager, q, ®ions) + if err != nil && errors.Cause(err) != sql.ErrNoRows { + return nil, httperrors.NewGeneralError(err) + } + + if len(input.CloudregionIds) > 0 && len(input.CloudregionIds) != len(regions) { + return nil, httperrors.NewInputParameterError("input data contains invalid cloudregion id") + } + + if len(regions) == 0 { + return nil, httperrors.NewInputParameterError("no cloudregion found to sync skus") + } + + // start cloudregion skus sync tasks + params := jsonutils.NewDict() + params.Set("resource", jsonutils.NewString(resourceKey)) + // replaced with NewParallelTask?? + ret := jsonutils.NewDict() + taskIds := jsonutils.NewArray() + for i := range regions { + task, err := taskman.TaskManager.NewTask(ctx, "CloudRegionSyncSkusTask", ®ions[i], userCred, params, "", "", nil) + if err != nil { + return nil, errors.Wrapf(err, "CloudRegionSyncSkusTask") + } + + task.ScheduleRun(nil) + taskIds.Add(jsonutils.NewString(task.GetId())) + } + ret.Set("tasks", taskIds) + return ret, nil +} + +func GetPropertySkusSyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query apis.SkuTaskQueryInput) (jsonutils.JSONObject, error) { + tasks := []taskman.STask{} + q := taskman.TaskManager.Query() + q = q.Equals("obj_name", CloudregionManager.Keyword()) + q = q.Equals("task_name", "CloudRegionSyncSkusTask") + if len(query.TaskIds) > 0 { + q = q.In("id", query.TaskIds) + } else { + q = q.NotIn("stage", []string{taskman.TASK_STAGE_FAILED, taskman.TASK_STAGE_COMPLETE}) + } + err := q.All(&tasks) + if err != nil { + return nil, httperrors.NewGeneralError(err) + } + + ret := jsonutils.NewDict() + items := jsonutils.NewArray() + for i := range tasks { + item := jsonutils.NewDict() + item.Set("id", jsonutils.NewString(tasks[i].GetId())) + item.Set("created_at", jsonutils.NewTimeString(tasks[i].GetStartTime())) + item.Set("stage", jsonutils.NewString(tasks[i].Stage)) + items.Add(item) + } + ret.Set("tasks", items) + return ret, nil +} diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index eb293638ff..b5be670a08 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -1498,3 +1498,19 @@ func (manager *SServerSkuManager) ListItemExportKeys(ctx context.Context, return q, nil } + +func (manager *SServerSkuManager) AllowSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowPerform(userCred, manager, "sync-skus") +} + +func (manager *SServerSkuManager) PerformSyncSkus(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.SkuSyncInput) (jsonutils.JSONObject, error) { + return PerformActionSyncSkus(ctx, userCred, manager.Keyword(), input) +} + +func (manager *SServerSkuManager) AllowGetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { + return db.IsAdminAllowGetSpec(userCred, manager, "sync-tasks") +} + +func (manager *SServerSkuManager) GetPropertySyncTasks(ctx context.Context, userCred mcclient.TokenCredential, query api.SkuTaskQueryInput) (jsonutils.JSONObject, error) { + return GetPropertySkusSyncTasks(ctx, userCred, query) +} diff --git a/pkg/compute/models/skus_tools.go b/pkg/compute/models/skus_tools.go index ed58d388e4..ed8fa357bf 100644 --- a/pkg/compute/models/skus_tools.go +++ b/pkg/compute/models/skus_tools.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/compare" v "yunion.io/x/pkg/util/version" apis "yunion.io/x/onecloud/pkg/apis/compute" @@ -387,16 +388,28 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt } // 同步指定region sku列表 -func SyncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) error { - meta, err := FetchSkuResourcesMeta() - if err != nil { - return errors.Wrap(err, "SyncServerSkusByRegion.FetchSkuResourcesMeta") +func SyncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult { + result := compare.SyncResult{} + var err error + if extSkuMeta == nil { + extSkuMeta, err = FetchSkuResourcesMeta() + if err != nil { + result.AddError(errors.Wrap(err, "SyncServerSkusByRegion.FetchSkuResourcesMeta")) + return result + } } - result := ServerSkuManager.SyncServerSkus(ctx, userCred, region, meta) + result = ServerSkuManager.SyncServerSkus(ctx, userCred, region, extSkuMeta) notes := fmt.Sprintf("SyncServerSkusByRegion %s result: %s", region.Name, result.Result()) log.Infof(notes) - return nil + + // notfiy sched manager + _, err = modules.SchedManager.SyncSku(auth.GetAdminSession(ctx, options.Options.Region, ""), false) + if err != nil { + log.Errorf("SchedManager SyncSku %s", err) + } + + return result } func FetchSkuResourcesMeta() (*SSkuResourcesMeta, error) { diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index 7caaab02a9..5f19bc6339 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -602,6 +602,7 @@ func (self *SVpc) SyncWithCloudVpc(ctx context.Context, userCred mcclient.TokenC self.ExternalId = extVPC.GetGlobalId() self.IsEmulated = extVPC.IsEmulated() + self.ExternalAccessMode = extVPC.GetExternalAccessMode() return nil }) @@ -741,6 +742,27 @@ func (manager *SVpcManager) InitializeData() error { } } + { + // initialize default external access mode for public cloud + var vpcs []SVpc + q := manager.Query(). + IsNotEmpty("manager_id"). + IsNotEmpty("external_id"). + IsNullOrEmpty("external_access_mode") + if err := db.FetchModelObjects(manager, q, &vpcs); err != nil { + return errors.Wrap(err, "fetch public cloud vpc with external_access_mode not set") + } + for i := range vpcs { + vpc := &vpcs[i] + if _, err := db.Update(vpc, func() error { + vpc.ExternalAccessMode = api.VPC_EXTERNAL_ACCESS_MODE_EIP + return nil + }); err != nil { + return errors.Wrap(err, "db set default external_access_mode") + } + } + } + { vpcs := []SVpc{} q := manager.Query().IsTrue("is_emulated").IsNotEmpty("external_id").NotEquals("name", "-") @@ -800,16 +822,22 @@ func (manager *SVpcManager) ValidateCreateData( } } input.CloudproviderId = managerObj.GetId() + if input.ExternalAccessMode == "" { + input.ExternalAccessMode = api.VPC_EXTERNAL_ACCESS_MODE_EIP + } + // data.Add(jsonutils.NewString(managerObj.GetId()), "manager_id") } else { input.Status = api.VPC_STATUS_AVAILABLE if input.ExternalAccessMode == "" { input.ExternalAccessMode = options.Options.DefaultVpcExternalAccessMode } - if !utils.IsInStringArray(input.ExternalAccessMode, api.VPC_EXTERNAL_ACCESS_MODES) { - return input, httperrors.NewInputParameterError("invalid external_access_mode %q, want %s", - input.Status, api.VPC_EXTERNAL_ACCESS_MODES) - } + } + + // check external access mode + if !utils.IsInStringArray(input.ExternalAccessMode, api.VPC_EXTERNAL_ACCESS_MODES) { + return input, httperrors.NewInputParameterError("invalid external_access_mode %q, want %s", + input.Status, api.VPC_EXTERNAL_ACCESS_MODES) } cidrBlock := input.CidrBlock @@ -1640,3 +1668,11 @@ func (self *SVpc) newFromCloudPeerConnection(ctx context.Context, userCred mccli db.OpsLog.LogEvent(peer, db.ACT_CREATE, peer.GetShortDesc(ctx), userCred) return peer, nil } + +func (self *SVpc) IsSupportAssociateEip() bool { + if utils.IsInStringArray(self.ExternalAccessMode, []string{api.VPC_EXTERNAL_ACCESS_MODE_EIP_DISTGW, api.VPC_EXTERNAL_ACCESS_MODE_EIP}) { + return true + } + + return false +} diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index 187dc766cf..2a249209b2 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -1214,6 +1214,19 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateVpc(ctx context.Con return nil, errors.Wrap(err, "cloudprovider.WaitStatus") } + if ivpc.IsSupportSetExternalAccess() && vpc.ExternalAccessMode == api.VPC_EXTERNAL_ACCESS_MODE_EIP { + igw, err := iregion.CreateInternetGateway() + if err != nil { + return nil, errors.Wrap(err, "vpc.AttachInternetGateway") + + } + + err = ivpc.AttachInternetGateway(igw.GetId()) + if err != nil { + return nil, errors.Wrap(err, "vpc.AttachInternetGateway") + } + } + err = vpc.SyncWithCloudVpc(ctx, userCred, ivpc, nil) if err != nil { return nil, errors.Wrap(err, "vpc.SyncWithCloudVpc") diff --git a/pkg/compute/tasks/cloud_region_sync_skus_task.go b/pkg/compute/tasks/cloud_region_sync_skus_task.go new file mode 100644 index 0000000000..7069b4534e --- /dev/null +++ b/pkg/compute/tasks/cloud_region_sync_skus_task.go @@ -0,0 +1,61 @@ +package tasks + +import ( + "context" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/compare" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type CloudRegionSyncSkusTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(CloudRegionSyncSkusTask{}) +} + +func (self *CloudRegionSyncSkusTask) taskFailed(ctx context.Context, region *models.SCloudregion, msg string) { + db.OpsLog.LogEvent(region, db.ACT_SYNC_CLOUD_SKUS, msg, self.GetUserCred()) + logclient.AddActionLogWithStartable(self, region, logclient.ACT_CLOUD_SYNC, msg, self.UserCred, false) + self.SetStageFailed(ctx, jsonutils.NewString(msg)) +} + +func (self *CloudRegionSyncSkusTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + region := obj.(*models.SCloudregion) + res, _ := self.GetParams().GetString("resource") + meta, err := models.FetchSkuResourcesMeta() + if err != nil { + self.taskFailed(ctx, region, err.Error()) + return + } + + type SyncFunc func(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, extSkuMeta *models.SSkuResourcesMeta) compare.SyncResult + var syncFunc SyncFunc + switch res { + case models.ServerSkuManager.Keyword(): + syncFunc = models.SyncServerSkusByRegion + case models.ElasticcacheSkuManager.Keyword(): + syncFunc = models.ElasticcacheSkuManager.SyncElasticcacheSkus + case models.DBInstanceSkuManager.Keyword(): + syncFunc = models.DBInstanceSkuManager.SyncDBInstanceSkus + } + + if syncFunc != nil { + result := syncFunc(ctx, self.GetUserCred(), region, meta) + log.Infof("Sync %s %s skus for region %s result: %s", region.Provider, res, region.Name, result.Result()) + if result.IsError() { + self.taskFailed(ctx, region, result.Result()) + return + } + } + + self.SetStageComplete(ctx, nil) +} diff --git a/pkg/mcclient/options/skus.go b/pkg/mcclient/options/skus.go new file mode 100644 index 0000000000..dd79b5b930 --- /dev/null +++ b/pkg/mcclient/options/skus.go @@ -0,0 +1,35 @@ +package options + +import ( + "fmt" + + "yunion.io/x/jsonutils" +) + +type SkuSyncOptions struct { + // 云平台名称 + // example: Google + Provider string `json:"provider,omitempty" help:"cloud provider name"` + + // 区域ID + CloudregionIds []string `json:"cloudregion_ids" help:"cloud region id list"` +} + +func (opts *SkuSyncOptions) Params() (jsonutils.JSONObject, error) { + return jsonutils.Marshal(opts), nil +} + +type SkuTaskQueryOptions struct { + // 异步任务ID + TaskIds []string `json:"task_ids" help:"task ids"` +} + +func (opts *SkuTaskQueryOptions) Params() (jsonutils.JSONObject, error) { + if len(opts.TaskIds) == 0 { + return nil, fmt.Errorf("task_ids is empty") + } + + params := jsonutils.NewDict() + params.Set("task_ids", jsonutils.Marshal(opts.TaskIds)) + return params, nil +} diff --git a/pkg/multicloud/aws/internetgateway.go b/pkg/multicloud/aws/internetgateway.go new file mode 100644 index 0000000000..5854af0a7d --- /dev/null +++ b/pkg/multicloud/aws/internetgateway.go @@ -0,0 +1,51 @@ +package aws + +import ( + "github.com/pkg/errors" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudprovider" +) + +type SInternetGateway struct { + region *SRegion + + Attachments []InternetGatewayAttachment `json:"Attachments"` + InternetGatewayID string `json:"InternetGatewayId"` + OwnerID string `json:"OwnerId"` + Tags []interface{} `json:"Tags"` +} + +type InternetGatewayAttachment struct { + State string `json:"State"` + VpcID string `json:"VpcId"` +} + +func (i *SInternetGateway) GetId() string { + return i.InternetGatewayID +} + +func (i *SInternetGateway) GetName() string { + return i.InternetGatewayID +} + +func (i *SInternetGateway) GetGlobalId() string { + return i.GetId() +} + +func (i *SInternetGateway) GetStatus() string { + return "" +} + +func (i *SInternetGateway) Refresh() error { + return errors.Wrap(cloudprovider.ErrNotImplemented, "Refresh") +} + +func (i *SInternetGateway) IsEmulated() bool { + return false +} + +func (i *SInternetGateway) GetMetadata() *jsonutils.JSONDict { + return nil +} diff --git a/pkg/multicloud/aws/region.go b/pkg/multicloud/aws/region.go index 17775f4c52..7248b74fe3 100644 --- a/pkg/multicloud/aws/region.go +++ b/pkg/multicloud/aws/region.go @@ -1099,3 +1099,20 @@ func (self *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreat func (region *SRegion) GetCapabilities() []string { return region.client.GetCapabilities() } + +func (region *SRegion) CreateInternetGateway() (cloudprovider.ICloudInternetGateway, error) { + input := ec2.CreateInternetGatewayInput{} + output, err := region.ec2Client.CreateInternetGateway(&input) + if err != nil { + return nil, errors.Wrap(err, "CreateInternetGateway") + } + + ret := &SInternetGateway{} + ret.region = region + err = unmarshalAwsOutput(output, "InternetGateway", ret) + if err != nil { + return nil, errors.Wrap(err, "unmarshalAwsOutput") + } + + return ret, nil +} diff --git a/pkg/multicloud/aws/vpc.go b/pkg/multicloud/aws/vpc.go index 85dfde24df..d78c3af4df 100644 --- a/pkg/multicloud/aws/vpc.go +++ b/pkg/multicloud/aws/vpc.go @@ -384,6 +384,37 @@ func (self *SVpc) acceptSVpcPeeringConnection(id string) error { return nil } +func (self *SVpc) IsSupportSetExternalAccess() bool { + return true +} + +func (self *SVpc) GetExternalAccessMode() string { + igws, err := self.region.GetInternetGateways(self.GetId()) + if err != nil { + log.Errorf("GetExternalAccessMode.GetInternetGateways %s", err) + + } + + if len(igws) > 0 { + return api.VPC_EXTERNAL_ACCESS_MODE_EIP + } + + return api.VPC_EXTERNAL_ACCESS_MODE_NONE +} + +func (self *SVpc) AttachInternetGateway(igwId string) error { + input := ec2.AttachInternetGatewayInput{} + input.SetInternetGatewayId(igwId) + input.SetVpcId(self.GetId()) + + _, err := self.region.ec2Client.AttachInternetGateway(&input) + if err != nil { + return errors.Wrap(err, "AttachInternetGateway") + } + + return nil +} + func (self *SRegion) getVpc(vpcId string) (*SVpc, error) { if len(vpcId) == 0 { return nil, fmt.Errorf("GetVpc vpc id should not be empty.") @@ -491,3 +522,31 @@ func (self *SRegion) GetVpcs(vpcId []string, offset int, limit int) ([]SVpc, int return vpcs, len(vpcs), nil } + +func (self *SRegion) GetInternetGateways(vpcId string) ([]SInternetGateway, error) { + input := ec2.DescribeInternetGatewaysInput{} + filters := make([]*ec2.Filter, 0) + if len(vpcId) > 0 { + filters = AppendSingleValueFilter(filters, "attachment.vpc-id", vpcId) + } + + if len(filters) > 0 { + input.SetFilters(filters) + } + output, err := self.ec2Client.DescribeInternetGateways(&input) + if err != nil { + return nil, errors.Wrap(err, "DescribeInternetGateways") + } + + igws := make([]SInternetGateway, len(output.InternetGateways)) + err = unmarshalAwsOutput(output, "InternetGateways", &igws) + if err != nil { + return nil, errors.Wrap(err, "unmarshalAwsOutput") + } + + for i := range igws { + igws[i].region = self + } + + return igws, nil +} diff --git a/pkg/multicloud/region_base.go b/pkg/multicloud/region_base.go index 1b4ff4b7fe..befa6a1b8e 100644 --- a/pkg/multicloud/region_base.go +++ b/pkg/multicloud/region_base.go @@ -143,3 +143,7 @@ func (self *SRegion) GetICloudEvents(start time.Time, end time.Time, withReadEve func (self *SRegion) GetICloudQuotas() ([]cloudprovider.ICloudQuota, error) { return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "GetICloudQuotas") } + +func (self *SRegion) CreateInternetGateway() (cloudprovider.ICloudInternetGateway, error) { + return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "CreateInternetGateway") +} diff --git a/pkg/multicloud/vpc_base.go b/pkg/multicloud/vpc_base.go index 724cf54345..c262ef9fb2 100644 --- a/pkg/multicloud/vpc_base.go +++ b/pkg/multicloud/vpc_base.go @@ -17,6 +17,7 @@ package multicloud import ( "yunion.io/x/pkg/errors" + apis "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -63,3 +64,15 @@ func (self *SVpc) DeleteVpcPeeringConnectionRoute(vpcPeeringConnectionId string) func (self *SVpc) ProposeJoinICloudInterVpcNetwork(opts *cloudprovider.SVpcJointInterVpcNetworkOption) error { return errors.Wrapf(cloudprovider.ErrNotImplemented, "ProposeJoinICloudInterVpcNetwork") } + +func (self *SVpc) IsSupportSetExternalAccess() bool { + return false +} + +func (self *SVpc) GetExternalAccessMode() string { + return apis.VPC_EXTERNAL_ACCESS_MODE_EIP +} + +func (self *SVpc) AttachInternetGateway(igwId string) error { + return errors.Wrap(cloudprovider.ErrNotSupported, "AttachInternetGateway") +}