From 859ab97175c4a00c89d0a863ab8acedbcc5efd77 Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Tue, 7 Jul 2020 16:33:49 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20openstack=E5=A4=9A=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/apis/compute/cloudaccount.go | 2 +- pkg/apis/compute/storage_const.go | 2 +- pkg/cloudprovider/cloudprovider.go | 6 + pkg/cloudprovider/fakeregion.go | 2 +- pkg/cloudprovider/network.go | 22 + pkg/cloudprovider/resources.go | 14 +- pkg/cloudprovider/securitygroup.go | 15 +- pkg/compute/guestdrivers/managedvirtual.go | 13 +- pkg/compute/models/cloudaccounts.go | 18 +- pkg/compute/models/elasticcache_instances.go | 2 +- pkg/compute/models/networks.go | 18 +- pkg/compute/models/regiondrivers.go | 6 +- pkg/compute/models/secgroupcache.go | 20 +- pkg/compute/regiondrivers/base.go | 9 +- pkg/compute/regiondrivers/managedvirtual.go | 35 +- pkg/compute/regiondrivers/openstack.go | 5 + pkg/compute/tasks/network_create_task.go | 14 +- .../tasks/secgroup_group_cache_task.go | 2 +- pkg/multicloud/aliyun/region.go | 4 +- pkg/multicloud/aliyun/wire.go | 4 +- pkg/multicloud/aws/region.go | 4 +- pkg/multicloud/aws/wire.go | 4 +- pkg/multicloud/azure/classic_wire.go | 2 +- pkg/multicloud/azure/region.go | 8 +- pkg/multicloud/azure/wire.go | 4 +- pkg/multicloud/ctyun/region.go | 6 +- pkg/multicloud/ctyun/wire.go | 4 +- pkg/multicloud/google/securitygroup.go | 6 +- pkg/multicloud/google/wire.go | 4 +- pkg/multicloud/host_base.go | 1 + pkg/multicloud/huawei/region.go | 4 +- pkg/multicloud/huawei/wire.go | 4 +- pkg/multicloud/openstack/disk.go | 213 ++++--- pkg/multicloud/openstack/eip.go | 130 +++-- pkg/multicloud/openstack/flavor.go | 78 +-- pkg/multicloud/openstack/host.go | 490 +--------------- pkg/multicloud/openstack/hypervisor.go | 351 ++++++++++++ pkg/multicloud/openstack/image.go | 70 ++- pkg/multicloud/openstack/instance.go | 540 +++++++++++------- pkg/multicloud/openstack/instancenic.go | 88 +-- pkg/multicloud/openstack/keypaire.go | 61 +- .../{shell/version.go => message.go} | 41 +- pkg/multicloud/openstack/network.go | 143 +++-- pkg/multicloud/openstack/nova_disk.go | 174 ++++++ pkg/multicloud/openstack/novastorage.go | 21 +- pkg/multicloud/openstack/openstack.go | 488 ++++++++++------ pkg/multicloud/openstack/port.go | 61 +- pkg/multicloud/openstack/project.go | 65 ++- pkg/multicloud/openstack/provider/provider.go | 6 +- pkg/multicloud/openstack/quota.go | 50 +- pkg/multicloud/openstack/region.go | 344 +++++------ pkg/multicloud/openstack/router.go | 12 +- pkg/multicloud/openstack/routetable.go | 2 +- pkg/multicloud/openstack/securitygroup.go | 138 ++--- pkg/multicloud/openstack/shell/disk.go | 4 +- pkg/multicloud/openstack/shell/eip.go | 19 +- pkg/multicloud/openstack/shell/host.go | 34 +- pkg/multicloud/openstack/shell/image.go | 12 +- pkg/multicloud/openstack/shell/instance.go | 19 + .../shell/{instancenic.go => messages.go} | 10 +- pkg/multicloud/openstack/shell/network.go | 16 + pkg/multicloud/openstack/shell/port.go | 5 +- pkg/multicloud/openstack/shell/project.go | 2 +- .../openstack/shell/securitygroup.go | 12 +- pkg/multicloud/openstack/shell/storage.go | 19 +- pkg/multicloud/openstack/shell/vpc.go | 24 +- pkg/multicloud/openstack/shell/zone.go | 2 +- pkg/multicloud/openstack/snapshot.go | 110 ++-- pkg/multicloud/openstack/storage.go | 120 +++- pkg/multicloud/openstack/storagecache.go | 8 +- pkg/multicloud/openstack/vpc.go | 183 +++--- pkg/multicloud/openstack/wire.go | 46 +- pkg/multicloud/openstack/zone.go | 303 +++------- pkg/multicloud/qcloud/region.go | 4 +- pkg/multicloud/qcloud/wire.go | 4 +- pkg/multicloud/ucloud/region.go | 4 +- pkg/multicloud/ucloud/wire.go | 4 +- pkg/multicloud/zstack/network.go | 5 +- pkg/multicloud/zstack/region.go | 4 +- pkg/multicloud/zstack/wire.go | 4 +- 80 files changed, 2717 insertions(+), 2090 deletions(-) create mode 100644 pkg/cloudprovider/network.go create mode 100644 pkg/multicloud/openstack/hypervisor.go rename pkg/multicloud/openstack/{shell/version.go => message.go} (50%) create mode 100644 pkg/multicloud/openstack/nova_disk.go rename pkg/multicloud/openstack/shell/{instancenic.go => messages.go} (70%) diff --git a/pkg/apis/compute/cloudaccount.go b/pkg/apis/compute/cloudaccount.go index c347c06447..323c329720 100644 --- a/pkg/apis/compute/cloudaccount.go +++ b/pkg/apis/compute/cloudaccount.go @@ -180,7 +180,7 @@ type CloudaccountCreateInput struct { // 自动同步间隔时间 SyncIntervalSeconds int `json:"sync_interval_seconds"` - // 自动根据云上项目或订阅创建本地项目 + // 自动根据云上项目或订阅创建本地项目, OpenStack此参数为true // default: false AutoCreateProject *bool `json:"auto_create_project"` diff --git a/pkg/apis/compute/storage_const.go b/pkg/apis/compute/storage_const.go index 2e9dfd0b50..e0f557056e 100644 --- a/pkg/apis/compute/storage_const.go +++ b/pkg/apis/compute/storage_const.go @@ -127,7 +127,7 @@ var ( STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_ZSTACK_CEPH, STORAGE_GPFS, } - HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE} + HOST_STORAGE_LOCAL_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_ZSTACK_LOCAL_STORAGE, STORAGE_OPENSTACK_NOVA} STORAGE_LIMITED_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_NAS, STORAGE_RBD, STORAGE_NFS, STORAGE_GPFS} diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 716761ed34..6ddac9d193 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -190,6 +190,8 @@ type ICloudProviderFactory interface { GetMaxCloudEventSyncDays() int GetMaxCloudEventKeepDays() int + IsNeedForceAutoCreateProject() bool + IsSupportClouduser() bool IsSupportClouduserPolicy() bool IsSupportResetClouduserPassword() bool @@ -483,6 +485,10 @@ func (factory *baseProviderFactory) GetMaxCloudEventKeepDays() int { return 7 } +func (factory *baseProviderFactory) IsNeedForceAutoCreateProject() bool { + return false +} + func (factory *baseProviderFactory) IsSupportClouduser() bool { return false } diff --git a/pkg/cloudprovider/fakeregion.go b/pkg/cloudprovider/fakeregion.go index 4504f6b78e..e9936ae97a 100644 --- a/pkg/cloudprovider/fakeregion.go +++ b/pkg/cloudprovider/fakeregion.go @@ -97,7 +97,7 @@ func (region *SFakeOnPremiseRegion) GetISecurityGroupById(id string) (ICloudSecu return nil, ErrNotSupported } -func (region *SFakeOnPremiseRegion) GetISecurityGroupByName(vpcId string, name string) (ICloudSecurityGroup, error) { +func (region *SFakeOnPremiseRegion) GetISecurityGroupByName(opts *SecurityGroupFilterOptions) (ICloudSecurityGroup, error) { return nil, ErrNotSupported } diff --git a/pkg/cloudprovider/network.go b/pkg/cloudprovider/network.go new file mode 100644 index 0000000000..3e863d6495 --- /dev/null +++ b/pkg/cloudprovider/network.go @@ -0,0 +1,22 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +type SNetworkCreateOptions struct { + Name string + Desc string + ProjectId string + Cidr string +} diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 4b63ef12e7..f72d9fee8f 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type ICloudResource interface { @@ -70,7 +71,7 @@ type ICloudRegion interface { GetIDiskById(id string) (ICloudDisk, error) GetISecurityGroupById(secgroupId string) (ICloudSecurityGroup, error) - GetISecurityGroupByName(vpcId string, name string) (ICloudSecurityGroup, error) + GetISecurityGroupByName(opts *SecurityGroupFilterOptions) (ICloudSecurityGroup, error) CreateISecurityGroup(conf *SecurityGroupCreateInput) (ICloudSecurityGroup, error) CreateIVpc(name string, desc string, cidr string) (ICloudVpc, error) @@ -345,7 +346,7 @@ type ICloudEIP interface { } type ICloudSecurityGroup interface { - ICloudResource + IVirtualResource GetDescription() string GetRules() ([]SecurityRule, error) @@ -453,7 +454,7 @@ type ICloudWire interface { GetINetworkById(netid string) (ICloudNetwork, error) - CreateINetwork(name string, cidr string, desc string) (ICloudNetwork, error) + CreateINetwork(opts *SNetworkCreateOptions) (ICloudNetwork, error) } type ICloudNetwork interface { @@ -466,8 +467,11 @@ type ICloudNetwork interface { GetIpMask() int8 GetGateway() string GetServerType() string - // GetIsPublic() bool - // GetPublicScope() rbacutils.TRbacScope + //GetIsPublic() bool + // 仅私有云有用，公有云无效 + // 1. scope = none 非共享, network仅会属于一个项目,并且私有 + // 2. scope = system 系统共享 云账号共享会跟随云账号共享，云账号非共享,会共享到network所在域 + GetPublicScope() rbacutils.TRbacScope Delete() error diff --git a/pkg/cloudprovider/securitygroup.go b/pkg/cloudprovider/securitygroup.go index 1d304be8cf..20cad61232 100644 --- a/pkg/cloudprovider/securitygroup.go +++ b/pkg/cloudprovider/securitygroup.go @@ -24,11 +24,18 @@ import ( const DEFAULT_CLOUD_RULE_ID = "default_cloud_rule_id" +type SecurityGroupFilterOptions struct { + VpcId string + Name string + ProjectId string +} + type SecurityGroupCreateInput struct { - Name string - Desc string - VpcId string - Rules []secrules.SecurityRule + Name string + Desc string + VpcId string + ProjectId string + Rules []secrules.SecurityRule } type SecurityRule struct { diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 10f83edbe2..b3a2b78f71 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -321,7 +321,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestDeployGuestOnHost(ctx context secgroups := guest.GetSecgroups() for i, secgroup := range secgroups { - externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup) + externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup, desc.ProjectId) if err != nil { return errors.Wrap(err, "RequestSyncSecurityGroup") } @@ -976,10 +976,19 @@ func (self *SManagedVirtualizedGuestDriver) RequestSyncSecgroupsOnHost(ctx conte return errors.Wrap(err, "GetSecurityGroupVpcId") } + remoteProjectId := "" + provider := host.GetCloudprovider() + if provider != nil { + remoteProjectId, err = provider.SyncProject(ctx, task.GetUserCred(), guest.ProjectId) + if err != nil { + log.Errorf("failed to sync project %s for guest %s error: %v", guest.ProjectId, guest.Name, err) + } + } + secgroups := guest.GetSecgroups() externalIds := []string{} for _, secgroup := range secgroups { - externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup) + externalId, err := region.GetDriver().RequestSyncSecurityGroup(ctx, task.GetUserCred(), vpcId, vpc, &secgroup, remoteProjectId) if err != nil { return errors.Wrap(err, "RequestSyncSecurityGroup") } diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index a9e1477688..8991e9fef8 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -809,6 +809,16 @@ func (manager *SCloudaccountManager) ValidateCreateData( return input, err } + if !cloudprovider.IsSupported(input.Provider) { + return input, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider) + } + providerDriver, _ := cloudprovider.GetProviderFactory(input.Provider) + + forceAutoCreateProject := providerDriver.IsNeedForceAutoCreateProject() + if forceAutoCreateProject { + input.AutoCreateProject = &forceAutoCreateProject + } + if len(input.ProjectId) > 0 { var proj *db.STenant proj, input.ProjectizedResourceInput, err = db.ValidateProjectizedResourceInput(ctx, input.ProjectizedResourceInput) @@ -821,17 +831,15 @@ func (manager *SCloudaccountManager) ValidateCreateData( if input.AutoCreateProject != nil && *input.AutoCreateProject { log.Warningf("project_id and auto_create_project should not be turned on at the same time") } - input.AutoCreateProject = nil + if !forceAutoCreateProject { + input.AutoCreateProject = nil + } } else if input.AutoCreateProject == nil || !*input.AutoCreateProject { log.Warningf("auto_create_project is off while no project_id specified") createProject := true input.AutoCreateProject = &createProject } - if !cloudprovider.IsSupported(input.Provider) { - return input, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider) - } - providerDriver, _ := cloudprovider.GetProviderFactory(input.Provider) input.SCloudaccount, err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, input.SCloudaccountCredential) if err != nil { return input, err diff --git a/pkg/compute/models/elasticcache_instances.go b/pkg/compute/models/elasticcache_instances.go index f2bde4a98d..2b0ea6a4fa 100644 --- a/pkg/compute/models/elasticcache_instances.go +++ b/pkg/compute/models/elasticcache_instances.go @@ -921,7 +921,7 @@ func (self *SElasticcache) GetCreateHuaweiElasticcacheParams(data *jsonutils.JSO // fill security group here if len(self.SecurityGroupId) > 0 { - sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, self.GetRegion().Id, self.GetCloudprovider().Id) + sgCache, err := SecurityGroupCacheManager.GetSecgroupCache(context.Background(), nil, self.SecurityGroupId, self.VpcId, self.GetRegion().Id, self.GetCloudprovider().Id, "") if err != nil { return nil, errors.Wrap(err, "elasticcache.GetCreateHuaweiElasticcacheParams.SecurityGroup") } diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 62df322d3c..3650cbbf7d 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -711,7 +711,14 @@ func (self *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclien SyncCloudProject(userCred, self, syncOwnerId, extNet, vpc.ManagerId) if provider != nil { - self.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) + shareInfo := provider.getAccountShareInfo() + if utils.IsInStringArray(provider.Provider, api.PRIVATE_CLOUD_PROVIDERS) && extNet.GetPublicScope() == rbacutils.ScopeNone { + shareInfo = apis.SAccountShareInfo{ + IsPublic: false, + PublicScope: rbacutils.ScopeNone, + } + } + self.SyncShareState(ctx, userCred, shareInfo) } return nil @@ -753,7 +760,14 @@ func (manager *SNetworkManager) newFromCloudNetwork(ctx context.Context, userCre SyncCloudProject(userCred, &net, syncOwnerId, extNet, vpc.ManagerId) if provider != nil { - net.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) + shareInfo := provider.getAccountShareInfo() + if utils.IsInStringArray(provider.Provider, api.PRIVATE_CLOUD_PROVIDERS) && extNet.GetPublicScope() == rbacutils.ScopeNone { + shareInfo = apis.SAccountShareInfo{ + IsPublic: false, + PublicScope: rbacutils.ScopeNone, + } + } + net.SyncShareState(ctx, userCred, shareInfo) } db.OpsLog.LogEvent(&net, db.ACT_CREATE, net.GetShortDesc(ctx), userCred) diff --git a/pkg/compute/models/regiondrivers.go b/pkg/compute/models/regiondrivers.go index 5b08852c83..c7a785a8b6 100644 --- a/pkg/compute/models/regiondrivers.go +++ b/pkg/compute/models/regiondrivers.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type IRegionDriver interface { @@ -117,8 +118,8 @@ type IRegionDriver interface { RequestUnBindIPFromNatgateway(ctx context.Context, task taskman.ITask, nat INatHelper, natgateway *SNatGateway) error BindIPToNatgatewayRollback(ctx context.Context, eipId string) error - RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, secgroup *SSecurityGroup, classic bool, task taskman.ITask) error - RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *SVpc, secgroup *SSecurityGroup) (string, error) + RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, vpc *SVpc, secgroup *SSecurityGroup, classic bool, removeProjectId string, task taskman.ITask) error + RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *SVpc, secgroup *SSecurityGroup, removeProjectId string) (string, error) GetSecurityGroupRuleOrder() cloudprovider.TPriorityOrder // Desc(priority值越大,优先级越高) Asc(priority值越小,优先级越高) GetDefaultSecurityGroupInRule() cloudprovider.SecurityRule GetDefaultSecurityGroupOutRule() cloudprovider.SecurityRule @@ -131,6 +132,7 @@ type IRegionDriver interface { IsSecurityGroupBelongGlobalVpc() bool //安全组子账号范围内可用 GetDefaultSecurityGroupVpcId() string GetSecurityGroupVpcId(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, host *SHost, vpc *SVpc, classic bool) (string, error) + GetSecurityGroupPublicScope() rbacutils.TRbacScope IsSupportedBillingCycle(bc billing.SBillingCycle, resource string) bool GetSecgroupVpcid(vpcId string) string diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index d6ff6c8171..40f7dd096a 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -57,7 +57,8 @@ type SSecurityGroupCache struct { // SecgroupId string `width:"128" charset:"ascii" list:"user" create:"required"` // 虚拟私有网络外部Id - VpcId string `width:"128" charset:"ascii" list:"user" create:"required"` + VpcId string `width:"128" charset:"ascii" list:"user" create:"required"` + ExternalProjectId string `width:"128" charset:"ascii" list:"user" create:"optional"` } var SecurityGroupCacheManager *SSecurityGroupCacheManager @@ -260,7 +261,7 @@ func (manager *SSecurityGroupCacheManager) FetchCustomizeColumns( return rows } -func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId string, regionId string, providerId string) (*SSecurityGroupCache, error) { +func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId string, regionId string, providerId string, projectId string) (*SSecurityGroupCache, error) { secgroupCache := SSecurityGroupCache{} query := manager.Query() conds := []sqlchemy.ICondition{ @@ -268,6 +269,9 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, sqlchemy.Equals(query.Field("vpc_id"), vpcId), sqlchemy.Equals(query.Field("manager_id"), providerId), } + if len(projectId) > 0 { + conds = append(conds, sqlchemy.Equals(query.Field("external_project_id"), projectId)) + } _region, err := CloudregionManager.FetchById(regionId) if err != nil { return nil, errors.Wrapf(err, "CloudregionManager.FetchById(%s)", regionId) @@ -290,7 +294,7 @@ func (manager *SSecurityGroupCacheManager) GetSecgroupCache(ctx context.Context, return &secgroupCache, nil } -func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) (*SSecurityGroupCache, error) { +func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string, projectId string) (*SSecurityGroupCache, error) { lockman.LockClass(ctx, manager, userCred.GetProjectId()) defer lockman.ReleaseClass(ctx, manager, userCred.GetProjectId()) @@ -306,6 +310,7 @@ func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCre secgroupCache.Status = api.SECGROUP_CACHE_STATUS_CACHING secgroupCache.CloudregionId = regionId secgroupCache.Name = secgroup.GetName() + secgroupCache.ExternalProjectId = projectId secgroupCache.SetModelManager(manager, secgroupCache) if err := manager.TableSpec().Insert(ctx, secgroupCache); err != nil { log.Errorf("insert secgroupcache error: %v", err) @@ -314,8 +319,8 @@ func (manager *SSecurityGroupCacheManager) NewCache(ctx context.Context, userCre return secgroupCache, nil } -func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string) (*SSecurityGroupCache, error) { - secgroupCache, err := manager.GetSecgroupCache(ctx, userCred, secgroupId, vpcId, regionId, providerId) +func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCred mcclient.TokenCredential, secgroupId, vpcId, regionId string, providerId string, projectId string) (*SSecurityGroupCache, error) { + secgroupCache, err := manager.GetSecgroupCache(ctx, userCred, secgroupId, vpcId, regionId, providerId, projectId) if err != nil { return nil, err } @@ -324,7 +329,7 @@ func (manager *SSecurityGroupCacheManager) Register(ctx context.Context, userCre return secgroupCache, nil } - return manager.NewCache(ctx, userCred, secgroupId, vpcId, regionId, providerId) + return manager.NewCache(ctx, userCred, secgroupId, vpcId, regionId, providerId, projectId) } func (manager *SSecurityGroupCacheManager) getSecgroupcachesByProvider(provider *SCloudprovider, region *SCloudregion, vpcId string) ([]SSecurityGroupCache, error) { @@ -414,6 +419,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C commondb[i].Status = api.SECGROUP_CACHE_STATUS_READY commondb[i].Name = commonext[i].GetName() commondb[i].Description = commonext[i].GetDescription() + commondb[i].ExternalProjectId = commonext[i].GetProjectId() return nil }) if err != nil { @@ -442,7 +448,7 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C log.Warningf("failed to set secgroup %s(%s) project sharable", secgroup.Name, secgroup.Id) } } - cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id) + cache, err := manager.NewCache(ctx, userCred, secgroup.Id, vpcId, vpc.CloudregionId, provider.Id, added[i].GetProjectId()) if err != nil { syncResult.AddError(fmt.Errorf("failed to create secgroup cache for secgroup %s(%s) provider: %s: %s", secgroup.Name, secgroup.Name, provider.Name, err)) continue diff --git a/pkg/compute/regiondrivers/base.go b/pkg/compute/regiondrivers/base.go index 7ef5b018bb..ae4969cb35 100644 --- a/pkg/compute/regiondrivers/base.go +++ b/pkg/compute/regiondrivers/base.go @@ -30,6 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SBaseRegionDriver struct { @@ -217,7 +218,7 @@ func (self *SBaseRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mc return fmt.Errorf("Not implement RequestDeleteVpc") } -func (self *SBaseRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, task taskman.ITask) error { +func (self *SBaseRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, remoteProjectId string, task taskman.ITask) error { return fmt.Errorf("Not Implemented RequestCacheSecurityGroup") } @@ -241,11 +242,15 @@ func (self *SBaseRegionDriver) GetDefaultSecurityGroupVpcId() string { return api.NORMAL_VPC_ID } +func (self *SBaseRegionDriver) GetSecurityGroupPublicScope() rbacutils.TRbacScope { + return rbacutils.ScopeSystem +} + func (self *SBaseRegionDriver) GetSecurityGroupVpcId(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, host *models.SHost, vpc *models.SVpc, classic bool) (string, error) { return "", cloudprovider.ErrNotImplemented } -func (self *SBaseRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup) (string, error) { +func (self *SBaseRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup, removeProjectId string) (string, error) { return "", fmt.Errorf("Not Implemented RequestSyncSecurityGroup") } diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index 0429be6e9a..aab701e77c 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -39,6 +39,7 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/billing" "yunion.io/x/onecloud/pkg/util/rand" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SManagedVirtualizationRegionDriver struct { @@ -1457,7 +1458,7 @@ func (self *SManagedVirtualizationRegionDriver) GetSecurityGroupVpcId(ctx contex return region.GetDriver().GetDefaultSecurityGroupVpcId(), nil } -func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup) (string, error) { +func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, vpcId string, vpc *models.SVpc, secgroup *models.SSecurityGroup, remoteProjectId string) (string, error) { lockman.LockRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-%s-%s", secgroup.Id, vpcId, vpc.ManagerId)) defer lockman.ReleaseRawObject(ctx, "secgroupcache", fmt.Sprintf("%s-%s-%s", secgroup.Id, vpcId, vpc.ManagerId)) @@ -1466,7 +1467,11 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con return "", errors.Wrap(err, "vpc.GetRegon") } - cache, err := models.SecurityGroupCacheManager.Register(ctx, userCred, secgroup.Id, vpcId, region.Id, vpc.ManagerId) + if region.GetDriver().GetSecurityGroupPublicScope() == rbacutils.ScopeSystem { + remoteProjectId = "" + } + + cache, err := models.SecurityGroupCacheManager.Register(ctx, userCred, secgroup.Id, vpcId, region.Id, vpc.ManagerId, remoteProjectId) if err != nil { return "", errors.Wrap(err, "SSecurityGroupCache.Register") } @@ -1495,9 +1500,13 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con randomString := func(prefix string, length int) string { return fmt.Sprintf("%s-%s", prefix, rand.String(length)) } - groupName := randomString(secgroup.Name, 1) + opts := &cloudprovider.SecurityGroupFilterOptions{ + Name: randomString(secgroup.Name, 1), + VpcId: vpcId, + ProjectId: remoteProjectId, + } for i := 2; i < 30; i++ { - _, err := iRegion.GetISecurityGroupByName(vpc.ExternalId, groupName) + _, err := iRegion.GetISecurityGroupByName(opts) if err != nil { if errors.Cause(err) == cloudprovider.ErrNotFound { break @@ -1506,13 +1515,14 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con return "", err } } - groupName = randomString(secgroup.Name, i) + opts.Name = randomString(secgroup.Name, i) } conf := &cloudprovider.SecurityGroupCreateInput{ - Name: groupName, - Desc: secgroup.Description, - VpcId: vpcId, - Rules: secgroup.GetSecRules(""), + Name: opts.Name, + Desc: secgroup.Description, + VpcId: vpcId, + ProjectId: remoteProjectId, + Rules: secgroup.GetSecRules(""), } iSecgroup, err = iRegion.CreateISecurityGroup(conf) if err != nil { @@ -1560,14 +1570,13 @@ func (self *SManagedVirtualizationRegionDriver) RequestSyncSecurityGroup(ctx con return cache.ExternalId, nil } -func (self *SManagedVirtualizationRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, task taskman.ITask) error { - +func (self *SManagedVirtualizationRegionDriver) RequestCacheSecurityGroup(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, vpc *models.SVpc, secgroup *models.SSecurityGroup, classic bool, removeProjectId string, task taskman.ITask) error { vpcId, err := region.GetDriver().GetSecurityGroupVpcId(ctx, userCred, region, nil, vpc, classic) if err != nil { return errors.Wrap(err, "GetSecurityGroupVpcId") } taskman.LocalTaskRun(task, func() (jsonutils.JSONObject, error) { - _, err := self.RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup) + _, err := self.RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup, removeProjectId) return nil, err }) return nil @@ -1636,7 +1645,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateDBInstance(ctx cont if err != nil { return nil, errors.Wrap(err, "GetSecurityGroupVpcId") } - desc.SecgroupId, err = region.GetDriver().RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup) + desc.SecgroupId, err = region.GetDriver().RequestSyncSecurityGroup(ctx, userCred, vpcId, vpc, secgroup, desc.ProjectId) if err != nil { return nil, errors.Wrap(err, "SyncSecurityGroup") } diff --git a/pkg/compute/regiondrivers/openstack.go b/pkg/compute/regiondrivers/openstack.go index 02eeae19ca..0fa4ba277b 100644 --- a/pkg/compute/regiondrivers/openstack.go +++ b/pkg/compute/regiondrivers/openstack.go @@ -26,6 +26,7 @@ import ( "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SOpenStackRegionDriver struct { @@ -61,6 +62,10 @@ func (self *SOpenStackRegionDriver) IsOnlySupportAllowRules() bool { return true } +func (self *SOpenStackRegionDriver) GetSecurityGroupPublicScope() rbacutils.TRbacScope { + return rbacutils.ScopeProject +} + func (self *SOpenStackRegionDriver) GetProvider() string { return api.CLOUD_PROVIDER_OPENSTACK } diff --git a/pkg/compute/tasks/network_create_task.go b/pkg/compute/tasks/network_create_task.go index f2c626d981..232903f93f 100644 --- a/pkg/compute/tasks/network_create_task.go +++ b/pkg/compute/tasks/network_create_task.go @@ -72,7 +72,19 @@ func (self *NetworkCreateTask) OnInit(ctx context.Context, obj db.IStandaloneMod return } - inet, err := iwire.CreateINetwork(network.Name, prefix.String(), network.Description) + opts := cloudprovider.SNetworkCreateOptions{ + Name: network.Name, + Cidr: prefix.String(), + Desc: network.Description, + } + + provider := wire.GetCloudprovider() + opts.ProjectId, err = provider.SyncProject(ctx, self.GetUserCred(), network.ProjectId) + if err != nil { + log.Errorf("failed to sync project %s for create %s network %s error: %v", network.ProjectId, provider.Provider, network.Name, err) + } + + inet, err := iwire.CreateINetwork(&opts) if err != nil { self.taskFailed(ctx, network, "createinetwork", err) return diff --git a/pkg/compute/tasks/secgroup_group_cache_task.go b/pkg/compute/tasks/secgroup_group_cache_task.go index 9d57d19141..307c9a145f 100644 --- a/pkg/compute/tasks/secgroup_group_cache_task.go +++ b/pkg/compute/tasks/secgroup_group_cache_task.go @@ -69,7 +69,7 @@ func (self *SecurityGroupCacheTask) OnInit(ctx context.Context, obj db.IStandalo self.SetStage("OnCacheSecurityGroupComplete", nil) - err = region.GetDriver().RequestCacheSecurityGroup(ctx, self.UserCred, region, vpc, secgroup, classic, self) + err = region.GetDriver().RequestCacheSecurityGroup(ctx, self.UserCred, region, vpc, secgroup, classic, "", self) if err != nil { self.taskFailed(ctx, secgroup, jsonutils.Marshal(err)) return diff --git a/pkg/multicloud/aliyun/region.go b/pkg/multicloud/aliyun/region.go index fa8e275baf..0a027911d5 100644 --- a/pkg/multicloud/aliyun/region.go +++ b/pkg/multicloud/aliyun/region.go @@ -769,8 +769,8 @@ func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.I return secgroup, nil } -func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, total, err := region.GetSecurityGroups(vpcId, name, []string{}, 0, 0) +func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, total, err := region.GetSecurityGroups(opts.VpcId, opts.Name, []string{}, 0, 0) if err != nil { return nil, err } diff --git a/pkg/multicloud/aliyun/wire.go b/pkg/multicloud/aliyun/wire.go index 59f377370b..d775ee103a 100644 --- a/pkg/multicloud/aliyun/wire.go +++ b/pkg/multicloud/aliyun/wire.go @@ -112,8 +112,8 @@ func (self *SWire) GetBandwidth() int { return 10000 } -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - vswitchId, err := self.zone.region.createVSwitch(self.zone.ZoneId, self.vpc.VpcId, name, cidr, desc) +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + vswitchId, err := self.zone.region.createVSwitch(self.zone.ZoneId, self.vpc.VpcId, opts.Name, opts.Cidr, opts.Desc) if err != nil { log.Errorf("createVSwitch error %s", err) return nil, err diff --git a/pkg/multicloud/aws/region.go b/pkg/multicloud/aws/region.go index fe769e3809..9ad3d0505c 100644 --- a/pkg/multicloud/aws/region.go +++ b/pkg/multicloud/aws/region.go @@ -1027,8 +1027,8 @@ func (self *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICl return &secgroups[0], nil } -func (self *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, total, err := self.GetSecurityGroups(vpcId, name, "", 0, 1) +func (self *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, total, err := self.GetSecurityGroups(opts.VpcId, opts.Name, "", 0, 1) if err != nil { return nil, err } diff --git a/pkg/multicloud/aws/wire.go b/pkg/multicloud/aws/wire.go index 17e48e8f93..46952d6d76 100644 --- a/pkg/multicloud/aws/wire.go +++ b/pkg/multicloud/aws/wire.go @@ -93,8 +93,8 @@ func (self *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, e return nil, ErrorNotFound() } -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - networkId, err := self.zone.region.createNetwork(self.zone.ZoneId, self.vpc.VpcId, name, cidr, desc) +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + networkId, err := self.zone.region.createNetwork(self.zone.ZoneId, self.vpc.VpcId, opts.Name, opts.Cidr, opts.Desc) if err != nil { log.Errorf("createNetwork error %s", err) return nil, err diff --git a/pkg/multicloud/azure/classic_wire.go b/pkg/multicloud/azure/classic_wire.go index 7a7c2f5f69..b6bb2beee1 100644 --- a/pkg/multicloud/azure/classic_wire.go +++ b/pkg/multicloud/azure/classic_wire.go @@ -74,7 +74,7 @@ func (self *SClassicWire) addNetwork(network *SClassicNetwork) { } } -func (self *SClassicWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { +func (self *SClassicWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { return nil, cloudprovider.ErrNotImplemented // if network, err := self.zone.region.createNetwork(self.vpc, name, cidr, desc); err != nil { // return nil, err diff --git a/pkg/multicloud/azure/region.go b/pkg/multicloud/azure/region.go index 8d626ec323..6b07e03626 100644 --- a/pkg/multicloud/azure/region.go +++ b/pkg/multicloud/azure/region.go @@ -540,9 +540,9 @@ func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.I return region.GetSecurityGroupDetails(secgroupId) } -func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - if strings.Contains(strings.ToLower(vpcId), "microsoft.classicnetwork") { - secgroups, err := region.GetClassicSecurityGroups(name) +func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + if strings.Contains(strings.ToLower(opts.VpcId), "microsoft.classicnetwork") { + secgroups, err := region.GetClassicSecurityGroups(opts.Name) if err != nil { return nil, err } @@ -554,7 +554,7 @@ func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloud } return &secgroups[0], nil } - secgroups, err := region.GetSecurityGroups(name) + secgroups, err := region.GetSecurityGroups(opts.Name) if err != nil { return nil, err } diff --git a/pkg/multicloud/azure/wire.go b/pkg/multicloud/azure/wire.go index 0eb5e28f4a..5394310776 100644 --- a/pkg/multicloud/azure/wire.go +++ b/pkg/multicloud/azure/wire.go @@ -100,8 +100,8 @@ func (self *SRegion) createNetwork(vpc *SVpc, subnetName string, cidr string, de return &subnet, nil } -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - network, err := self.zone.region.createNetwork(self.vpc, name, cidr, desc) +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + network, err := self.zone.region.createNetwork(self.vpc, opts.Name, opts.Cidr, opts.Desc) if err != nil { return nil, err } diff --git a/pkg/multicloud/ctyun/region.go b/pkg/multicloud/ctyun/region.go index 620d36b5d7..69c38423a9 100644 --- a/pkg/multicloud/ctyun/region.go +++ b/pkg/multicloud/ctyun/region.go @@ -131,14 +131,14 @@ func (self *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICl return self.GetSecurityGroupDetails(secgroupId) } -func (self *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - segroups, err := self.GetSecurityGroups(vpcId) +func (self *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + segroups, err := self.GetSecurityGroups(opts.VpcId) if err != nil { return nil, errors.Wrap(err, "SRegion.GetISecurityGroupByName.GetSecurityGroups") } for i := range segroups { - if segroups[i].Name == name { + if segroups[i].Name == opts.Name { return &segroups[i], nil } } diff --git a/pkg/multicloud/ctyun/wire.go b/pkg/multicloud/ctyun/wire.go index 525c5434f7..80255135f1 100644 --- a/pkg/multicloud/ctyun/wire.go +++ b/pkg/multicloud/ctyun/wire.go @@ -106,7 +106,7 @@ func getDefaultGateWay(cidr string) (string, error) { return startIp.String(), nil } -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { var ZoneId string if len(self.inetworks) > 0 { ZoneId = self.inetworks[0].(*SNetwork).ZoneID @@ -117,7 +117,7 @@ func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudp return nil, fmt.Errorf("SWire.CreateINetwork region %s zone is empty", self.region.GetName()) } } - network, err := self.region.CreateNetwork(self.vpc.GetId(), ZoneId, name, cidr, "true") + network, err := self.region.CreateNetwork(self.vpc.GetId(), ZoneId, opts.Name, opts.Cidr, "true") if err != nil { return nil, errors.Wrap(err, "SWire.CreateINetwork.CreateNetwork") } diff --git a/pkg/multicloud/google/securitygroup.go b/pkg/multicloud/google/securitygroup.go index 356f3ecd20..47a64cf401 100644 --- a/pkg/multicloud/google/securitygroup.go +++ b/pkg/multicloud/google/securitygroup.go @@ -329,8 +329,8 @@ func (region *SRegion) GetISecurityGroupById(id string) (cloudprovider.ICloudSec return nil, cloudprovider.ErrNotFound } -func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - ivpc, err := region.GetIVpcById(vpcId) +func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + ivpc, err := region.GetIVpcById(opts.VpcId) if err != nil { return nil, err } @@ -339,7 +339,7 @@ func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloud return nil, errors.Wrap(err, "ivpc.GetISecurityGroups") } for _, secgroup := range secgroups { - if strings.ToLower(secgroup.GetName()) == strings.ToLower(name) { + if strings.ToLower(secgroup.GetName()) == strings.ToLower(opts.Name) { return secgroup, nil } } diff --git a/pkg/multicloud/google/wire.go b/pkg/multicloud/google/wire.go index b9972d597b..1d75438427 100644 --- a/pkg/multicloud/google/wire.go +++ b/pkg/multicloud/google/wire.go @@ -38,8 +38,8 @@ func (wire *SWire) GetName() string { return wire.vpc.GetName() } -func (wire *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - network, err := wire.vpc.region.CreateNetwork(name, wire.vpc.globalnetwork.SelfLink, cidr, desc) +func (wire *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + network, err := wire.vpc.region.CreateNetwork(opts.Name, wire.vpc.globalnetwork.SelfLink, opts.Cidr, opts.Desc) if err != nil { return nil, err } diff --git a/pkg/multicloud/host_base.go b/pkg/multicloud/host_base.go index 4e9ce17152..214d6d6b45 100644 --- a/pkg/multicloud/host_base.go +++ b/pkg/multicloud/host_base.go @@ -15,6 +15,7 @@ package multicloud type SHostBase struct { + SResourceBase } func (self *SHostBase) GetCpuCmtbound() float32 { diff --git a/pkg/multicloud/huawei/region.go b/pkg/multicloud/huawei/region.go index 6d2e978260..8c9f661318 100644 --- a/pkg/multicloud/huawei/region.go +++ b/pkg/multicloud/huawei/region.go @@ -516,8 +516,8 @@ func (self *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICl return self.GetSecurityGroupDetails(secgroupId) } -func (self *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, err := self.GetSecurityGroups(vpcId, name) +func (self *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, err := self.GetSecurityGroups(opts.VpcId, opts.Name) if err != nil { return nil, err } diff --git a/pkg/multicloud/huawei/wire.go b/pkg/multicloud/huawei/wire.go index 9320c82384..e6a2de4d0b 100644 --- a/pkg/multicloud/huawei/wire.go +++ b/pkg/multicloud/huawei/wire.go @@ -101,8 +101,8 @@ func (self *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, e 华为云子网可用区，类似一个zone标签。即使指定了zone子网在整个region依然是可用。 通过华为web控制台创建子网需要指定可用区。这里是不指定的。 */ -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - networkId, err := self.region.createNetwork(self.vpc.GetId(), name, cidr, desc) +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + networkId, err := self.region.createNetwork(self.vpc.GetId(), opts.Name, opts.Cidr, opts.Desc) if err != nil { log.Errorf("createNetwork error %s", err) return nil, err diff --git a/pkg/multicloud/openstack/disk.go b/pkg/multicloud/openstack/disk.go index d8df156dc7..34259e30b4 100644 --- a/pkg/multicloud/openstack/disk.go +++ b/pkg/multicloud/openstack/disk.go @@ -17,6 +17,7 @@ package openstack import ( "context" "fmt" + "net/url" "strings" "time" @@ -28,6 +29,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" + "yunion.io/x/onecloud/pkg/util/version" ) const ( @@ -58,12 +60,12 @@ const ( ) type Attachment struct { - ServerID string - AttachmentID string + ServerId string + AttachmentId string HostName string - VolumeID string + VolumeId string Device string - ID string + Id string } type Link struct { @@ -78,18 +80,17 @@ type VolumeImageMetadata struct { MinRAM int DiskFormat string ImageName string - ImageID string + ImageId string ContainerFormat string MinDisk int Size int } type SDisk struct { - nova *SNovaStorage storage *SStorage multicloud.SDisk - ID string + Id string Name string MigrationStatus string @@ -100,10 +101,10 @@ type SDisk struct { Host string `json:"os-vol-host-attr:host"` Encrypted bool ReplicationStatus string - SnapshotID string + SnapshotId string Size int - UserID string - TenantID string `json:"os-vol-tenant-attr:tenant_id"` + UserId string + TenantId string `json:"os-vol-tenant-attr:tenant_id"` Migstat string `json:"os-vol-mig-status-attr:migstat"` Metadata Metadata @@ -111,9 +112,9 @@ type SDisk struct { Description string Multiattach string SourceVolid string - ConsistencygroupID string + ConsistencygroupId string VolumeImageMetadata VolumeImageMetadata - NameID string `json:"os-vol-mig-status-attr:name_id"` + NameId string `json:"os-vol-mig-status-attr:name_id"` Bootable bool CreatedAt time.Time VolumeType string @@ -126,61 +127,40 @@ func (disk *SDisk) GetMetadata() *jsonutils.JSONDict { return data } -func (region *SRegion) GetDisks(category, volumeBackendName string) ([]SDisk, error) { - url := "/volumes/detail" +func (region *SRegion) GetDisks() ([]SDisk, error) { disks := []SDisk{} - for len(url) > 0 { - _, resp, err := region.CinderList(url, "", nil) + resource := "/volumes/detail" + query := url.Values{} + query.Set("all_tenants", "true") + for { + resp, err := region.bsList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "bsList") } - _disks := []SDisk{} - err = resp.Unmarshal(&_disks, "volumes") + part := struct { + Volumes []SDisk + VolumesLinks SNextLinks + }{} + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_disks, "volumes")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - disks = append(disks, _disks...) - url = "" - if resp.Contains("volumes_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "volumes_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "volumes_links")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } - } + disks = append(disks, part.Volumes...) + marker := part.VolumesLinks.GetNextMark() + if len(marker) == 0 { + break } + query.Set("marker", marker) } - result := []SDisk{} - for _, disk := range disks { - if len(category) > 0 { - if disk.VolumeType == category { - result = append(result, disk) - } - } else if len(volumeBackendName) > 0 { - if strings.HasSuffix(disk.Host, "#"+volumeBackendName) { - result = append(result, disk) - } - } else { - result = append(result, disk) - } - } - return result, nil + return disks, nil } func (disk *SDisk) GetId() string { - return disk.ID + return disk.Id } func (disk *SDisk) Delete(ctx context.Context) error { - if disk.nova != nil { - return nil - } - err := disk.storage.zone.region.DeleteDisk(disk.ID) + err := disk.storage.zone.region.DeleteDisk(disk.Id) if err != nil { return err } @@ -191,12 +171,12 @@ func (disk *SDisk) attachInstances(attachments []Attachment) error { for _, attachment := range attachments { startTime := time.Now() for time.Now().Sub(startTime) < 5*time.Minute { - if err := disk.storage.zone.region.AttachDisk(attachment.ServerID, disk.ID); err != nil { + if err := disk.storage.zone.region.AttachDisk(attachment.ServerId, disk.Id); err != nil { if strings.Contains(err.Error(), "status must be available or downloading") { time.Sleep(time.Second * 10) continue } - log.Errorf("recover attach disk %s => instance %s error: %v", disk.ID, attachment.ServerID, err) + log.Errorf("recover attach disk %s => instance %s error: %v", disk.Id, attachment.ServerId, err) return err } else { return nil @@ -207,18 +187,24 @@ func (disk *SDisk) attachInstances(attachments []Attachment) error { } func (disk *SDisk) Resize(ctx context.Context, sizeMb int64) error { - if disk.nova != nil { - return cloudprovider.ErrNotSupported + maxVersion := "" + for _, service := range []string{OPENSTACK_SERVICE_VOLUMEV3, OPENSTACK_SERVICE_VOLUMEV2, OPENSTACK_SERVICE_VOLUME} { + maxVersion, _ = disk.storage.zone.region.GetMaxVersion(service) + if len(maxVersion) > 0 { + break + } + } + if version.GE(maxVersion, "3.42") { + return disk.storage.zone.region.ResizeDisk(disk.Id, sizeMb) } instanceIds := []string{} - for _, attachement := range disk.Attachments { - if err := disk.storage.zone.region.DetachDisk(attachement.ServerID, disk.ID); err != nil { + if err := disk.storage.zone.region.DetachDisk(attachement.ServerId, disk.Id); err != nil { return err } - instanceIds = append(instanceIds, attachement.ServerID) + instanceIds = append(instanceIds, attachement.ServerId) } - err := disk.storage.zone.region.ResizeDisk(disk.ID, sizeMb) + err := disk.storage.zone.region.ResizeDisk(disk.Id, sizeMb) if err != nil { disk.attachInstances(disk.Attachments) return err @@ -230,11 +216,11 @@ func (disk *SDisk) GetName() string { if len(disk.Name) > 0 { return disk.Name } - return disk.ID + return disk.Id } func (disk *SDisk) GetGlobalId() string { - return disk.ID + return disk.Id } func (disk *SDisk) IsEmulated() bool { @@ -242,9 +228,6 @@ func (disk *SDisk) IsEmulated() bool { } func (disk *SDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { - if disk.nova != nil { - return disk.nova, nil - } return disk.storage, nil } @@ -270,10 +253,7 @@ func (disk *SDisk) GetStatus() string { } func (disk *SDisk) Refresh() error { - if disk.nova != nil { - return nil - } - _disk, err := disk.storage.zone.region.GetDisk(disk.ID) + _disk, err := disk.storage.zone.region.GetDisk(disk.Id) if err != nil { return err } @@ -281,10 +261,7 @@ func (disk *SDisk) Refresh() error { } func (disk *SDisk) ResizeDisk(sizeMb int64) error { - if disk.nova != nil { - return cloudprovider.ErrNotSupported - } - return disk.storage.zone.region.ResizeDisk(disk.ID, sizeMb) + return disk.storage.zone.region.ResizeDisk(disk.Id, sizeMb) } func (disk *SDisk) GetDiskFormat() string { @@ -300,7 +277,7 @@ func (disk *SDisk) GetIsAutoDelete() bool { } func (disk *SDisk) GetTemplateId() string { - return disk.VolumeImageMetadata.ImageID + return disk.VolumeImageMetadata.ImageId } func (disk *SDisk) GetDiskType() string { @@ -330,11 +307,11 @@ func (disk *SDisk) GetMountpoint() string { return "" } -func (region *SRegion) CreateDisk(imageRef string, category string, name string, sizeGb int, desc string, projectId string) (*SDisk, error) { +func (region *SRegion) CreateDisk(imageRef string, volumeType string, name string, sizeGb int, desc string, projectId string) (*SDisk, error) { params := map[string]map[string]interface{}{ "volume": { "size": sizeGb, - "volume_type": category, + "volume_type": volumeType, "name": name, "description": desc, }, @@ -342,14 +319,15 @@ func (region *SRegion) CreateDisk(imageRef string, category string, name string, if len(imageRef) > 0 { params["volume"]["imageRef"] = imageRef } - _, resp, err := region.CinderCreate(projectId, "/volumes", "", jsonutils.Marshal(params)) + resp, err := region.bsCreate(projectId, "/volumes", params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "bsCreate") } disk := &SDisk{} - if err := resp.Unmarshal(disk, "volume"); err != nil { - return nil, err + err = resp.Unmarshal(disk, "volume") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") } //这里由于不好初始化disk的storage就手动循环了,如果是通过镜像创建，有个下载过程,比较慢，等待时间较长 startTime := time.Now() @@ -361,7 +339,7 @@ func (region *SRegion) CreateDisk(imageRef string, category string, name string, for time.Now().Sub(startTime) < timeout { disk, err = region.GetDisk(disk.GetGlobalId()) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "GetDisk(%s)", disk.GetGlobalId()) } log.Debugf("disk status %s expect %s", disk.GetStatus(), api.DISK_READY) status := disk.GetStatus() @@ -369,29 +347,37 @@ func (region *SRegion) CreateDisk(imageRef string, category string, name string, break } if status == api.DISK_ALLOC_FAILED { - region.DeleteDisk(disk.GetGlobalId()) - return nil, fmt.Errorf("allocate disk failed, status is error") + messages, _ := region.GetMessages(disk.Id) + if len(messages) > 0 { + return nil, fmt.Errorf("allocate disk %s failed, status is %s message: %s", disk.Name, disk.Status, messages[0].UserMessage) + } + return nil, fmt.Errorf("allocate disk %s failed, status is %s", disk.Name, disk.Status) } time.Sleep(time.Second * 10) } if disk.GetStatus() != api.DISK_READY { - region.DeleteDisk(disk.GetGlobalId()) return nil, fmt.Errorf("timeout for waitting disk ready, current status: %s", disk.Status) } return disk, nil } func (region *SRegion) GetDisk(diskId string) (*SDisk, error) { - _, resp, err := region.CinderGet("/volumes/"+diskId, "", nil) + resource := fmt.Sprintf("/volumes/%s", diskId) + resp, err := region.bsGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "bsGet") } disk := &SDisk{} - return disk, resp.Unmarshal(disk, "volume") + err = resp.Unmarshal(disk, "volume") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return disk, nil } func (region *SRegion) DeleteDisk(diskId string) error { - _, err := region.CinderDelete("/volumes/"+diskId, "") + resource := fmt.Sprintf("/volumes/%s", diskId) + _, err := region.bsDelete(resource) return err } @@ -401,27 +387,24 @@ func (region *SRegion) ResizeDisk(diskId string, sizeMb int64) error { "new_size": sizeMb / 1024, }, } - _, _, err := region.CinderAction(fmt.Sprintf("/volumes/%s/action", diskId), "", jsonutils.Marshal(params)) + resource := fmt.Sprintf("/volumes/%s/action", diskId) + _, err := region.bsPost(resource, params) return err } func (region *SRegion) ResetDisk(diskId, snapshotId string) error { - //目前测试接口不能使用 - return cloudprovider.ErrNotSupported - // params := map[string]map[string]interface{}{ - // "revert": { - // "snapshot_id": snapshotId, - // }, - // } - // _, _, err := region.CinderAction(fmt.Sprintf("/volumes/%s/action", diskId), "3.40", jsonutils.Marshal(params)) - // return err + params := map[string]map[string]interface{}{ + "revert": { + "snapshot_id": snapshotId, + }, + } + resource := fmt.Sprintf("/volumes/%s/action", diskId) + _, err := region.bsPost(resource, params) + return err } func (disk *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { - if disk.nova != nil { - return nil, cloudprovider.ErrNotSupported - } - snapshot, err := disk.storage.zone.region.CreateSnapshot(disk.ID, name, desc) + snapshot, err := disk.storage.zone.region.CreateSnapshot(disk.Id, name, desc) if err != nil { return nil, err } @@ -429,24 +412,24 @@ func (disk *SDisk) CreateISnapshot(ctx context.Context, name, desc string) (clou } func (disk *SDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) { - if disk.nova != nil { - return nil, cloudprovider.ErrNotFound - } return disk.storage.zone.region.GetISnapshotById(snapshotId) } func (disk *SDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { - if disk.nova != nil { - return []cloudprovider.ICloudSnapshot{}, nil + snapshots, err := disk.storage.zone.region.GetSnapshots(disk.Id) + if err != nil { + return nil, errors.Wrapf(err, "GetSnapshots(%s)", disk.Id) } - return disk.storage.zone.region.GetSnapshots(disk.ID) + isnapshots := []cloudprovider.ICloudSnapshot{} + for i := range snapshots { + snapshots[i].region = disk.storage.zone.region + isnapshots = append(isnapshots, &snapshots[i]) + } + return isnapshots, nil } func (disk *SDisk) Reset(ctx context.Context, snapshotId string) (string, error) { - if disk.nova != nil { - return "", cloudprovider.ErrNotSupported - } - return disk.ID, disk.storage.zone.region.ResetDisk(disk.ID, snapshotId) + return disk.Id, disk.storage.zone.region.ResetDisk(disk.Id, snapshotId) } func (disk *SDisk) GetBillingType() string { @@ -470,5 +453,5 @@ func (disk *SDisk) Rebuild(ctx context.Context) error { } func (disk *SDisk) GetProjectId() string { - return disk.TenantID + return disk.TenantId } diff --git a/pkg/multicloud/openstack/eip.go b/pkg/multicloud/openstack/eip.go index a82e672167..de26166230 100644 --- a/pkg/multicloud/openstack/eip.go +++ b/pkg/multicloud/openstack/eip.go @@ -56,33 +56,28 @@ type SEipAddress struct { RevisionNumber int `json:"revision_number"` ProjectId string `json:"project_id"` PortId string `json:"port_id"` - ID string `json:"id"` + Id string `json:"id"` QosPolicyId string `json:"qos_policy_id"` } func (region *SRegion) GetEip(eipId string) (*SEipAddress, error) { - _, resp, err := region.Get("network", "/v2.0/floatingips/"+eipId, "", nil) + resource := fmt.Sprintf("/v2.0/floatingips/%s", eipId) + resp, err := region.vpcGet(resource) if err != nil { return nil, err } eip := &SEipAddress{region: region} - return eip, resp.Unmarshal(eip, "floatingip") + err = resp.Unmarshal(eip, "floatingip") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return eip, nil } func (region *SRegion) GetEipByIp(ip string) (*SEipAddress, error) { - params := url.Values{} - if len(ip) == 0 { - return nil, cloudprovider.ErrNotFound - } - params.Add("floating_ip_address", ip) - _, resp, err := region.List("network", "/v2.0/floatingips?"+params.Encode(), "", nil) + eips, err := region.GetEips(ip) if err != nil { - return nil, err - } - eips := []SEipAddress{} - err = resp.Unmarshal(&eips, "floatingips") - if err != nil { - return nil, err + return nil, errors.Wrap(err, "GetEips") } if len(eips) == 1 { eips[0].region = region @@ -94,40 +89,38 @@ func (region *SRegion) GetEipByIp(ip string) (*SEipAddress, error) { return nil, cloudprovider.ErrDuplicateId } -func (region *SRegion) GetEips() ([]SEipAddress, error) { - url := "/v2.0/floatingips" +func (region *SRegion) GetEips(ip string) ([]SEipAddress, error) { + resource := "/v2.0/floatingips" eips := []SEipAddress{} - for len(url) > 0 { - _, resp, err := region.List("network", url, "", nil) + query := url.Values{} + if len(ip) > 0 { + query.Set("floating_ip_address", ip) + } + for { + part := struct { + Floatingips []SEipAddress + FloatingipsLinks SNextLinks + }{} + resp, err := region.vpcList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vpcList") } - _eips := []SEipAddress{} - err = resp.Unmarshal(&_eips, "floatingips") + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_eips, "floatingips")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - eips = append(eips, _eips...) - url = "" - if resp.Contains("floatingips_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "floatingips_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "floatingips_links")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } - } + eips = append(eips, part.Floatingips...) + marker := part.FloatingipsLinks.GetNextMark() + if len(marker) == 0 { + break } + query.Set("marker", marker) } return eips, nil } func (eip *SEipAddress) GetId() string { - return eip.ID + return eip.Id } func (eip *SEipAddress) GetName() string { @@ -135,7 +128,7 @@ func (eip *SEipAddress) GetName() string { } func (eip *SEipAddress) GetGlobalId() string { - return eip.ID + return eip.Id } func (eip *SEipAddress) GetStatus() string { @@ -147,17 +140,17 @@ func (eip *SEipAddress) GetStatus() string { case "ERROR": return api.EIP_STATUS_UNKNOWN default: - log.Errorf("Unknown eip %s status %s", eip.ID, eip.Status) + log.Errorf("Unknown eip %s status %s", eip.Id, eip.Status) return api.EIP_STATUS_UNKNOWN } } func (eip *SEipAddress) Refresh() error { - new, err := eip.region.GetEip(eip.ID) + _eip, err := eip.region.GetEip(eip.Id) if err != nil { return err } - return jsonutils.Update(eip, new) + return jsonutils.Update(eip, _eip) } func (eip *SEipAddress) IsEmulated() bool { @@ -208,7 +201,7 @@ func (eip *SEipAddress) GetExpiredAt() time.Time { } func (eip *SEipAddress) Delete() error { - return eip.region.DeleteEip(eip.ID) + return eip.region.DeleteEip(eip.Id) } func (eip *SEipAddress) GetBandwidth() int { @@ -223,7 +216,7 @@ func (eip *SEipAddress) GetINetworkId() string { } for _, network := range networks { if network.Contains(eip.FloatingIPAddress) { - return network.ID + return network.Id } } log.Errorf("failed to find eip %s(%s) networkId", eip.FloatingIPAddress, eip.FloatingNetworkId) @@ -235,11 +228,11 @@ func (eip *SEipAddress) GetInternetChargeType() string { } func (eip *SEipAddress) Associate(conf *cloudprovider.AssociateConfig) error { - return eip.region.AssociateEip(conf.InstanceId, eip.ID) + return eip.region.AssociateEip(conf.InstanceId, eip.Id) } func (eip *SEipAddress) Dissociate() error { - return eip.region.DisassociateEip(eip.ID) + return eip.region.DisassociateEip(eip.Id) } func (eip *SEipAddress) ChangeBandwidth(bw int) error { @@ -250,27 +243,30 @@ func (eip *SEipAddress) GetProjectId() string { return eip.ProjectId } -func (region *SRegion) CreateEip(eip *cloudprovider.SEip) (*SEipAddress, error) { - network, err := region.GetNetwork(eip.NetworkExternalId) - if err != nil { - log.Errorf("failed to get subnet %s", eip.NetworkExternalId) - return nil, err - } - parmas := map[string]map[string]string{ - "floatingip": { - "floating_network_id": network.NetworkID, - "subnet_id": network.ID, +func (region *SRegion) CreateEip(vpcId, networkId, ip string, projectId string) (*SEipAddress, error) { + params := map[string]map[string]string{ + "floatingip": map[string]string{ + "floating_network_id": vpcId, + "subnet_id": networkId, }, } - if len(eip.IP) > 0 { - parmas["floatingip"]["floating_ip_address"] = eip.IP + if len(projectId) > 0 { + params["floatingip"]["tenant_id"] = projectId } - _, resp, err := region.PostWithProject(eip.ProjectId, "network", "/v2.0/floatingips", "", jsonutils.Marshal(parmas)) + if len(ip) > 0 { + params["floatingip"]["floating_ip_address"] = ip + } + resource := "/v2.0/floatingips" + resp, err := region.vpcPost(resource, params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vpcPost") } - ieip := &SEipAddress{region: region} - return ieip, resp.Unmarshal(ieip, "floatingip") + eip := &SEipAddress{region: region} + err = resp.Unmarshal(eip, "floatingip") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return eip, nil } func (region *SRegion) AssociateEip(instanceId, eipId string) error { @@ -281,7 +277,7 @@ func (region *SRegion) AssociateEip(instanceId, eipId string) error { for networkName, address := range instance.Addresses { for i := 0; i < len(address); i++ { if instance.Addresses[networkName][i].Type == "fixed" { - ports, err := region.GetPorts(instance.Addresses[networkName][i].MacAddr) + ports, err := region.GetPorts(instance.Addresses[networkName][i].MacAddr, "") if err != nil { return err } @@ -292,7 +288,8 @@ func (region *SRegion) AssociateEip(instanceId, eipId string) error { "port_id": ports[0].ID, }, } - _, _, err = region.Update("network", "/v2.0/floatingips/"+eipId, "", jsonutils.Marshal(params)) + resource := "/v2.0/floatingips/" + eipId + _, err = region.vpcUpdate(resource, params) return err } @@ -313,11 +310,12 @@ func (region *SRegion) DisassociateEip(eipId string) error { "port_id": null, }, }`)) - _, _, err := region.Update("network", "/v2.0/floatingips/"+eipId, "", params) + _, err := region.vpcUpdate("/v2.0/floatingips/"+eipId, params) return err } func (region *SRegion) DeleteEip(eipId string) error { - _, err := region.Delete("network", "/v2.0/floatingips/"+eipId, "") + resource := "/v2.0/floatingips/" + eipId + _, err := region.vpcDelete(resource) return err } diff --git a/pkg/multicloud/openstack/flavor.go b/pkg/multicloud/openstack/flavor.go index ccbb6844d2..68b29e7285 100644 --- a/pkg/multicloud/openstack/flavor.go +++ b/pkg/multicloud/openstack/flavor.go @@ -16,6 +16,7 @@ package openstack import ( "fmt" + "net/url" "github.com/pkg/errors" @@ -29,7 +30,7 @@ type SFlavor struct { multicloud.SServerSku region *SRegion - ID string + Id string Disk int Ephemeral int ExtraSpecs ExtraSpecs @@ -41,44 +42,45 @@ type SFlavor struct { } func (region *SRegion) GetFlavors() ([]SFlavor, error) { - url := "/flavors/detail" + resource := "/flavors/detail" flavors := []SFlavor{} - for len(url) > 0 { - _, resp, err := region.List("compute", url, "", nil) + query := url.Values{} + for { + resp, err := region.ecsList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsList") } - _flavors := []SFlavor{} - err = resp.Unmarshal(&_flavors, "flavors") + part := struct { + Flavors []SFlavor + FlavorsLinks SNextLinks + }{} + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_flavors, "flavors")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - flavors = append(flavors, _flavors...) - url = "" - if resp.Contains("flavors_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "flavors_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "flavors_links")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } - } + flavors = append(flavors, part.Flavors...) + + marker := part.FlavorsLinks.GetNextMark() + if len(marker) == 0 { + break } + query.Set("marker", marker) } return flavors, nil } func (region *SRegion) GetFlavor(flavorId string) (*SFlavor, error) { - _, resp, err := region.Get("compute", "/flavors/"+flavorId, "", nil) + resource := fmt.Sprintf("/flavors/%s", flavorId) + resp, err := region.ecsGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsGet") } flavor := &SFlavor{region: region} - return flavor, resp.Unmarshal(flavor, "flavor") + err = resp.Unmarshal(flavor, "flavor") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return flavor, nil } func (region *SRegion) SyncFlavor(name string, cpu, memoryMb, diskGB int) (string, error) { @@ -100,7 +102,7 @@ func (region *SRegion) syncFlavor(name string, cpu, memoryMb, diskGB int) (strin flavorName := flavor.GetName() if (len(name) == 0 || flavorName == name || flavorName == fmt.Sprintf("%s-%d", name, diskGB)) && flavor.GetCpuCoreCount() == cpu && flavor.GetMemorySizeMB() == memoryMb { if diskGB <= flavor.GetSysDiskMaxSizeGB() { - return flavor.ID, nil + return flavor.Id, nil } match = true } @@ -115,7 +117,7 @@ func (region *SRegion) syncFlavor(name string, cpu, memoryMb, diskGB int) (strin if err != nil { return "", errors.Wrap(err, "CreateFlavor") } - return flavor.ID, nil + return flavor.Id, nil } func (region *SRegion) CreateISku(name string, vCpu int, memoryMb int) error { @@ -135,21 +137,25 @@ func (region *SRegion) CreateFlavor(name string, cpu int, memoryMb int, diskGB i "disk": diskGB, }, } - _, resp, err := region.Post("compute", "/flavors", "", jsonutils.Marshal(params)) + resp, err := region.ecsPost("/flavors", params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsPost") } flavor := &SFlavor{region: region} - return flavor, resp.Unmarshal(flavor, "flavor") + err = resp.Unmarshal(flavor, "flavor") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return flavor, nil } func (region *SRegion) DeleteFlavor(flavorId string) error { - _, err := region.Delete("compute", "/flavors/"+flavorId, "") + _, err := region.ecsDelete("/flavors/" + flavorId) return err } func (flavor *SFlavor) Delete() error { - return flavor.region.DeleteFlavor(flavor.ID) + return flavor.region.DeleteFlavor(flavor.Id) } func (flavor *SFlavor) GetMetadata() *jsonutils.JSONDict { @@ -161,11 +167,11 @@ func (flavor *SFlavor) IsEmulated() bool { } func (flavor *SFlavor) Refresh() error { - new, err := flavor.region.GetFlavor(flavor.ID) + _flavor, err := flavor.region.GetFlavor(flavor.Id) if err != nil { return err } - return jsonutils.Update(flavor, new) + return jsonutils.Update(flavor, _flavor) } func (flavor *SFlavor) GetName() string { @@ -176,11 +182,11 @@ func (flavor *SFlavor) GetName() string { } func (flavor *SFlavor) GetId() string { - return flavor.ID + return flavor.Id } func (flavor *SFlavor) GetGlobalId() string { - return flavor.ID + return flavor.Id } func (flavor *SFlavor) GetInstanceTypeFamily() string { diff --git a/pkg/multicloud/openstack/host.go b/pkg/multicloud/openstack/host.go index bc049900eb..8446a3058b 100644 --- a/pkg/multicloud/openstack/host.go +++ b/pkg/multicloud/openstack/host.go @@ -15,486 +15,15 @@ package openstack import ( - "fmt" - "strconv" "time" "github.com/pkg/errors" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/pkg/utils" - - api "yunion.io/x/onecloud/pkg/apis/compute" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/multicloud" ) -const ( - VOLUME_TYPES_API_VERSION = "2.67" -) - -type CpuInfo struct { - Arch string - Model string - Vendor string - Feature []string - Topology map[string]int -} - -type Service struct { - Host string - ID string - DisabledReason string -} - -type SResource struct { - CPU int - DiskGB int - Host string - MemoryMb int - Project string -} - type SHost struct { - multicloud.SHostBase - zone *SZone - - CpuInfo string - - Aggregates []string - CurrentWorkload int - Status string - State string - DiskAvailableLeast int - HostIP string - FreeDiskGB int - FreeRamMB int - HypervisorHostname string - HypervisorType string - HypervisorVersion string - ID string - LocalGB int - LocalGbUsed int - MemoryMB int - MemoryMbUsed int - RunningVms int - Service Service - Vcpus int - VcpusUsed int8 - - // less then version 2.28 HostName string Zone string - Resource []map[string]SResource -} - -func (host *SHost) GetId() string { - if len(host.ID) > 0 { - return host.ID - } - return host.HostName -} - -func (host *SHost) GetName() string { - if len(host.Service.Host) > 0 { - return host.Service.Host - } - return host.HostName -} - -func (host *SHost) GetGlobalId() string { - return host.GetId() -} - -func (host *SHost) GetMetadata() *jsonutils.JSONDict { - return nil -} - -func (host *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { - return host.zone.GetIWires() -} - -func (host *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { - istorages, err := host.zone.GetIStorages() - if err != nil { - return nil, err - } - - storageTypes := host.zone.getAvailableStorages() - storageTypes = append(storageTypes, host.zone.getUnavailableStorage()...) - result := []cloudprovider.ICloudStorage{} - - for _, istorage := range istorages { - if utils.IsInStringArray(istorage.GetStorageType(), storageTypes) { - result = append(result, istorage) - } - } - return result, nil -} - -func (host *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - return host.zone.GetIStorageById(id) -} - -func (host *SHost) GetIVMs() ([]cloudprovider.ICloudVM, error) { - instances, err := host.zone.region.GetInstances(host.GetName()) - if err != nil { - return nil, err - } - iVMs := []cloudprovider.ICloudVM{} - for i := 0; i < len(instances); i++ { - instances[i].host = host - iVMs = append(iVMs, &instances[i]) - } - return iVMs, nil -} - -func (host *SHost) GetIVMById(gid string) (cloudprovider.ICloudVM, error) { - instance, err := host.zone.region.GetInstance(gid) - if err != nil { - return nil, err - } - instance.host = host - return instance, nil -} - -func (host *SHost) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) { - network, err := host.zone.region.GetNetwork(desc.ExternalNetworkId) - if err != nil { - return nil, err - } - - zoneName := "" - for zone, hosts := range host.zone.cachedHosts { - if utils.IsInStringArray(host.GetName(), hosts) { - zoneName = zone - break - } - } - if len(zoneName) == 0 { - return nil, fmt.Errorf("failed to find zone info for host %s", host.GetName()) - } - - secgroups := []map[string]string{} - - for _, secgroupId := range desc.ExternalSecgroupIds { - if secgroupId != SECGROUP_NOT_SUPPORT { - secgroups = append(secgroups, map[string]string{"name": secgroupId}) - } - } - - image, err := host.zone.region.GetImage(desc.ExternalImageId) - if err != nil { - return nil, err - } - - sysDiskSizeGB := image.Size / 1024 / 1024 / 1024 - if desc.SysDisk.SizeGB < sysDiskSizeGB { - desc.SysDisk.SizeGB = sysDiskSizeGB - } - - if desc.SysDisk.SizeGB < image.GetMinOsDiskSizeGb() { - desc.SysDisk.SizeGB = image.GetMinOsDiskSizeGb() - } - - BlockDeviceMappingV2 := []map[string]interface{}{} - - diskIds := []string{} - - defer func() { - for _, diskId := range diskIds { - err = host.zone.region.DeleteDisk(diskId) - if err != nil { - log.Errorf("clean disk %s error: %v", diskId, err) - } - } - }() - - if desc.SysDisk.StorageType != api.STORAGE_OPENSTACK_NOVA { //新建volume - istorage, err := host.zone.GetIStorageById(desc.SysDisk.StorageExternalId) - if err != nil { - return nil, errors.Wrapf(err, "GetIStorageById(%s)", desc.SysDisk.StorageExternalId) - } - - _sysDisk, err := host.zone.region.CreateDisk(desc.ExternalImageId, istorage.GetName(), "", desc.SysDisk.SizeGB, desc.SysDisk.Name, desc.ProjectId) - if err != nil { - return nil, errors.Wrapf(err, "CreateDisk %s", desc.SysDisk.Name) - } - - diskIds = append(diskIds, _sysDisk.GetGlobalId()) - - BlockDeviceMappingV2 = append(BlockDeviceMappingV2, map[string]interface{}{ - "boot_index": 0, - "uuid": _sysDisk.GetGlobalId(), - "source_type": "volume", - "destination_type": "volume", - "delete_on_termination": true, - }) - } else { - BlockDeviceMappingV2 = append(BlockDeviceMappingV2, map[string]interface{}{ - "boot_index": 0, - "uuid": image.ID, - "source_type": "image", - "destination_type": "local", - "delete_on_termination": true, - }) - } - - var _disk *SDisk - for index, disk := range desc.DataDisks { - istorage, err := host.zone.GetIStorageById(disk.StorageExternalId) - if err != nil { - return nil, errors.Wrapf(err, "GetIStorageById(%s)", disk.StorageExternalId) - } - _disk, err = host.zone.region.CreateDisk("", istorage.GetName(), "", disk.SizeGB, disk.Name, desc.ProjectId) - if err != nil { - return nil, errors.Wrapf(err, "CreateDisk %s", disk.Name) - } - diskIds = append(diskIds, _disk.ID) - - mapping := map[string]interface{}{ - "source_type": "volume", - "destination_type": "volume", - "delete_on_termination": true, - "boot_index": index + 1, - "uuid": _disk.ID, - } - - BlockDeviceMappingV2 = append(BlockDeviceMappingV2, mapping) - } - - params := map[string]map[string]interface{}{ - "server": { - "name": desc.Name, - "adminPass": desc.Password, - //"description": desc.Description, - "accessIPv4": desc.IpAddr, - "availability_zone": fmt.Sprintf("%s:%s", zoneName, host.GetName()), - "networks": []map[string]string{ - { - "uuid": network.NetworkID, - "fixed_ip": desc.IpAddr, - }, - }, - "security_groups": secgroups, - "user_data": desc.UserData, - "imageRef": desc.ExternalImageId, - "block_device_mapping_v2": BlockDeviceMappingV2, - }, - } - - flavorId, err := host.zone.region.syncFlavor(desc.InstanceType, desc.Cpu, desc.MemoryMB, desc.SysDisk.SizeGB) - if err != nil { - return nil, err - } - params["server"]["flavorRef"] = flavorId - - if len(desc.PublicKey) > 0 { - keypairName, err := host.zone.region.syncKeypair(desc.Name, desc.PublicKey) - if err != nil { - return nil, err - } - params["server"]["key_name"] = keypairName - } - - _, resp, err := host.zone.region.PostWithProject(desc.ProjectId, "compute", "/servers", "", jsonutils.Marshal(params)) - if err != nil { - return nil, err - } - diskIds = []string{} - serverId, err := resp.GetString("server", "id") - if err != nil { - return nil, err - } - instance, err := host.zone.region.GetInstance(serverId) - if err != nil { - return nil, err - } - instance.host = host - return instance, nil -} - -func (host *SHost) GetEnabled() bool { - return true -} - -func (host *SHost) GetAccessIp() string { - return host.HostIP -} - -func (host *SHost) GetAccessMac() string { - return "" -} - -func (host *SHost) GetSysInfo() jsonutils.JSONObject { - info := jsonutils.NewDict() - info.Add(jsonutils.NewString(CLOUD_PROVIDER_OPENSTACK), "manufacture") - return info -} - -func (host *SHost) GetSN() string { - return "" -} - -func (host *SHost) GetCpuCmtbound() float32 { - aggregates, err := host.zone.region.GetAggregates() - if err != nil || len(aggregates) == 0 { - return 16.0 - } - CpuCmtbound := 1000000.0 - for _, aggregate := range aggregates { - if utils.IsInStringArray(host.GetName(), aggregate.Hosts) { - if _cmtbound, ok := aggregate.Metadata["cpu_allocation_ratio"]; ok { - cmtbound, err := strconv.ParseFloat(_cmtbound, 32) - if err == nil && CpuCmtbound > cmtbound { - CpuCmtbound = cmtbound - } - } - } - } - if CpuCmtbound >= 1000000.0 { - return 16.0 - } - return float32(CpuCmtbound) -} - -func (host *SHost) GetMemCmtbound() float32 { - aggregates, err := host.zone.region.GetAggregates() - if err != nil || len(aggregates) == 0 { - return 1.5 - } - MemCmtbound := 1000000.0 - for _, aggregate := range aggregates { - if utils.IsInStringArray(host.GetName(), aggregate.Hosts) { - if _cmtbound, ok := aggregate.Metadata["ram_allocation_ratio"]; ok { - cmtbound, err := strconv.ParseFloat(_cmtbound, 32) - if err == nil && MemCmtbound > cmtbound { - MemCmtbound = cmtbound - } - } - } - } - if MemCmtbound >= 1000000.0 { - return 1.5 - } - return float32(MemCmtbound) -} - -func (host *SHost) GetCpuCount() int { - if host.Vcpus > 0 { - return host.Vcpus - } - host.Refresh() - return host.Vcpus -} - -func (host *SHost) GetNodeCount() int8 { - if len(host.CpuInfo) > 0 { - info, err := jsonutils.Parse([]byte(host.CpuInfo)) - if err == nil { - cpuInfo := &CpuInfo{} - err = info.Unmarshal(cpuInfo) - if err == nil { - if cell, ok := cpuInfo.Topology["cells"]; ok { - return int8(cell) - } - } - } - } - return int8(host.GetCpuCount()) -} - -func (host *SHost) GetCpuDesc() string { - return host.CpuInfo -} - -func (host *SHost) GetCpuMhz() int { - return 0 -} - -func (host *SHost) GetMemSizeMB() int { - if host.MemoryMB > 0 { - return host.MemoryMB - } - host.Refresh() - return host.MemoryMB -} - -func (host *SHost) GetStorageSizeMB() int { - if host.LocalGB > 0 { - return host.LocalGB * 1024 - } - host.Refresh() - return host.LocalGB * 1024 -} - -func (host *SHost) GetStorageType() string { - return api.DISK_TYPE_HYBRID -} - -func (host *SHost) GetHostType() string { - return api.HOST_TYPE_OPENSTACK -} - -func (host *SHost) GetHostStatus() string { - if host.Status == "disabled" { - return api.HOST_OFFLINE - } - switch host.State { - case "up", "": - return api.HOST_ONLINE - default: - return api.HOST_OFFLINE - } -} - -func (host *SHost) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { - return nil, cloudprovider.ErrNotSupported -} - -func (host *SHost) GetIsMaintenance() bool { - switch host.Status { - case "enabled", "": - return false - default: - return true - } -} - -func (host *SHost) GetVersion() string { - _, version, _ := host.zone.region.GetVersion("compute") - return version -} - -func (host *SHost) GetStatus() string { - return api.HOST_STATUS_RUNNING -} - -func (host *SHost) IsEmulated() bool { - return false -} - -func (host *SHost) Refresh() error { - new, err := host.zone.region.GetIHostById(host.GetId()) - if err != nil { - return err - } - if err := jsonutils.Update(host, new); err != nil { - return err - } - if len(host.Resource) > 0 { - for _, resouce := range host.Resource { - for _, info := range resouce { - if info.Project == "(total)" { - host.LocalGB = info.DiskGB - host.Vcpus = info.CPU - host.MemoryMB = info.MemoryMb - } - } - } - } - return nil + Service string } type SAggregate struct { @@ -509,9 +38,9 @@ type SAggregate struct { } func (region *SRegion) GetAggregates() ([]SAggregate, error) { - _, resp, err := region.List("compute", "/os-aggregates", "", nil) + resp, err := region.ecsList("/os-aggregates", nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsList") } aggregates := []SAggregate{} err = resp.Unmarshal(&aggregates, "aggregates") @@ -520,3 +49,16 @@ func (region *SRegion) GetAggregates() ([]SAggregate, error) { } return aggregates, nil } + +func (region *SRegion) GetHosts() ([]SHost, error) { + hosts := []SHost{} + resp, err := region.ecsList("/os-hosts", nil) + if err != nil { + return nil, errors.Wrap(err, "ecsList(os-hosts)") + } + err = resp.Unmarshal(&hosts, "hosts") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return hosts, nil +} diff --git a/pkg/multicloud/openstack/hypervisor.go b/pkg/multicloud/openstack/hypervisor.go new file mode 100644 index 0000000000..e856cd0dc4 --- /dev/null +++ b/pkg/multicloud/openstack/hypervisor.go @@ -0,0 +1,351 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + "fmt" + "net/url" + "strconv" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type CpuInfo struct { + Arch string + Model string + Vendor string + Feature []string + Topology map[string]int +} + +type Service struct { + Host string + ID string + DisabledReason string +} + +type SResource struct { + CPU int + DiskGB int + Host string + MemoryMb int + Project string +} + +type SHypervisor struct { + multicloud.SHostBase + zone *SZone + + CpuInfo string + + Aggregates []string + CurrentWorkload int + Status string + State string + DiskAvailableLeast int + HostIP string + FreeDiskGB int + FreeRamMB int + HypervisorHostname string + HypervisorType string + HypervisorVersion string + Id string + LocalGB int + LocalGbUsed int + MemoryMB int + MemoryMbUsed int + RunningVms int + Service Service + Vcpus int + VcpusUsed int8 +} + +func (host *SHypervisor) GetId() string { + return host.Id +} + +func (host *SHypervisor) GetName() string { + if len(host.HypervisorHostname) > 0 { + return host.HypervisorHostname + } + return host.Service.Host +} + +func (host *SHypervisor) GetGlobalId() string { + return host.GetId() +} + +func (host *SHypervisor) GetIWires() ([]cloudprovider.ICloudWire, error) { + return host.zone.GetIWires() +} + +func (host *SHypervisor) GetIStorages() ([]cloudprovider.ICloudStorage, error) { + istorages := []cloudprovider.ICloudStorage{} + storages, err := host.zone.region.GetStorageTypes() + if err != nil { + return nil, errors.Wrap(err, "GetStorageTypes") + } + for i := range storages { + storages[i].zone = host.zone + istorages = append(istorages, &storages[i]) + } + nova := &SNovaStorage{host: host, zone: host.zone} + istorages = append(istorages, nova) + return istorages, nil +} + +func (host *SHypervisor) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { + return host.zone.GetIStorageById(id) +} + +func (host *SHypervisor) GetIVMs() ([]cloudprovider.ICloudVM, error) { + instances, err := host.zone.region.GetInstances(host.HypervisorHostname) + if err != nil { + return nil, err + } + iVMs := []cloudprovider.ICloudVM{} + for i := 0; i < len(instances); i++ { + instances[i].host = host + iVMs = append(iVMs, &instances[i]) + } + return iVMs, nil +} + +func (host *SHypervisor) GetIVMById(gid string) (cloudprovider.ICloudVM, error) { + instance, err := host.zone.region.GetInstance(gid) + if err != nil { + return nil, err + } + instance.host = host + return instance, nil +} + +func (host *SHypervisor) CreateVM(desc *cloudprovider.SManagedVMCreateConfig) (cloudprovider.ICloudVM, error) { + instance, err := host.zone.CreateVM(host.Service.Host, desc) + if err != nil { + return nil, errors.Wrap(err, "CreateVM") + } + instance.host = host + return instance, nil +} + +func (host *SHypervisor) GetEnabled() bool { + return true +} + +func (host *SHypervisor) GetAccessIp() string { + return host.HostIP +} + +func (host *SHypervisor) GetAccessMac() string { + return "" +} + +func (host *SHypervisor) GetSysInfo() jsonutils.JSONObject { + info := jsonutils.NewDict() + info.Add(jsonutils.NewString(CLOUD_PROVIDER_OPENSTACK), "manufacture") + return info +} + +func (host *SHypervisor) GetSN() string { + return "" +} + +func (host *SHypervisor) GetCpuCmtbound() float32 { + aggregates, err := host.zone.region.GetAggregates() + if err != nil || len(aggregates) == 0 { + return 16.0 + } + CpuCmtbound := 1000000.0 + for _, aggregate := range aggregates { + if utils.IsInStringArray(host.GetName(), aggregate.Hosts) { + if _cmtbound, ok := aggregate.Metadata["cpu_allocation_ratio"]; ok { + cmtbound, err := strconv.ParseFloat(_cmtbound, 32) + if err == nil && CpuCmtbound > cmtbound { + CpuCmtbound = cmtbound + } + } + } + } + if CpuCmtbound >= 1000000.0 { + return 16.0 + } + return float32(CpuCmtbound) +} + +func (host *SHypervisor) GetMemCmtbound() float32 { + aggregates, err := host.zone.region.GetAggregates() + if err != nil || len(aggregates) == 0 { + return 1.5 + } + MemCmtbound := 1000000.0 + for _, aggregate := range aggregates { + if utils.IsInStringArray(host.GetName(), aggregate.Hosts) { + if _cmtbound, ok := aggregate.Metadata["ram_allocation_ratio"]; ok { + cmtbound, err := strconv.ParseFloat(_cmtbound, 32) + if err == nil && MemCmtbound > cmtbound { + MemCmtbound = cmtbound + } + } + } + } + if MemCmtbound >= 1000000.0 { + return 1.5 + } + return float32(MemCmtbound) +} + +func (host *SHypervisor) GetCpuCount() int { + if host.Vcpus > 0 { + return host.Vcpus + } + host.Refresh() + return host.Vcpus +} + +func (host *SHypervisor) GetNodeCount() int8 { + if len(host.CpuInfo) > 0 { + info, err := jsonutils.Parse([]byte(host.CpuInfo)) + if err == nil { + cpuInfo := &CpuInfo{} + err = info.Unmarshal(cpuInfo) + if err == nil { + if cell, ok := cpuInfo.Topology["cells"]; ok { + return int8(cell) + } + } + } + } + return int8(host.GetCpuCount()) +} + +func (host *SHypervisor) GetCpuDesc() string { + return host.CpuInfo +} + +func (host *SHypervisor) GetCpuMhz() int { + return 0 +} + +func (host *SHypervisor) GetMemSizeMB() int { + if host.MemoryMB > 0 { + return host.MemoryMB + } + host.Refresh() + return host.MemoryMB +} + +func (host *SHypervisor) GetStorageSizeMB() int { + return host.LocalGB * 1024 +} + +func (host *SHypervisor) GetStorageType() string { + return api.DISK_TYPE_HYBRID +} + +func (host *SHypervisor) GetHostType() string { + return api.HOST_TYPE_OPENSTACK +} + +func (host *SHypervisor) GetHostStatus() string { + if host.Status == "disabled" { + return api.HOST_OFFLINE + } + switch host.State { + case "up", "": + return api.HOST_ONLINE + default: + return api.HOST_OFFLINE + } +} + +func (host *SHypervisor) GetIHostNics() ([]cloudprovider.ICloudHostNetInterface, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (host *SHypervisor) GetIsMaintenance() bool { + switch host.Status { + case "enabled", "": + return false + default: + return true + } +} + +func (host *SHypervisor) GetVersion() string { + version, _ := host.zone.region.GetMaxVersion(OPENSTACK_SERVICE_COMPUTE) + return version +} + +func (host *SHypervisor) GetStatus() string { + return api.HOST_STATUS_RUNNING +} + +func (host *SHypervisor) IsEmulated() bool { + return false +} + +func (host *SHypervisor) Refresh() error { + return nil +} + +func (region *SRegion) GetHypervisors() ([]SHypervisor, error) { + hypervisors := []SHypervisor{} + resource := "/os-hypervisors/detail" + query := url.Values{} + for { + resp, err := region.ecsList(resource, query) + if err != nil { + return nil, errors.Wrap(err, "ecsList") + } + + part := struct { + Hypervisors []SHypervisor + HypervisorsLinks SNextLinks + }{} + + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + hypervisors = append(hypervisors, part.Hypervisors...) + marker := part.HypervisorsLinks.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) + } + return hypervisors, nil +} + +func (region *SRegion) GetHypervisor(id string) (*SHypervisor, error) { + resource := fmt.Sprintf("/os-hypervisors/%s", id) + resp, err := region.ecsGet(resource) + if err != nil { + return nil, errors.Wrap(err, "ecsGet") + } + hypervisor := &SHypervisor{} + err = resp.Unmarshal(hypervisor, "hypervisor") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return hypervisor, nil +} diff --git a/pkg/multicloud/openstack/image.go b/pkg/multicloud/openstack/image.go index 2e958d1e4c..6254520aad 100644 --- a/pkg/multicloud/openstack/image.go +++ b/pkg/multicloud/openstack/image.go @@ -16,6 +16,8 @@ package openstack import ( "context" + "fmt" + "io" "net/url" "strings" "time" @@ -24,7 +26,6 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/osprofile" - "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -58,7 +59,7 @@ type SImage struct { Self string MinDisk int Protected bool - ID string + Id string File string Checksum string OsHashAlgo string @@ -79,30 +80,36 @@ func (image *SImage) GetMinRamSizeMb() int { } func (region *SRegion) GetImages(name string, status string, imageId string) ([]SImage, error) { - params := url.Values{} - if utils.IsInStringArray(status, []string{QUEUED, SAVING, ACTIVE, KILLED, DELETED, PENDING_DELETE, DEACTIVATED, UPLOADING, IMPORTING}) { - params.Add("status", status) + query := url.Values{} + if len(status) > 0 { + query.Set("status", status) } if len(name) > 0 { - params.Add("name", name) + query.Set("name", name) } if len(imageId) > 0 { - params.Add("id", imageId) + query.Set("id", imageId) } - url := "/v2/images?" + params.Encode() images := []SImage{} - for len(url) > 0 { - _, resp, err := region.List("image", url, "", nil) + resource := "/v2/images" + for { + resp, err := region.imageList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "imageList") } - _images := []SImage{} - err = resp.Unmarshal(&_images, "images") + part := struct { + Images []SImage + Next string + }{} + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrapf(err, `resp.Unmarshal(&_images, "images")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - images = append(images, _images...) - url, _ = resp.GetString("next") + images = append(images, part.Images...) + if len(part.Next) == 0 { + break + } + resource = part.Next } return images, nil } @@ -112,7 +119,7 @@ func (image *SImage) GetMetadata() *jsonutils.JSONDict { } func (image *SImage) GetId() string { - return image.ID + return image.Id } func (image *SImage) GetName() string { @@ -124,11 +131,11 @@ func (image *SImage) IsEmulated() bool { } func (image *SImage) GetGlobalId() string { - return image.ID + return image.Id } func (image *SImage) Delete(ctx context.Context) error { - return image.storageCache.region.DeleteImage(image.ID) + return image.storageCache.region.DeleteImage(image.Id) } func (image *SImage) GetStatus() string { @@ -160,11 +167,11 @@ func (image *SImage) GetImageStatus() string { } func (image *SImage) Refresh() error { - new, err := image.storageCache.region.GetImage(image.ID) + _image, err := image.storageCache.region.GetImage(image.Id) if err != nil { - return err + return errors.Wrap(err, "GetImage") } - return jsonutils.Update(image, new) + return jsonutils.Update(image, _image) } func (image *SImage) GetImageType() string { @@ -243,7 +250,7 @@ func (image *SImage) GetIStoragecache() cloudprovider.ICloudStoragecache { } func (region *SRegion) DeleteImage(imageId string) error { - _, err := region.Delete("image", "/v2/images/"+imageId, "") + _, err := region.imageDelete("/v2/images/" + imageId) return err } @@ -266,7 +273,7 @@ func (region *SRegion) GetImageByName(name string) (*SImage, error) { return &images[0], nil } -func (region *SRegion) CreateImage(imageName string, osType string, osDist string, minDiskGb int, minRam int) (*SImage, error) { +func (region *SRegion) CreateImage(imageName string, osType string, osDist string, minDiskGb int, minRam int, body io.Reader) (*SImage, error) { params := map[string]interface{}{ "container_format": "bare", "disk_format": string(qemuimg.QCOW2), @@ -278,12 +285,21 @@ func (region *SRegion) CreateImage(imageName string, osType string, osDist strin "hw_qemu_guest_agent": "yes", } - _, resp, err := region.Post("image", "/v2/images", "", jsonutils.Marshal(params)) + resp, err := region.imagePost("/v2/images", params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "imagePost") } image := &SImage{} - return image, resp.Unmarshal(image) + err = resp.Unmarshal(image) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + url := fmt.Sprintf("/v2/images/%s/file", image.Id) + err = region.imageUpload(url, body) + if err != nil { + return nil, errors.Wrap(err, "imageUpload") + } + return image, nil } func (self *SImage) UEFI() bool { diff --git a/pkg/multicloud/openstack/instance.go b/pkg/multicloud/openstack/instance.go index 4fbd5929f3..8ebb663202 100644 --- a/pkg/multicloud/openstack/instance.go +++ b/pkg/multicloud/openstack/instance.go @@ -20,10 +20,11 @@ import ( "net/url" "time" + "gopkg.in/fatih/set.v0" + "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/utils" billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -62,7 +63,7 @@ type SPrivate struct { } type SecurityGroup struct { - ID string + Id string Name string Description string } @@ -73,17 +74,17 @@ type ExtraSpecs struct { } type Resource struct { - ID string + Id string Links []Link } type Image struct { - ID string + Id string Links []Link } type VolumesAttached struct { - ID string + Id string DeleteOnTermination bool } @@ -95,8 +96,7 @@ type SFault struct { type SInstance struct { multicloud.SInstanceBase - - host *SHost + host *SHypervisor DiskConfig string `json:"OS-DCF:diskConfig,omitempty"` AvailabilityZone string `json:"OS-EXT-AZ:availability_zone,omitempty"` @@ -104,10 +104,10 @@ type SInstance struct { Hostname string `json:"OS-EXT-SRV-ATTR:hostname,omitempty"` HypervisorHostname string `json:"OS-EXT-SRV-ATTR:hypervisor_hostname,omitempty"` InstanceName string `json:"OS-EXT-SRV-ATTR:instance_name,omitempty"` - KernelID string `json:"OS-EXT-SRV-ATTR:kernel_id,omitempty"` + KernelId string `json:"OS-EXT-SRV-ATTR:kernel_id,omitempty"` LaunchIndex int `json:"OS-EXT-SRV-ATTR:launch_index,omitempty"` - RamdiskID string `json:"OS-EXT-SRV-ATTR:ramdisk_id,omitempty"` - ReservationID string `json:"OS-EXT-SRV-ATTR:reservation_id,omitempty"` + RamdiskId string `json:"OS-EXT-SRV-ATTR:ramdisk_id,omitempty"` + ReservationId string `json:"OS-EXT-SRV-ATTR:reservation_id,omitempty"` RootDeviceName string `json:"OS-EXT-SRV-ATTR:root_device_name,omitempty"` UserData string `json:"OS-EXT-SRV-ATTR:user_data,omitempty"` PowerState int `json:"OS-EXT-STS:power_state,omitempty"` @@ -123,9 +123,9 @@ type SInstance struct { Created time.Time Description string Flavor SFlavor - HostID string + HostId string HostStatus string - ID string + Id string image Image //有可能是字符串 KeyName string Links []Link @@ -137,101 +137,91 @@ type SInstance struct { SecurityGroups []SecurityGroup Status string Tags []string - TenantID string + TenantId string TrustedImageCertificates []string Updated time.Time - UserID string + UserId string Fault SFault } func (region *SRegion) GetSecurityGroupsByInstance(instanceId string) ([]SecurityGroup, error) { - _, resp, err := region.Get("compute", fmt.Sprintf("/servers/%s/os-security-groups", instanceId), "", nil) + resource := fmt.Sprintf("/servers/%s/os-security-groups", instanceId) + resp, err := region.ecsGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsGet") } secgroups := []SecurityGroup{} - return secgroups, resp.Unmarshal(&secgroups, "security_groups") + err = resp.Unmarshal(&secgroups, "security_groups") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return secgroups, nil } -func (region *SRegion) GetInstances(hostName string) ([]SInstance, error) { - _, maxVersion, _ := region.GetVersion("compute") - url := "/servers/detail?all_tenants=True" +func (region *SRegion) GetInstances(host string) ([]SInstance, error) { instances := []SInstance{} - for len(url) > 0 { - _, resp, err := region.List("compute", url, maxVersion, nil) + resource := "/servers/detail" + query := url.Values{} + query.Set("all_tenants", "True") + for { + resp, err := region.ecsList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsList") } - _instances := []SInstance{} - err = resp.Unmarshal(&_instances, "servers") + part := struct { + Servers []SInstance + ServersLinks SNextLinks + }{} + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_instances, "servers")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - instances = append(instances, _instances...) - url = "" - if resp.Contains("servers_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "servers_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "servers")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } + for i := range part.Servers { + if len(host) == 0 || part.Servers[i].Host == host || part.Servers[i].HypervisorHostname == host { + instances = append(instances, part.Servers[i]) } } - } - result := []SInstance{} - for i := 0; i < len(instances); i++ { - if len(hostName) == 0 || hostName == instances[i].Host { - result = append(result, instances[i]) + marker := part.ServersLinks.GetNextMark() + if len(marker) == 0 { + break } + query.Set("marker", marker) } - return result, nil + return instances, nil } func (region *SRegion) GetInstance(instanceId string) (*SInstance, error) { - _, maxVersion, _ := region.GetVersion("compute") - _, resp, err := region.Get("compute", "/servers/"+instanceId, maxVersion, nil) + resource := "/servers/" + instanceId + resp, err := region.ecsGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsGet") } instance := &SInstance{} - return instance, resp.Unmarshal(instance, "server") + err = resp.Unmarshal(instance, "server") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarsha") + } + return instance, nil } func (instance *SInstance) GetSecurityGroupIds() ([]string, error) { secgroupIds := []string{} - secgroups, err := instance.host.zone.region.GetSecurityGroupsByInstance(instance.ID) + secgroups, err := instance.host.zone.region.GetSecurityGroupsByInstance(instance.Id) if err != nil { return nil, err } for _, secgroup := range secgroups { - secgroupIds = append(secgroupIds, secgroup.ID) + secgroupIds = append(secgroupIds, secgroup.Id) } return secgroupIds, nil } -func (instance *SInstance) GetMetadata() *jsonutils.JSONDict { - data := jsonutils.NewDict() - - instance.fetchFlavor() - - priceKey := fmt.Sprintf("%s::%s", instance.host.zone.ZoneName, instance.Flavor.OriginalName) - data.Add(jsonutils.NewString(priceKey), "price_key") - - data.Add(jsonutils.NewString(instance.host.zone.GetGlobalId()), "zone_ext_id") - return data -} - func (instance *SInstance) GetIHost() cloudprovider.ICloudHost { return instance.host } func (instance *SInstance) GetId() string { - return instance.ID + return instance.Id } func (instance *SInstance) GetName() string { @@ -239,7 +229,7 @@ func (instance *SInstance) GetName() string { } func (instance *SInstance) GetGlobalId() string { - return instance.ID + return instance.Id } func (instance *SInstance) IsEmulated() bool { @@ -247,8 +237,8 @@ func (instance *SInstance) IsEmulated() bool { } func (instance *SInstance) fetchFlavor() error { - if len(instance.Flavor.ID) > 0 && instance.Flavor.Vcpus == 0 { - flavor, err := instance.host.zone.region.GetFlavor(instance.Flavor.ID) + if len(instance.Flavor.Id) > 0 && instance.Flavor.Vcpus == 0 { + flavor, err := instance.host.zone.region.GetFlavor(instance.Flavor.Id) if err != nil { return err } @@ -258,7 +248,10 @@ func (instance *SInstance) fetchFlavor() error { } func (instance *SInstance) GetInstanceType() string { - instance.fetchFlavor() + err := instance.fetchFlavor() + if err != nil { + return "" + } return instance.Flavor.GetName() } @@ -266,54 +259,60 @@ func (instance *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) { disks := []SDisk{} hasSysDisk := false for i := 0; i < len(instance.VolumesAttached); i++ { - disk, err := instance.host.zone.region.GetDisk(instance.VolumesAttached[i].ID) + disk, err := instance.host.zone.region.GetDisk(instance.VolumesAttached[i].Id) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "GetDisk(%s)", instance.VolumesAttached[i].Id) } disks = append(disks, *disk) - if disk.Bootable { + if disk.GetDiskType() == api.DISK_TYPE_SYS { hasSysDisk = true } } - iDisks := []cloudprovider.ICloudDisk{} + idisks := []cloudprovider.ICloudDisk{} for i := 0; i < len(disks); i++ { - store, err := instance.host.zone.getStorageByCategory(disks[i].VolumeType) + store, err := instance.host.zone.getStorageByCategory(disks[i].VolumeType, disks[i].Host) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "getStorageByCategory(%s.%s)", disks[i].Id, disks[i].VolumeType) } disks[i].storage = store - iDisks = append(iDisks, &disks[i]) + idisks = append(idisks, &disks[i]) } if !hasSysDisk { - nova := SNovaStorage{zone: instance.host.zone} - sysDisk, _ := nova.GetIDiskById(instance.ID) - iDisks = append([]cloudprovider.ICloudDisk{sysDisk}, iDisks...) + store := &SNovaStorage{zone: instance.host.zone, host: instance.host} + disk := &SNovaDisk{storage: store, instanceId: instance.Id, region: instance.host.zone.region} + idisks = append([]cloudprovider.ICloudDisk{disk}, idisks...) } - return iDisks, nil + return idisks, nil } func (instance *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) { - nics := []cloudprovider.ICloudNic{} - for networkName, address := range instance.Addresses { - for i := 0; i < len(address); i++ { - if instance.Addresses[networkName][i].Type == "fixed" { - instance.Addresses[networkName][i].instance = instance - nics = append(nics, &instance.Addresses[networkName][i]) - } - } + nics, err := instance.host.zone.region.GetInstancePorts(instance.Id) + if err != nil { + return nil, errors.Wrap(err, "GetInstancePorts") } - return nics, nil + inics := []cloudprovider.ICloudNic{} + for i := range nics { + nics[i].region = instance.host.zone.region + inics = append(inics, &nics[i]) + } + return inics, nil } func (instance *SInstance) GetVcpuCount() int { - instance.fetchFlavor() + err := instance.fetchFlavor() + if err != nil { + return 0 + } return instance.Flavor.Vcpus } func (instance *SInstance) GetVmemSizeMB() int { - instance.fetchFlavor() + err := instance.fetchFlavor() + if err != nil { + return 0 + } return instance.Flavor.RAM } @@ -374,11 +373,11 @@ func (instance *SInstance) GetStatus() string { } func (instance *SInstance) Refresh() error { - new, err := instance.host.zone.region.GetInstance(instance.ID) + _instance, err := instance.host.zone.region.GetInstance(instance.Id) if err != nil { return err } - return jsonutils.Update(instance, new) + return jsonutils.Update(instance, _instance) } func (instance *SInstance) UpdateVM(ctx context.Context, name string) error { @@ -388,7 +387,8 @@ func (instance *SInstance) UpdateVM(ctx context.Context, name string) error { "name": name, }, } - _, _, err := instance.host.zone.region.Update("compute", "/servers/"+instance.ID, "", jsonutils.Marshal(params)) + resource := "/servers/" + instance.Id + _, err := instance.host.zone.region.ecsUpdate(resource, params) return err } return nil @@ -399,59 +399,61 @@ func (instance *SInstance) GetHypervisor() string { } func (instance *SInstance) StartVM(ctx context.Context) error { - if err := instance.host.zone.region.StartVM(instance.ID); err != nil { - return err + err := instance.host.zone.region.StartVM(instance.Id) + if err != nil { + return errors.Wrapf(err, "StartVM(%s)", instance.Id) } return cloudprovider.WaitStatus(instance, api.VM_RUNNING, 10*time.Second, 8*time.Minute) } func (instance *SInstance) StopVM(ctx context.Context, isForce bool) error { - if err := instance.host.zone.region.StopVM(instance.ID, isForce); err != nil { - return err + err := instance.host.zone.region.StopVM(instance.Id, isForce) + if err != nil { + return errors.Wrapf(err, "StopVM(%s)", instance.Id) } return cloudprovider.WaitStatus(instance, api.VM_READY, 10*time.Second, 8*time.Minute) } func (region *SRegion) GetInstanceVNCUrl(instanceId string) (string, error) { - _, maxVersion, _ := region.GetVersion("compute") params := map[string]map[string]string{ "remote_console": { "protocol": "vnc", "type": "novnc", }, } - _, resp, err := region.Post("compute", fmt.Sprintf("/servers/%s/remote-consoles", instanceId), maxVersion, jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/remote-consoles", instanceId) + resp, err := region.ecsPost(resource, params) if err != nil { - return "", err + return "", errors.Wrap(err, "ecsPost") } return resp.GetString("remote_console", "url") } func (instance *SInstance) GetVNCInfo() (jsonutils.JSONObject, error) { - url, err := instance.host.zone.region.GetInstanceVNCUrl(instance.ID) + url, err := instance.host.zone.region.GetInstanceVNCUrl(instance.Id) if err != nil { - return nil, err + return nil, errors.Wrap(err, "GetInstanceVNCUrl") } ret := jsonutils.NewDict() ret.Add(jsonutils.NewString(url), "url") ret.Add(jsonutils.NewString("openstack"), "protocol") - ret.Add(jsonutils.NewString(instance.ID), "instance_id") + ret.Add(jsonutils.NewString(instance.Id), "instance_id") return ret, nil } func (instance *SInstance) DeployVM(ctx context.Context, name string, username string, password string, publicKey string, deleteKeypair bool, description string) error { - return instance.host.zone.region.DeployVM(instance.ID, name, password, publicKey, deleteKeypair, description) + return instance.host.zone.region.DeployVM(instance.Id, name, password, publicKey, deleteKeypair, description) } func (instance *SInstance) RebuildRoot(ctx context.Context, desc *cloudprovider.SManagedVMRebuildRootConfig) (string, error) { - return instance.ID, instance.host.zone.region.ReplaceSystemDisk(instance.ID, desc.ImageId, desc.Password, desc.PublicKey, desc.SysSizeGB) + return instance.Id, instance.host.zone.region.ReplaceSystemDisk(instance.Id, desc.ImageId, desc.Password, desc.PublicKey, desc.SysSizeGB) } func (instance *SInstance) ChangeConfig(ctx context.Context, config *cloudprovider.SManagedVMChangeConfig) error { if (len(config.InstanceType) > 0 && instance.GetInstanceType() != config.InstanceType) || instance.GetVcpuCount() != config.Cpu || instance.GetVmemSizeMB() != config.MemoryMB { flavorId, err := instance.host.zone.region.syncFlavor(config.InstanceType, config.Cpu, config.MemoryMB, 40) if err != nil { - return err + return errors.Wrapf(err, "syncFlavor(%s)", config.InstanceType) } return instance.host.zone.region.ChangeConfig(instance, flavorId) } @@ -464,35 +466,30 @@ func (region *SRegion) ChangeConfig(instance *SInstance, flavorId string) error "flavorRef": flavorId, }, } - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instance.ID), maxVersion, jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/action", instance.Id) + _, err := region.ecsPost(resource, params) if err != nil { - return err + return errors.Wrap(err, "ecsPost") } - if err := cloudprovider.WaitStatus(instance, api.VM_SYNC_CONFIG, time.Second*3, time.Minute*4); err != nil { - return err + err = cloudprovider.WaitStatus(instance, api.VM_SYNC_CONFIG, time.Second*3, time.Minute*4) + if err != nil { + return errors.Wrap(err, "WaitStatsAfterChangeConfig") } - return region.instanceOperation(instance.ID, "confirmResize") + return region.instanceOperation(instance.Id, "confirmResize") } func (instance *SInstance) AttachDisk(ctx context.Context, diskId string) error { - return instance.host.zone.region.AttachDisk(instance.ID, diskId) + return instance.host.zone.region.AttachDisk(instance.Id, diskId) } func (instance *SInstance) DetachDisk(ctx context.Context, diskId string) error { - return instance.host.zone.region.DetachDisk(instance.ID, diskId) -} - -func (region *SRegion) CreateInstance(name string, imageId string, instanceType string, securityGroupId string, - zoneId string, desc string, passwd string, disks []SDisk, networkId string, ipAddr string, - keypair string, userData string, bc *billing.SBillingCycle) (string, error) { - return "", cloudprovider.ErrNotImplemented + return instance.host.zone.region.DetachDisk(instance.Id, diskId) } func (region *SRegion) instanceOperation(instanceId, operate string) error { - params := jsonutils.Marshal(map[string]string{operate: ""}) - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, params) + params := map[string]string{operate: ""} + resource := fmt.Sprintf("/servers/%s/action", instanceId) + _, err := region.ecsPost(resource, params) return err } @@ -515,11 +512,10 @@ func (region *SRegion) StopVM(instanceId string, isForce bool) error { func (region *SRegion) DeleteVM(instanceId string) error { instance, err := region.GetInstance(instanceId) if err != nil { - if err == cloudprovider.ErrNotFound { + if errors.Cause(err) == cloudprovider.ErrNotFound { return nil } - log.Errorf("failed to get instance %s %v", instanceId, err) - return err + return errors.Wrapf(err, "GetInstance(%s)", instanceId) } status := instance.GetStatus() log.Debugf("Instance status on delete is %s", status) @@ -536,15 +532,15 @@ func (region *SRegion) DeployVM(instanceId string, name string, password string, "adminPass": password, }, } - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/action", instanceId) + _, err := region.ecsPost(resource, params) return err } return nil } func (instance *SInstance) DeleteVM(ctx context.Context) error { - return instance.host.zone.region.DeleteVM(instance.ID) + return instance.host.zone.region.DeleteVM(instance.Id) } func (region *SRegion) ReplaceSystemDisk(instanceId string, imageId string, passwd string, publicKey string, sysDiskSizeGB int) error { @@ -565,31 +561,23 @@ func (region *SRegion) ReplaceSystemDisk(instanceId string, imageId string, pass if len(passwd) > 0 { params["rebuild"]["adminPass"] = passwd } - - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/action", instanceId) + _, err := region.ecsPost(resource, params) return err } -func (region *SRegion) ChangeVMConfig(zoneId string, instanceId string, ncpu int, vmem int, disks []*SDisk) error { - return cloudprovider.ErrNotImplemented -} - -func (region *SRegion) ChangeVMConfig2(zoneId string, instanceId string, instanceType string, disks []*SDisk) error { - return cloudprovider.ErrNotImplemented -} - func (region *SRegion) DetachDisk(instanceId string, diskId string) error { - _, err := region.Delete("compute", fmt.Sprintf("/servers/%s/os-volume_attachments/%s", instanceId, diskId), "") + resource := fmt.Sprintf("/servers/%s/os-volume_attachments/%s", instanceId, diskId) + _, err := region.ecsDelete(resource) if err != nil { - return err + return errors.Wrap(err, "ecsDelete") } status := "" startTime := time.Now() for time.Now().Sub(startTime) < time.Minute*10 { disk, err := region.GetDisk(diskId) if err != nil { - return err + return errors.Wrapf(err, "GetDisk(%s)", diskId) } status = disk.Status log.Debugf("status %s expect %s", status, DISK_STATUS_AVAILABLE) @@ -607,16 +595,17 @@ func (region *SRegion) AttachDisk(instanceId string, diskId string) error { "volumeId": diskId, }, } - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/os-volume_attachments", instanceId), "", jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/os-volume_attachments", instanceId) + _, err := region.ecsPost(resource, params) if err != nil { - return err + return errors.Wrap(err, "ecsPost") } status := "" startTime := time.Now() for time.Now().Sub(startTime) < time.Minute*10 { disk, err := region.GetDisk(diskId) if err != nil { - return err + return errors.Wrapf(err, "GetDisk(%s)", diskId) } status = disk.Status log.Debugf("status %s expect %s", status, DISK_STATUS_IN_USE) @@ -636,8 +625,8 @@ func (region *SRegion) MigrateVM(instanceId string, hostName string) error { migrate.Add(jsonutils.NewString(hostName), "host") } params.Add(migrate, "migrate") - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/action", instanceId) + _, err := region.ecsPost(resource, params) if err != nil { return errors.Wrapf(err, "On Requst Migrate instance:%s", instanceId) } @@ -653,8 +642,8 @@ func (region *SRegion) LiveMigrateVM(instanceId string, hostName string) error { osMigrateLive.Add(jsonutils.NewString(hostName), "host") } params.Add(osMigrateLive, "os-migrateLive") - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/action", instanceId), maxVersion, params) + resource := fmt.Sprintf("/servers/%s/action", instanceId) + _, err := region.ecsPost(resource, params) if err != nil { return errors.Wrapf(err, "On Requst LiveMigrate instance:%s", instanceId) } @@ -663,8 +652,8 @@ func (region *SRegion) LiveMigrateVM(instanceId string, hostName string) error { //仅live-migration func (region *SRegion) ListServerMigration(instanceId string) error { - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Get("compute", fmt.Sprintf("/servers/%s/migrations", instanceId), maxVersion, nil) + resource := fmt.Sprintf("/servers/%s/migrations", instanceId) + _, err := region.ecsGet(resource) if err != nil { return errors.Wrapf(err, "ListServerMigration") } @@ -673,8 +662,8 @@ func (region *SRegion) ListServerMigration(instanceId string) error { //仅live-migration func (region *SRegion) DeleteMigration(instanceId string, migrationId string) error { - _, maxVersion, _ := region.GetVersion("compute") - _, err := region.Delete("compute", fmt.Sprintf("/servers/%s/migrations/%s", instanceId, migrationId), maxVersion) + resource := fmt.Sprintf("/servers/%s/migrations/%s", instanceId, migrationId) + _, err := region.ecsDelete(resource) if err != nil { return errors.Wrapf(err, "On Requst delete LiveMigrate:%s", migrationId) } @@ -685,8 +674,8 @@ func (region *SRegion) DeleteMigration(instanceId string, migrationId string) er func (region *SRegion) ForceCompleteMigration(instanceId string, migrationId string) error { params := jsonutils.NewDict() params.Add(jsonutils.JSONNull, "force_complete") - _, maxVersion, _ := region.GetVersion("compute") - _, _, err := region.Post("compute", fmt.Sprintf("/servers/%s/migrations/%s/action", instanceId, migrationId), maxVersion, params) + resource := fmt.Sprintf("/servers/%s/migrations/%s/action", instanceId, migrationId) + _, err := region.ecsPost(resource, params) if err != nil { return errors.Wrapf(err, "On Requst delete LiveMigrate:%s", migrationId) } @@ -694,14 +683,11 @@ func (region *SRegion) ForceCompleteMigration(instanceId string, migrationId str } func (region *SRegion) GetMigrations(instanceId string, migrationType string) (jsonutils.JSONObject, error) { - params := jsonutils.NewDict() - params.Add(jsonutils.NewString(instanceId), "instance_uuid") - params.Add(jsonutils.NewString(migrationType), "migration_type") query := url.Values{} query.Set("instance_uuid", instanceId) query.Set("migration_type", migrationType) - _, maxVersion, _ := region.GetVersion("compute") - _, migrations, err := region.Get("compute", "/os-migrations?"+query.Encode(), maxVersion, nil) + resource := "/os-migrations" + migrations, err := region.ecsList(resource, query) if err != nil { return nil, errors.Wrapf(err, "On Get instance :%s Migration,migration_type:%s", instanceId, migrationType) } @@ -714,14 +700,15 @@ func (instance *SInstance) AssignSecurityGroup(secgroupId string) error { } secgroup, err := instance.host.zone.region.GetSecurityGroup(secgroupId) if err != nil { - return err + return errors.Wrapf(err, "GetSecurityGroup(%s)", secgroupId) } params := map[string]map[string]string{ "addSecurityGroup": { "name": secgroup.Name, }, } - _, _, err = instance.host.zone.region.Post("compute", fmt.Sprintf("/servers/%s/action", instance.ID), "", jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/action", instance.Id) + _, err = instance.host.zone.region.ecsDo(instance.GetProjectId(), resource, params) return err } @@ -732,36 +719,43 @@ func (instance *SInstance) RevokeSecurityGroup(secgroupId string) error { } secgroup, err := instance.host.zone.region.GetSecurityGroup(secgroupId) if err != nil { - return err + return errors.Wrapf(err, "GetSecurityGroup(%s)", secgroupId) } params := map[string]map[string]string{ "removeSecurityGroup": { "name": secgroup.Name, }, } - _, _, err = instance.host.zone.region.Post("compute", fmt.Sprintf("/servers/%s/action", instance.ID), "", jsonutils.Marshal(params)) + resource := fmt.Sprintf("/servers/%s/action", instance.Id) + _, err = instance.host.zone.region.ecsDo(instance.GetProjectId(), resource, params) return err } func (instance *SInstance) SetSecurityGroups(secgroupIds []string) error { - secgroups, err := instance.host.zone.region.GetSecurityGroupsByInstance(instance.ID) + secgroups, err := instance.host.zone.region.GetSecurityGroupsByInstance(instance.Id) if err != nil { - return err + return errors.Wrapf(err, "GetSecurityGroupsByInstance(%s)", instance.Id) } - originIds := []string{} + local := set.New(set.ThreadSafe) for _, secgroup := range secgroups { - if !utils.IsInStringArray(secgroup.ID, secgroupIds) { - if err := instance.RevokeSecurityGroup(secgroup.ID); err != nil { - return err - } - } - originIds = append(originIds, secgroup.ID) + local.Add(secgroup.Id) } + newG := set.New(set.ThreadSafe) for _, secgroupId := range secgroupIds { - if !utils.IsInStringArray(secgroupId, originIds) { - if err := instance.AssignSecurityGroup(secgroupId); err != nil { - return err - } + newG.Add(secgroupId) + } + for _, del := range set.Difference(local, newG).List() { + secgroupId := del.(string) + err := instance.RevokeSecurityGroup(secgroupId) + if err != nil { + return errors.Wrapf(err, "RevokeSecurityGroup(%s)", secgroupId) + } + } + for _, add := range set.Difference(newG, local).List() { + secgroupId := add.(string) + err := instance.AssignSecurityGroup(secgroupId) + if err != nil { + return errors.Wrapf(err, "AssignSecurityGroup(%s)", secgroupId) } } return nil @@ -807,12 +801,12 @@ func (region *SRegion) RenewInstances(instanceId []string, bc billing.SBillingCy } func (instance *SInstance) GetProjectId() string { - return instance.TenantID + return instance.TenantId } func (self *SInstance) GetError() error { if self.Status == INSTANCE_STATUS_ERROR && len(self.Fault.Message) > 0 { - return fmt.Errorf(self.Fault.Message) + return errors.Error(self.Fault.Message) } return nil } @@ -828,10 +822,12 @@ func (instance *SInstance) MigrateVM(hostId string) error { } previousHostName := instance.Host - if err := instance.host.zone.region.MigrateVM(instance.ID, hostName); err != nil { + err := instance.host.zone.region.MigrateVM(instance.Id, hostName) + if err != nil { return errors.Wrap(err, "MigrateVm") } - if err := cloudprovider.WaitMultiStatus(instance, []string{api.VM_SYNC_CONFIG, api.VM_READY, api.VM_UNKNOWN}, time.Second*10, time.Hour*3); err != nil { + err = cloudprovider.WaitMultiStatus(instance, []string{api.VM_SYNC_CONFIG, api.VM_READY, api.VM_UNKNOWN}, time.Second*10, time.Hour*3) + if err != nil { return errors.Wrap(err, "WaitMultiStatus") } if instance.GetStatus() == api.VM_UNKNOWN { @@ -843,7 +839,7 @@ func (instance *SInstance) MigrateVM(hostId string) error { } return nil } - return instance.host.zone.region.instanceOperation(instance.ID, "confirmResize") + return instance.host.zone.region.instanceOperation(instance.Id, "confirmResize") } func (instance *SInstance) LiveMigrateVM(hostId string) error { @@ -856,10 +852,12 @@ func (instance *SInstance) LiveMigrateVM(hostId string) error { hostName = iHost.GetName() } previousHostName := instance.Host - if err := instance.host.zone.region.LiveMigrateVM(instance.ID, hostName); err != nil { + err := instance.host.zone.region.LiveMigrateVM(instance.Id, hostName) + if err != nil { return errors.Wrap(err, "LiveMIgrateVm") } - if err := cloudprovider.WaitMultiStatus(instance, []string{api.VM_SYNC_CONFIG, api.VM_RUNNING, api.VM_UNKNOWN}, time.Second*10, time.Hour*3); err != nil { + err = cloudprovider.WaitMultiStatus(instance, []string{api.VM_SYNC_CONFIG, api.VM_RUNNING, api.VM_UNKNOWN}, time.Second*10, time.Hour*3) + if err != nil { return errors.Wrap(err, "WaitMultiStatus") } if instance.GetStatus() == api.VM_UNKNOWN { @@ -871,18 +869,178 @@ func (instance *SInstance) LiveMigrateVM(hostId string) error { } return nil } - return instance.host.zone.region.instanceOperation(instance.ID, "confirmResize") + return instance.host.zone.region.instanceOperation(instance.Id, "confirmResize") } func (instance *SInstance) GetIHostId() string { - iHosts, err := instance.host.zone.region.GetIHosts() + err := instance.host.zone.fetchHosts() if err != nil { - log.Errorf("instance.host.zone.region.GetIHosts error:%s", err) return "" } - for _, iHost := range iHosts { - if iHost.GetName() == instance.Host { - return iHost.GetId() + for _, host := range instance.host.zone.hosts { + if instance.HypervisorHostname == host.HypervisorHostname { + return host.GetGlobalId() } } return "" } + +func (region *SRegion) GetInstanceMetadata(instanceId string) (map[string]string, error) { + resource := fmt.Sprintf("/servers/%s/metadata", instanceId) + resp, err := region.ecsList(resource, nil) + if err != nil { + return nil, errors.Wrap(err, "ecsList") + } + result := struct { + Metadata map[string]string + }{} + err = resp.Unmarshal(&result) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return result.Metadata, nil +} + +func (zone *SZone) CreateVM(hypervisor string, opts *cloudprovider.SManagedVMCreateConfig) (*SInstance, error) { + region := zone.region + network, err := region.GetNetwork(opts.ExternalNetworkId) + if err != nil { + return nil, err + } + + secgroups := []map[string]string{} + for _, secgroupId := range opts.ExternalSecgroupIds { + if secgroupId != SECGROUP_NOT_SUPPORT { + secgroups = append(secgroups, map[string]string{"name": secgroupId}) + } + } + + image, err := region.GetImage(opts.ExternalImageId) + if err != nil { + return nil, errors.Wrapf(err, "GetImage(%s)", opts.ExternalImageId) + } + + sysDiskSizeGB := image.Size / 1024 / 1024 / 1024 + if opts.SysDisk.SizeGB < sysDiskSizeGB { + opts.SysDisk.SizeGB = sysDiskSizeGB + } + + if opts.SysDisk.SizeGB < image.GetMinOsDiskSizeGb() { + opts.SysDisk.SizeGB = image.GetMinOsDiskSizeGb() + } + + BlockDeviceMappingV2 := []map[string]interface{}{} + + diskIds := []string{} + + defer func() { + for _, diskId := range diskIds { + err = region.DeleteDisk(diskId) + if err != nil { + log.Errorf("clean disk %s error: %v", diskId, err) + } + } + }() + + if opts.SysDisk.StorageType != api.STORAGE_OPENSTACK_NOVA { //新建volume + istorage, err := zone.GetIStorageById(opts.SysDisk.StorageExternalId) + if err != nil { + return nil, errors.Wrapf(err, "GetIStorageById(%s)", opts.SysDisk.StorageExternalId) + } + + _sysDisk, err := region.CreateDisk(opts.ExternalImageId, istorage.GetName(), "", opts.SysDisk.SizeGB, opts.SysDisk.Name, opts.ProjectId) + if err != nil { + return nil, errors.Wrapf(err, "CreateDisk %s", opts.SysDisk.Name) + } + + diskIds = append(diskIds, _sysDisk.GetGlobalId()) + + BlockDeviceMappingV2 = append(BlockDeviceMappingV2, map[string]interface{}{ + "boot_index": 0, + "uuid": _sysDisk.GetGlobalId(), + "source_type": "volume", + "destination_type": "volume", + "delete_on_termination": true, + }) + } else { + BlockDeviceMappingV2 = append(BlockDeviceMappingV2, map[string]interface{}{ + "boot_index": 0, + "uuid": image.Id, + "source_type": "image", + "destination_type": "local", + "delete_on_termination": true, + }) + } + + var _disk *SDisk + for index, disk := range opts.DataDisks { + istorage, err := zone.GetIStorageById(disk.StorageExternalId) + if err != nil { + return nil, errors.Wrapf(err, "GetIStorageById(%s)", disk.StorageExternalId) + } + _disk, err = region.CreateDisk("", istorage.GetName(), "", disk.SizeGB, disk.Name, opts.ProjectId) + if err != nil { + return nil, errors.Wrapf(err, "CreateDisk %s", disk.Name) + } + diskIds = append(diskIds, _disk.Id) + + mapping := map[string]interface{}{ + "source_type": "volume", + "destination_type": "volume", + "delete_on_termination": true, + "boot_index": index + 1, + "uuid": _disk.Id, + } + + BlockDeviceMappingV2 = append(BlockDeviceMappingV2, mapping) + } + + az := zone.ZoneName + if len(hypervisor) > 0 { + az = fmt.Sprintf("%s:%s", zone.ZoneName, hypervisor) + } + + params := map[string]map[string]interface{}{ + "server": { + "name": opts.Name, + "adminPass": opts.Password, + "accessIPv4": opts.IpAddr, + "availability_zone": az, + "networks": []map[string]string{ + { + "uuid": network.NetworkId, + "fixed_ip": opts.IpAddr, + }, + }, + "security_groups": secgroups, + "user_data": opts.UserData, + "imageRef": opts.ExternalImageId, + "block_device_mapping_v2": BlockDeviceMappingV2, + }, + } + + flavorId, err := region.syncFlavor(opts.InstanceType, opts.Cpu, opts.MemoryMB, opts.SysDisk.SizeGB) + if err != nil { + return nil, err + } + params["server"]["flavorRef"] = flavorId + + if len(opts.PublicKey) > 0 { + keypairName, err := region.syncKeypair(opts.Name, opts.PublicKey) + if err != nil { + return nil, err + } + params["server"]["key_name"] = keypairName + } + + resp, err := region.ecsCreate(opts.ProjectId, "/servers", params) + if err != nil { + return nil, errors.Wrap(err, "ecsCreate") + } + diskIds = []string{} + instance := &SInstance{} + err = resp.Unmarshal(instance, "server") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return instance, nil +} diff --git a/pkg/multicloud/openstack/instancenic.go b/pkg/multicloud/openstack/instancenic.go index 7a18c851da..3c49a74c80 100644 --- a/pkg/multicloud/openstack/instancenic.go +++ b/pkg/multicloud/openstack/instancenic.go @@ -15,56 +15,80 @@ package openstack import ( + "fmt" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/regutils" + "yunion.io/x/onecloud/pkg/cloudprovider" ) type SInstanceNic struct { - instance *SInstance - MacAddr string `json:"OS-EXT-IPS-MAC:mac_addr"` - Version int `json:"version"` - Addr string `json:"addr"` - Type string `json:"OS-EXT-IPS:type"` + MacAddr string `json:"OS-EXT-IPS-MAC:mac_addr"` + Version int `json:"version"` + Addr string `json:"addr"` + Type string `json:"OS-EXT-IPS:type"` } -func (nic *SInstanceNic) GetIP() string { - return nic.Addr +type SFixedIp struct { + IpAddress string + SubnetId string } -func (nic *SInstanceNic) GetMAC() string { +type SInstancePort struct { + region *SRegion + FixedIps []SFixedIp + MacAddr string + NetId string + PortId string + PortState string +} + +func (region *SRegion) GetInstancePorts(instanceId string) ([]SInstancePort, error) { + resource := fmt.Sprintf("/servers/%s/os-interface", instanceId) + resp, err := region.ecsList(resource, nil) + if err != nil { + return nil, errors.Wrap(err, "ecsList") + } + ports := []SInstancePort{} + err = resp.Unmarshal(&ports, "interfaceAttachments") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return ports, nil +} + +func (nic *SInstancePort) GetIP() string { + for i := range nic.FixedIps { + if regutils.MatchIPAddr(nic.FixedIps[i].IpAddress) { + return nic.FixedIps[i].IpAddress + } + } + return "" +} + +func (nic *SInstancePort) GetMAC() string { return nic.MacAddr } -func (nic *SInstanceNic) GetDriver() string { +func (nic *SInstancePort) GetDriver() string { return "virtio" } -func (nic *SInstanceNic) InClassicNetwork() bool { +func (nic *SInstancePort) InClassicNetwork() bool { return false } -func (nic *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork { - ports, err := nic.instance.host.zone.region.GetPorts(nic.MacAddr) - if err == nil { - for i := 0; i < len(ports); i++ { - for j := 0; j < len(ports[i].FixedIps); j++ { - if ports[i].FixedIps[j].IpAddress == nic.Addr { - network, err := nic.instance.host.zone.region.GetNetwork(ports[i].FixedIps[j].SubnetID) - if err != nil { - return nil - } - wires, err := nic.instance.host.zone.GetIWires() - if err != nil { - return nil - } - for k := 0; k < len(wires); k++ { - wire := wires[k].(*SWire) - if net, _ := wire.GetINetworkById(network.ID); net != nil { - return net - } - } - return nil - } +func (nic *SInstancePort) GetINetwork() cloudprovider.ICloudNetwork { + for i := range nic.FixedIps { + if regutils.MatchIPAddr(nic.FixedIps[i].IpAddress) { + network, err := nic.region.GetNetwork(nic.FixedIps[i].SubnetId) + if err != nil { + log.Errorf("failed to found network by %s error: %v", nic.FixedIps[i].SubnetId, err) + return nil } + return network } } return nil diff --git a/pkg/multicloud/openstack/keypaire.go b/pkg/multicloud/openstack/keypaire.go index efaeb5ff29..12f75070db 100644 --- a/pkg/multicloud/openstack/keypaire.go +++ b/pkg/multicloud/openstack/keypaire.go @@ -16,12 +16,11 @@ package openstack import ( "fmt" + "net/url" - "yunion.io/x/jsonutils" - "yunion.io/x/pkg/utils" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/util/version" ) type SKeypair struct { @@ -36,48 +35,70 @@ type SKeyPair struct { } func (region *SRegion) GetKeypairs() ([]SKeyPair, error) { - _, resp, err := region.List("compute", "/os-keypairs", "", nil) - if err != nil { - return nil, err - } keypairs := []SKeyPair{} - return keypairs, resp.Unmarshal(&keypairs, "keypairs") + resource := "/os-keypairs" + query := url.Values{} + for { + resp, err := region.ecsList(resource, query) + if err != nil { + return nil, errors.Wrap(err, "ecsList") + } + part := struct { + Keypairs []SKeyPair + KeypairsLinks SNextLinks + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + keypairs = append(keypairs, part.Keypairs...) + marker := part.KeypairsLinks.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) + } + return keypairs, nil } func (region *SRegion) CreateKeypair(name, publicKey, Type string) (*SKeyPair, error) { - if len(Type) > 0 && !utils.IsInStringArray(Type, []string{"ssh", "x509"}) { - return nil, fmt.Errorf("only support ssh or x509 type") - } params := map[string]map[string]string{ "keypair": { "name": name, "public_key": publicKey, }, } - _, maxVersion, _ := region.GetVersion("compute") - if len(Type) > 0 && version.GE(maxVersion, "2.2") { + if len(Type) > 0 { params["keypair"]["type"] = Type } - _, resp, err := region.Post("compute", "/os-keypairs", maxVersion, jsonutils.Marshal(params)) + resp, err := region.ecsPost("/os-keypairs", params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsPost") } keypair := &SKeyPair{} - return keypair, resp.Unmarshal(keypair) + err = resp.Unmarshal(keypair) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return keypair, nil } func (region *SRegion) DeleteKeypair(name string) error { - _, err := region.Delete("compute", "/os-keypairs/"+name, "") + _, err := region.ecsDelete("/os-keypairs/" + name) return err } func (region *SRegion) GetKeypair(name string) (*SKeyPair, error) { - _, resp, err := region.Get("compute", "/os-keypairs/"+name, "", nil) + resp, err := region.ecsGet("/os-keypairs/" + name) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsGet") } keypair := &SKeyPair{} - return keypair, resp.Unmarshal(keypair) + err = resp.Unmarshal(keypair) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return keypair, nil } func (region *SRegion) syncKeypair(namePrefix, publicKey string) (string, error) { diff --git a/pkg/multicloud/openstack/shell/version.go b/pkg/multicloud/openstack/message.go similarity index 50% rename from pkg/multicloud/openstack/shell/version.go rename to pkg/multicloud/openstack/message.go index 33cc038708..fe4521a1c1 100644 --- a/pkg/multicloud/openstack/shell/version.go +++ b/pkg/multicloud/openstack/message.go @@ -12,25 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -package shell +package openstack -import ( - "fmt" +import "net/url" - "yunion.io/x/onecloud/pkg/multicloud/openstack" - "yunion.io/x/onecloud/pkg/util/shellutils" -) +type SMessage struct { + Id string + MessageLevel string + EventId string + ResourceType string + UserMessage string +} -func init() { - type VersionOptions struct { - SERVICE string `help:"Service name" choices:"compute|volume|volumev2|volumev3"` +func (region *SRegion) GetMessages(resourceId string) ([]SMessage, error) { + messages := []SMessage{} + resource := "messages" + query := url.Values{} + if len(resourceId) > 0 { + query.Set("resource_uuid", resourceId) } - shellutils.R(&VersionOptions{}, "version-show", "Show a service version", func(cli *openstack.SRegion, args *VersionOptions) error { - minVersion, maxVersion, err := cli.GetVersion(args.SERVICE) - if err != nil { - return err - } - fmt.Printf("min version: %s max version: %s\n", minVersion, maxVersion) - return nil - }) + resp, err := region.bsList(resource, query) + if err != nil { + return nil, err + } + err = resp.Unmarshal(&messages, "messages") + if err != nil { + return nil, err + } + return messages, nil } diff --git a/pkg/multicloud/openstack/network.go b/pkg/multicloud/openstack/network.go index 5236db145d..9209d71687 100644 --- a/pkg/multicloud/openstack/network.go +++ b/pkg/multicloud/openstack/network.go @@ -15,6 +15,8 @@ package openstack import ( + "fmt" + "net/url" "time" "github.com/pkg/errors" @@ -33,6 +35,25 @@ type AllocationPool struct { End string } +type SNextLinks []SNextLink + +func (links SNextLinks) GetNextMark() string { + for _, link := range links { + if link.Rel == "next" && len(link.Href) > 0 { + href, err := url.Parse(link.Href) + if err != nil { + log.Errorf("failed parse next link %s error: %v", link.Href, err) + continue + } + marker := href.Query().Get("marker") + if len(marker) > 0 { + return marker + } + } + } + return "" +} + type SNextLink struct { Href string Rel string @@ -43,24 +64,24 @@ type SNetwork struct { Name string EnableDhcp bool - NetworkID string - SegmentID string - ProjectID string - TenantID string + NetworkId string + SegmentId string + ProjectId string + TenantId string DnsNameservers []string AllocationPools []AllocationPool HostRoutes []string IpVersion int GatewayIP string CIDR string - ID string + Id string CreatedAt time.Time Description string Ipv6AddressMode string Ipv6RaMode string RevisionNumber int ServiceTypes []string - SubnetpoolID string + SubnetpoolId string Tags []string UpdatedAt time.Time } @@ -70,18 +91,18 @@ func (network *SNetwork) GetMetadata() *jsonutils.JSONDict { } func (network *SNetwork) GetId() string { - return network.ID + return network.Id } func (network *SNetwork) GetName() string { if len(network.Name) > 0 { return network.Name } - return network.ID + return network.Id } func (network *SNetwork) GetGlobalId() string { - return network.ID + return network.Id } func (network *SNetwork) IsEmulated() bool { @@ -93,11 +114,12 @@ func (network *SNetwork) GetStatus() string { } func (network *SNetwork) Delete() error { - return network.wire.zone.region.DeleteNetwork(network.ID) + return network.wire.zone.region.DeleteNetwork(network.Id) } func (region *SRegion) DeleteNetwork(networkId string) error { - _, err := region.Delete("network", "/v2.0/subnets/"+networkId, "") + resource := fmt.Sprintf("/v2.0/subnets/%s", networkId) + _, err := region.vpcDelete(resource) return err } @@ -147,11 +169,14 @@ func (network *SNetwork) GetIpMask() int8 { } func (network *SNetwork) GetIsPublic() bool { - return false + return network.wire.vpc.Shared } func (network *SNetwork) GetPublicScope() rbacutils.TRbacScope { - return rbacutils.ScopeProject + if network.wire.vpc.Shared { + return rbacutils.ScopeSystem + } + return rbacutils.ScopeNone } func (network *SNetwork) GetServerType() string { @@ -159,63 +184,61 @@ func (network *SNetwork) GetServerType() string { } func (region *SRegion) GetNetwork(networkId string) (*SNetwork, error) { - _, resp, err := region.Get("network", "/v2.0/subnets/"+networkId, "", nil) + resource := fmt.Sprintf("/v2.0/subnets/%s", networkId) + resp, err := region.vpcGet(resource) if err != nil { return nil, err } - network := SNetwork{} - return &network, resp.Unmarshal(&network, "subnet") + network := &SNetwork{} + err = resp.Unmarshal(network, "subnet") + if err != nil { + return nil, err + } + return network, nil } func (region *SRegion) GetNetworks(vpcId string) ([]SNetwork, error) { - url := "/v2.0/subnets" + resource := "/v2.0/subnets" networks := []SNetwork{} - for len(url) > 0 { - _, resp, err := region.List("network", url, "", nil) - if err != nil { - return nil, err - } - _networks := []SNetwork{} - err = resp.Unmarshal(&_networks, "subnets") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_networks, "subnets")`) - } - networks = append(networks, _networks...) - url = "" - if resp.Contains("subnets_links") { - nextLinks := []SNextLink{} - err = resp.Unmarshal(&nextLinks, "subnets_links") - if err != nil { - return nil, errors.Wrapf(err, "resp.Unmarshal(subnets_links)") - } - for _, next := range nextLinks { - if next.Rel == "next" { - url = next.Href - break - } - } - } + query := url.Values{} + if len(vpcId) > 0 { + query.Set("network_id", vpcId) } + for { + resp, err := region.vpcList(resource, query) + if err != nil { + return nil, errors.Wrap(err, "vpcList") + } - result := []SNetwork{} - for i := 0; i < len(networks); i++ { - if len(vpcId) == 0 || vpcId == networks[i].NetworkID { - result = append(result, networks[i]) + part := struct { + Subnets []SNetwork + SubnetsLinks SNextLinks + }{} + + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") } + + networks = append(networks, part.Subnets...) + marker := part.SubnetsLinks.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) } - return result, nil + return networks, nil } func (network *SNetwork) Refresh() error { - log.Debugf("network refresh %s", network.Name) - new, err := network.wire.zone.region.GetNetwork(network.ID) + _network, err := network.wire.zone.region.GetNetwork(network.Id) if err != nil { return err } - return jsonutils.Update(network, new) + return jsonutils.Update(network, _network) } -func (region *SRegion) CreateNetwork(vpcId string, name string, cidr string, desc string) (string, error) { +func (region *SRegion) CreateNetwork(vpcId string, projectId, name string, cidr string, desc string) (*SNetwork, error) { params := map[string]map[string]interface{}{ "subnet": { "name": name, @@ -225,13 +248,21 @@ func (region *SRegion) CreateNetwork(vpcId string, name string, cidr string, des "ip_version": 4, }, } - _, resp, err := region.Post("network", "/v2.0/subnets", "", jsonutils.Marshal(params)) - if err != nil { - return "", err + if len(projectId) > 0 { + params["subnet"]["project_id"] = projectId } - return resp.GetString("subnet", "id") + resp, err := region.vpcPost("/v2.0/subnets", params) + if err != nil { + return nil, errors.Wrap(err, "vpcPost") + } + network := &SNetwork{} + err = resp.Unmarshal(network, "subnet") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return network, nil } func (network *SNetwork) GetProjectId() string { - return network.TenantID + return network.TenantId } diff --git a/pkg/multicloud/openstack/nova_disk.go b/pkg/multicloud/openstack/nova_disk.go new file mode 100644 index 0000000000..b9bbd6557b --- /dev/null +++ b/pkg/multicloud/openstack/nova_disk.go @@ -0,0 +1,174 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openstack + +import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + + billing_api "yunion.io/x/onecloud/pkg/apis/billing" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud" +) + +type SNovaDisk struct { + multicloud.SDisk + storage *SNovaStorage + region *SRegion + + instanceId string +} + +func (disk *SNovaDisk) GetMetadata() *jsonutils.JSONDict { + return nil +} + +func (disk *SNovaDisk) GetId() string { + return disk.instanceId +} + +func (disk *SNovaDisk) Delete(ctx context.Context) error { + return cloudprovider.ErrNotSupported +} + +func (disk *SNovaDisk) Resize(ctx context.Context, sizeMb int64) error { + return cloudprovider.ErrNotSupported +} + +func (disk *SNovaDisk) GetName() string { + return fmt.Sprintf("Sys disk for instance %s", disk.instanceId) +} + +func (disk *SNovaDisk) GetGlobalId() string { + return disk.instanceId +} + +func (disk *SNovaDisk) IsEmulated() bool { + return false +} + +func (disk *SNovaDisk) GetIStorage() (cloudprovider.ICloudStorage, error) { + return disk.storage, nil +} + +func (disk *SNovaDisk) GetStatus() string { + return api.DISK_READY +} + +func (disk *SNovaDisk) Refresh() error { + return nil +} + +func (disk *SNovaDisk) GetDiskFormat() string { + return "raw" +} + +func (disk *SNovaDisk) GetDiskSizeMB() int { + instance, err := disk.region.GetInstance(disk.instanceId) + if err != nil { + return 0 + } + if instance.Flavor.Disk != 0 { + return instance.Flavor.Disk * 1024 + } + if len(instance.Flavor.Id) > 0 { + flavor, err := disk.region.GetFlavor(instance.Flavor.Id) + if err != nil { + return 0 + } + return flavor.Disk * 1024 + } + return 0 +} + +func (disk *SNovaDisk) GetIsAutoDelete() bool { + return true +} + +func (disk *SNovaDisk) GetTemplateId() string { + return "" +} + +func (disk *SNovaDisk) GetDiskType() string { + return api.DISK_TYPE_SYS +} + +func (disk *SNovaDisk) GetFsFormat() string { + return "" +} + +func (disk *SNovaDisk) GetIsNonPersistent() bool { + return false +} + +func (disk *SNovaDisk) GetDriver() string { + return "scsi" +} + +func (disk *SNovaDisk) GetCacheMode() string { + return "none" +} + +func (disk *SNovaDisk) GetMountpoint() string { + return "" +} + +func (disk *SNovaDisk) CreateISnapshot(ctx context.Context, name, desc string) (cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (disk *SNovaDisk) GetISnapshot(snapshotId string) (cloudprovider.ICloudSnapshot, error) { + return nil, cloudprovider.ErrNotFound +} + +func (disk *SNovaDisk) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { + return []cloudprovider.ICloudSnapshot{}, nil +} + +func (disk *SNovaDisk) Reset(ctx context.Context, snapshotId string) (string, error) { + return "", cloudprovider.ErrNotSupported +} + +func (disk *SNovaDisk) GetBillingType() string { + return billing_api.BILLING_TYPE_POSTPAID +} + +func (disk *SNovaDisk) GetCreatedAt() time.Time { + return time.Time{} +} + +func (disk *SNovaDisk) GetExpiredAt() time.Time { + return time.Time{} +} + +func (disk *SNovaDisk) GetAccessPath() string { + return "" +} + +func (disk *SNovaDisk) Rebuild(ctx context.Context) error { + return cloudprovider.ErrNotSupported +} + +func (disk *SNovaDisk) GetProjectId() string { + instance, err := disk.region.GetInstance(disk.instanceId) + if err != nil { + return "" + } + return instance.GetProjectId() +} diff --git a/pkg/multicloud/openstack/novastorage.go b/pkg/multicloud/openstack/novastorage.go index 05105c3202..700ab610db 100644 --- a/pkg/multicloud/openstack/novastorage.go +++ b/pkg/multicloud/openstack/novastorage.go @@ -24,6 +24,7 @@ import ( ) type SNovaStorage struct { + host *SHypervisor zone *SZone } @@ -32,7 +33,7 @@ func (storage *SNovaStorage) GetMetadata() *jsonutils.JSONDict { } func (storage *SNovaStorage) GetId() string { - return fmt.Sprintf("%s-%s-%s", storage.zone.region.client.cpcfg.Id, storage.zone.GetGlobalId(), storage.GetName()) + return fmt.Sprintf("%s-%s-%s", storage.zone.GetGlobalId(), storage.host.GetId(), storage.GetName()) } func (storage *SNovaStorage) GetName() string { @@ -64,7 +65,7 @@ func (storage *SNovaStorage) GetMediumType() string { } func (storage *SNovaStorage) GetCapacityMB() int64 { - return 1000000000 + return int64(storage.host.GetStorageSizeMB()) } func (storage *SNovaStorage) GetStorageConf() jsonutils.JSONObject { @@ -94,21 +95,7 @@ func (storage *SNovaStorage) CreateIDisk(conf *cloudprovider.DiskCreateConfig) ( } func (storage *SNovaStorage) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, error) { - instance, err := storage.zone.region.GetInstance(idStr) - if err != nil { - return nil, err - } - disk := SDisk{ - ID: instance.ID, - Name: fmt.Sprintf("root disk for %s", instance.Name), - Size: instance.Flavor.Disk, - Status: DISK_STATUS_IN_USE, - Bootable: true, - CreatedAt: instance.Created, - VolumeType: api.STORAGE_OPENSTACK_NOVA, - } - disk.nova = storage - return &disk, nil + return &SNovaDisk{region: storage.zone.region, storage: storage, instanceId: idStr}, nil } func (storage *SNovaStorage) GetMountPoint() string { diff --git a/pkg/multicloud/openstack/openstack.go b/pkg/multicloud/openstack/openstack.go index 3871a93399..37a4cc03a6 100644 --- a/pkg/multicloud/openstack/openstack.go +++ b/pkg/multicloud/openstack/openstack.go @@ -19,11 +19,13 @@ import ( "fmt" "io" "net/http" + "net/url" "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -37,6 +39,14 @@ import ( const ( CLOUD_PROVIDER_OPENSTACK = api.CLOUD_PROVIDER_OPENSTACK OPENSTACK_DEFAULT_REGION = "RegionOne" + + OPENSTACK_SERVICE_COMPUTE = "compute" + OPENSTACK_SERVICE_NETWORK = "network" + OPENSTACK_SERVICE_IDENTITY = "identity" + OPENSTACK_SERVICE_VOLUMEV3 = "volumev3" + OPENSTACK_SERVICE_VOLUMEV2 = "volumev2" + OPENSTACK_SERVICE_VOLUME = "volume" + OPENSTACK_SERVICE_IMAGE = "image" ) type OpenstackClientConfig struct { @@ -88,18 +98,48 @@ func (cfg *OpenstackClientConfig) Debug(debug bool) *OpenstackClientConfig { type SOpenStackClient struct { *OpenstackClientConfig - client *mcclient.Client tokenCredential mcclient.TokenCredential iregions []cloudprovider.ICloudRegion + + defaultRegionName string + + projects []SProject } func NewOpenStackClient(cfg *OpenstackClientConfig) (*SOpenStackClient, error) { cli := &SOpenStackClient{ OpenstackClientConfig: cfg, } + err := cli.fetchToken() + if err != nil { + return nil, errors.Wrap(err, "fetchToken") + } return cli, cli.fetchRegions() } +func (cli *SOpenStackClient) getDefaultRegionName() string { + return cli.defaultRegionName +} + +func (cli *SOpenStackClient) getProjectToken(projectId, projectName string) (mcclient.TokenCredential, error) { + s := cli.getDefaultSession("") + roleId, err := modules.RolesV3.GetId(s, "admin", jsonutils.Marshal(map[string]string{})) + if err != nil { + return nil, errors.Wrap(err, "RolesV3.GetId") + } + _, err = modules.RolesV3.PutInContexts(s, roleId, nil, []modulebase.ManagerContext{{InstanceManager: &modules.Projects, InstanceId: projectId}, {InstanceManager: &modules.UsersV3, InstanceId: s.GetUserId()}}) + if err != nil { + return nil, errors.Wrap(err, "RolesV3.PutInContexts") + } + + client := cli.getDefaultClient() + tokenCredential, err := client.Authenticate(cli.username, cli.password, cli.domainName, projectName, cli.projectDomain) + if err != nil { + return nil, errors.Wrap(err, "Authenticate") + } + return tokenCredential, nil +} + func (cli *SOpenStackClient) GetCloudRegionExternalIdPrefix() string { return fmt.Sprintf("%s/%s/", CLOUD_PROVIDER_OPENSTACK, cli.cpcfg.Id) } @@ -116,138 +156,267 @@ func (cli *SOpenStackClient) GetSubAccounts() ([]cloudprovider.SSubAccount, erro } func (cli *SOpenStackClient) fetchRegions() error { - if err := cli.connect(); err != nil { - return err - } - regions := cli.tokenCredential.GetRegions() cli.iregions = make([]cloudprovider.ICloudRegion, len(regions)) for i := 0; i < len(regions); i++ { region := SRegion{client: cli, Name: regions[i]} cli.iregions[i] = ®ion + cli.defaultRegionName = regions[0] + } + return nil +} + +type OpenstackError struct { + httputils.JSONClientError +} + +func (ce *OpenstackError) ParseErrorFromJsonResponse(statusCode int, body jsonutils.JSONObject) error { + body.Unmarshal(ce) + if ce.Code == 0 { + ce.Code = statusCode + } + if len(ce.Details) == 0 { + ce.Details = body.String() + } + if len(ce.Class) == 0 { + ce.Class = http.StatusText(statusCode) + } + if statusCode == 404 { + return errors.Wrap(cloudprovider.ErrNotFound, ce.Error()) + } + return ce +} + +type sApiVersion struct { + MinVersion string + Version string +} + +type sApiVersions struct { + Versions []sApiVersion + Version sApiVersion +} + +func (v *sApiVersions) GetMaxVersion() string { + maxVersion := v.Version.Version + for _, _version := range v.Versions { + if version.GT(_version.Version, maxVersion) { + maxVersion = _version.Version + } + } + return maxVersion +} + +func getApiVerion(token mcclient.TokenCredential, url string, debug bool) (string, error) { + client := httputils.NewJsonClient(httputils.GetDefaultClient()) + req := httputils.NewJsonRequest(httputils.THttpMethod("GET"), strings.TrimSuffix(url, token.GetTenantId()), nil) + header := http.Header{} + header.Set("X-Auth-Token", token.GetTokenString()) + req.SetHeader(header) + oe := &OpenstackError{} + _, resp, err := client.Send(context.Background(), req, oe, debug) + if err != nil { + return "", errors.Wrap(err, "get api version") + } + versions := &sApiVersions{} + resp.Unmarshal(&versions) + return versions.GetMaxVersion(), nil +} + +func (cli *SOpenStackClient) GetMaxVersion(region, service string) (string, error) { + serviceUrl, err := cli.tokenCredential.GetServiceURL(service, region, "", cli.endpointType) + if err != nil { + return "", errors.Wrapf(err, "GetServiceURL(%s, %s, %s)", service, region, cli.endpointType) + } + header := http.Header{} + header.Set("X-Auth-Token", cli.tokenCredential.GetTokenString()) + return getApiVerion(cli.tokenCredential, serviceUrl, cli.debug) +} + +func jsonReuest(token mcclient.TokenCredential, service, region, endpointType string, method httputils.THttpMethod, resource string, query url.Values, body interface{}, debug bool) (jsonutils.JSONObject, error) { + serviceUrl, err := token.GetServiceURL(service, region, "", endpointType) + if err != nil { + return nil, errors.Wrapf(err, "GetServiceURL(%s, %s, %s)", service, region, endpointType) + } + header := http.Header{} + header.Set("X-Auth-Token", token.GetTokenString()) + apiVersion := "" + if !utils.IsInStringArray(service, []string{OPENSTACK_SERVICE_IMAGE, OPENSTACK_SERVICE_IDENTITY}) { + apiVersion, err = getApiVerion(token, serviceUrl, debug) + if err != nil { + log.Errorf("get service %s api version error: %v", service, err) + } + } + if len(apiVersion) > 0 { + switch service { + case OPENSTACK_SERVICE_COMPUTE: + header.Set("X-Openstack-Nova-API-Version", apiVersion) + case OPENSTACK_SERVICE_IMAGE: + header.Set("X-Openstack-Glance-API-Version", apiVersion) + case OPENSTACK_SERVICE_VOLUME, OPENSTACK_SERVICE_VOLUMEV2, OPENSTACK_SERVICE_VOLUMEV3: + header.Set("Openstack-API-Version", fmt.Sprintf("volume %s", apiVersion)) + case OPENSTACK_SERVICE_NETWORK: + header.Set("X-Openstack-Neutron-API-Version", apiVersion) + case OPENSTACK_SERVICE_IDENTITY: + header.Set("X-Openstack-Identity-API-Version", apiVersion) + } } - for _, region := range regions { - if serviceURL, err := cli.tokenCredential.GetServiceURL("compute", region, "", cli.endpointType); err != nil || len(serviceURL) == 0 { - for _, endpointType := range []string{"internal", "admin", "public"} { - if serviceURL, err := cli.tokenCredential.GetServiceURL("compute", region, "", endpointType); err == nil && len(serviceURL) > 0 { - cli.endpointType = endpointType - return nil - } - } - } else { + requestUrl := resource + if !strings.HasPrefix(resource, serviceUrl) { + requestUrl = fmt.Sprintf("%s/%s", serviceUrl, resource) + } + + if query != nil { + requestUrl = fmt.Sprintf("%s?%s", requestUrl, query.Encode()) + } + + return _jsonRequest(method, requestUrl, header, body, debug) +} + +func _jsonRequest(method httputils.THttpMethod, url string, header http.Header, params interface{}, debug bool) (jsonutils.JSONObject, error) { + client := httputils.NewJsonClient(httputils.GetDefaultClient()) + req := httputils.NewJsonRequest(method, url, params) + req.SetHeader(header) + oe := &OpenstackError{} + _, resp, err := client.Send(context.Background(), req, oe, debug) + return resp, err +} + +func (cli *SOpenStackClient) ecsRequest(region string, method httputils.THttpMethod, resource string, query url.Values, body interface{}) (jsonutils.JSONObject, error) { + token := cli.tokenCredential + if method == httputils.POST && query != nil && len(query.Get("project_id")) > 0 { + projectId := query.Get("project_id") + var err error + token, err = cli.getProjectTokenCredential(projectId) + if err != nil { + return nil, errors.Wrapf(err, "getProjectTokenCredential(%s)", projectId) + } + } + return jsonReuest(token, OPENSTACK_SERVICE_COMPUTE, region, cli.endpointType, method, resource, query, body, cli.debug) +} + +func (cli *SOpenStackClient) ecsCreate(projectId, region, resource string, body interface{}) (jsonutils.JSONObject, error) { + token := cli.tokenCredential + if len(projectId) > 0 { + var err error + token, err = cli.getProjectTokenCredential(projectId) + if err != nil { + return nil, errors.Wrapf(err, "getProjectTokenCredential(%s)", projectId) + } + } + return jsonReuest(token, OPENSTACK_SERVICE_COMPUTE, region, cli.endpointType, httputils.POST, resource, nil, body, cli.debug) +} + +func (cli *SOpenStackClient) ecsDo(projectId, region, resource string, body interface{}) (jsonutils.JSONObject, error) { + token := cli.tokenCredential + if len(projectId) > 0 { + var err error + token, err = cli.getProjectTokenCredential(projectId) + if err != nil { + return nil, errors.Wrapf(err, "getProjectTokenCredential(%s)", projectId) + } + } + return jsonReuest(token, OPENSTACK_SERVICE_COMPUTE, region, cli.endpointType, httputils.POST, resource, nil, body, cli.debug) +} + +func (cli *SOpenStackClient) iamRequest(region string, method httputils.THttpMethod, resource string, query url.Values, body interface{}) (jsonutils.JSONObject, error) { + return jsonReuest(cli.tokenCredential, OPENSTACK_SERVICE_IDENTITY, region, cli.endpointType, method, resource, query, body, cli.debug) +} + +func (cli *SOpenStackClient) vpcRequest(region string, method httputils.THttpMethod, resource string, query url.Values, body interface{}) (jsonutils.JSONObject, error) { + return jsonReuest(cli.tokenCredential, OPENSTACK_SERVICE_NETWORK, region, cli.endpointType, method, resource, query, body, cli.debug) +} + +func (cli *SOpenStackClient) imageRequest(region string, method httputils.THttpMethod, resource string, query url.Values, body interface{}) (jsonutils.JSONObject, error) { + return jsonReuest(cli.tokenCredential, OPENSTACK_SERVICE_IMAGE, region, cli.endpointType, method, resource, query, body, cli.debug) +} + +func (cli *SOpenStackClient) bsRequest(region string, method httputils.THttpMethod, resource string, query url.Values, body interface{}) (jsonutils.JSONObject, error) { + for _, service := range []string{OPENSTACK_SERVICE_VOLUMEV3, OPENSTACK_SERVICE_VOLUMEV2, OPENSTACK_SERVICE_VOLUME} { + _, err := cli.tokenCredential.GetServiceURL(service, region, "", cli.endpointType) + if err == nil { + return jsonReuest(cli.tokenCredential, service, region, cli.endpointType, method, resource, query, body, cli.debug) + } + } + return nil, fmt.Errorf("no valid volume service endpoint") +} + +func (cli *SOpenStackClient) bsCreate(projectId, region, resource string, body interface{}) (jsonutils.JSONObject, error) { + token := cli.tokenCredential + if len(projectId) > 0 { + var err error + token, err = cli.getProjectTokenCredential(projectId) + if err != nil { + return nil, errors.Wrapf(err, "getProjectTokenCredential(%s)", projectId) + } + } + for _, service := range []string{OPENSTACK_SERVICE_VOLUMEV3, OPENSTACK_SERVICE_VOLUMEV2, OPENSTACK_SERVICE_VOLUME} { + _, err := token.GetServiceURL(service, region, "", cli.endpointType) + if err == nil { + return jsonReuest(token, service, region, cli.endpointType, httputils.POST, resource, nil, body, cli.debug) + } + } + return nil, fmt.Errorf("no valid volume service endpoint") +} + +func (cli *SOpenStackClient) imageUpload(region, url string, body io.Reader) (*http.Response, error) { + header := http.Header{} + header.Set("Content-Type", "application/octet-stream") + session := cli.getDefaultSession(region) + return session.RawRequest(OPENSTACK_SERVICE_IMAGE, "", httputils.PUT, url, header, body) +} + +func (cli *SOpenStackClient) fetchToken() error { + if cli.tokenCredential != nil { + return nil + } + var err error + cli.tokenCredential, err = cli.getDefaultToken() + if err != nil { + return errors.Wrap(err, "getDefaultToken") + } + return cli.checkEndpointType() +} + +func (cli *SOpenStackClient) checkEndpointType() error { + for _, regionName := range cli.tokenCredential.GetRegions() { + _, err := cli.tokenCredential.GetServiceURL(OPENSTACK_SERVICE_COMPUTE, regionName, "", cli.endpointType) + if err == nil { return nil } - } - return fmt.Errorf("failed to find right endpoint type") -} - -func (cli *SOpenStackClient) Request(projectId string, region, service, method string, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - header := http.Header{} - if len(microversion) > 0 { - header.Set("X-Openstack-Nova-API-Version", microversion) - } - - var session *mcclient.ClientSession = nil - ctx := context.Background() - if method == "POST" && len(projectId) > 0 { - targetToken, err := cli.getProjectTokenCredential(projectId) - if err != nil { - log.Errorf("failed to get project %s token credential %v", projectId, err) - } else { - session = cli.client.NewSession(ctx, region, "", cli.endpointType, targetToken, "") - } - } - if session == nil { - session = cli.client.NewSession(ctx, region, "", cli.endpointType, cli.tokenCredential, "") - } - - serviceUrl, err := session.GetServiceURL(service, "") - if err != nil { - return nil, nil, errors.Wrapf(err, "GetServiceURL(%s)", service) - } - url = strings.TrimPrefix(url, serviceUrl) - header, resp, err := session.JSONRequest(service, "", httputils.THttpMethod(method), url, header, body) - if err != nil && body != nil { - log.Errorf("microversion %s url: %s, params: %s", microversion, serviceUrl+url, body.PrettyString()) - } - return header, resp, err -} - -func (cli *SOpenStackClient) RawRequest(region, service, method string, url string, microversion string, body jsonutils.JSONObject) (*http.Response, error) { - header := http.Header{} - if len(microversion) > 0 { - header.Set("X-Openstack-Nova-API-Version", microversion) - } - ctx := context.Background() - session := cli.client.NewSession(ctx, region, "", cli.endpointType, cli.tokenCredential, "") - data := strings.NewReader("") - if body != nil { - data = strings.NewReader(body.String()) - } - return session.RawRequest(service, "", httputils.THttpMethod(method), url, header, data) -} - -func (cli *SOpenStackClient) StreamRequest(region, service, method string, url string, microversion string, body io.Reader) (*http.Response, error) { - header := http.Header{} - if len(microversion) > 0 { - header.Set("X-Openstack-Nova-API-Version", microversion) - } - header.Set("Content-Type", "application/octet-stream") - ctx := context.Background() - session := cli.client.NewSession(ctx, region, "", cli.endpointType, cli.tokenCredential, "") - return session.RawRequest(service, "", httputils.THttpMethod(method), url, header, body) -} - -func (cli *SOpenStackClient) getVersion(region string, service string) (string, string, error) { - ctx := context.Background() - session := cli.client.NewSession(ctx, region, "", cli.endpointType, cli.tokenCredential, "") - uri, err := session.GetServiceURL(service, cli.endpointType) - if err != nil { - return "", "", err - } - url := uri - telnetID := cli.tokenCredential.GetTenantId() - if strings.Index(uri, telnetID) > 0 { - url = uri[0:strings.Index(uri, telnetID)] - } - _, resp, err := session.JSONRequest(url, "", "GET", "/", nil, nil) - if err != nil { - return "", "", err - } - minVersion, _ := resp.GetString("version", "min_version") - maxVersion, _ := resp.GetString("version", "version") - if resp.Contains("versions") { - minVersion, maxVersion = "1000.0", "" - versions, _ := resp.GetArray("versions") - for _, _version := range versions { - if _minVersion, _ := _version.GetString("min_version"); len(_minVersion) > 0 { - if version.LT(_minVersion, minVersion) { - minVersion = _minVersion - } - } - if _maxVersion, _ := _version.GetString("version"); len(_maxVersion) > 0 { - if version.GT(_maxVersion, maxVersion) { - maxVersion = _maxVersion - } + for _, endpointType := range []string{"internal", "admin", "public"} { + _, err = cli.tokenCredential.GetServiceURL(OPENSTACK_SERVICE_COMPUTE, regionName, "", endpointType) + if err == nil { + cli.endpointType = endpointType + return nil } } - if minVersion == "1000.0" { - minVersion, maxVersion = "", "" - } } - return minVersion, maxVersion, nil + return errors.Errorf("failed to find right endpoint type for compute service") } -func (cli *SOpenStackClient) connect() error { - cli.client = mcclient.NewClient(cli.authURL, 5, cli.debug, false, "", "") - cli.client.SetHttpTransportProxyFunc(cli.cpcfg.ProxyFunc) - tokenCredential, err := cli.client.Authenticate(cli.username, cli.password, cli.domainName, cli.project, cli.projectDomain) - if err != nil { - return err +func (cli *SOpenStackClient) getDefaultSession(regionName string) *mcclient.ClientSession { + if len(regionName) == 0 { + regionName = cli.getDefaultRegionName() } - cli.tokenCredential = tokenCredential - return nil + client := cli.getDefaultClient() + return client.NewSession(context.Background(), regionName, "", cli.endpointType, cli.tokenCredential, "") +} + +func (cli *SOpenStackClient) getDefaultClient() *mcclient.Client { + client := mcclient.NewClient(cli.authURL, 5, cli.debug, false, "", "") + client.SetHttpTransportProxyFunc(cli.cpcfg.ProxyFunc) + return client +} + +func (cli *SOpenStackClient) getDefaultToken() (mcclient.TokenCredential, error) { + client := cli.getDefaultClient() + token, err := client.Authenticate(cli.username, cli.password, cli.domainName, cli.project, cli.projectDomain) + if err != nil { + return nil, errors.Wrap(err, "Authenticate") + } + return token, nil } func (cli *SOpenStackClient) getProjectTokenCredential(projectId string) (mcclient.TokenCredential, error) { @@ -255,8 +424,8 @@ func (cli *SOpenStackClient) getProjectTokenCredential(projectId string) (mcclie if err != nil { return nil, errors.Wrapf(err, "GetProject(%s)", projectId) } - region := cli.iregions[0].(*SRegion) - s := cli.client.NewSession(context.Background(), region.Name, "", cli.endpointType, cli.tokenCredential, "") + + s := cli.getDefaultSession("") roleId, err := modules.RolesV3.GetId(s, "admin", jsonutils.Marshal(map[string]string{})) if err != nil { @@ -267,13 +436,7 @@ func (cli *SOpenStackClient) getProjectTokenCredential(projectId string) (mcclie return nil, errors.Wrap(err, "RolesV3.PutInContexts") } - cli.client = mcclient.NewClient(cli.authURL, 5, cli.debug, false, "", "") - cli.client.SetHttpTransportProxyFunc(cli.cpcfg.ProxyFunc) - tokenCredential, err := cli.client.Authenticate(cli.username, cli.password, cli.domainName, project.Name, cli.projectDomain) - if err != nil { - return nil, errors.Wrap(err, "Authenticate") - } - return tokenCredential, nil + return cli.getProjectToken(project.Id, project.Name) } func (cli *SOpenStackClient) GetRegion(regionId string) *SRegion { @@ -307,56 +470,47 @@ func (cli *SOpenStackClient) GetRegions() []SRegion { return regions } -func (cli *SOpenStackClient) GetIProjects() ([]cloudprovider.ICloudProject, error) { - if len(cli.iregions) > 0 { - region := cli.iregions[0].(*SRegion) - s := cli.client.NewSession(context.Background(), region.Name, "", cli.endpointType, cli.tokenCredential, "") - result, err := modules.Projects.List(s, jsonutils.NewDict()) - if err != nil { - return nil, err - } - iprojects := []cloudprovider.ICloudProject{} - for i := 0; i < len(result.Data); i++ { - project := &SProject{} - if err := result.Data[i].Unmarshal(project); err != nil { - return nil, err - } - iprojects = append(iprojects, project) - } - return iprojects, nil +func (cli *SOpenStackClient) fetchProjects() error { + var err error + cli.projects, err = cli.GetProjects() + if err != nil { + return errors.Wrap(err, "GetProjects") } - return nil, cloudprovider.ErrNotImplemented + return nil +} + +func (cli *SOpenStackClient) GetIProjects() ([]cloudprovider.ICloudProject, error) { + err := cli.fetchProjects() + if err != nil { + return nil, errors.Wrap(err, "fetchProjects") + } + iprojects := []cloudprovider.ICloudProject{} + for i := 0; i < len(cli.projects); i++ { + cli.projects[i].client = cli + iprojects = append(iprojects, &cli.projects[i]) + } + return iprojects, nil } func (cli *SOpenStackClient) GetProject(id string) (*SProject, error) { - if len(cli.iregions) > 0 { - region := cli.iregions[0].(*SRegion) - s := cli.client.NewSession(context.Background(), region.Name, "", cli.endpointType, cli.tokenCredential, "") - result, err := modules.Projects.Get(s, id, jsonutils.NewDict()) - if err != nil { - return nil, err - } - project := &SProject{} - err = result.Unmarshal(project) - if err != nil { - return nil, errors.Wrap(err, "result.Unmarshal") - } - return project, nil + err := cli.fetchProjects() + if err != nil { + return nil, errors.Wrap(err, "fetchProjects") } - return nil, fmt.Errorf("no region info") + for i := 0; i < len(cli.projects); i++ { + if cli.projects[i].Id == id { + return &cli.projects[i], nil + } + } + return nil, cloudprovider.ErrNotFound } func (cli *SOpenStackClient) CreateIProject(name string) (cloudprovider.ICloudProject, error) { - if len(cli.iregions) > 0 { - return cli.CreateProject(name, "") - } - return nil, cloudprovider.ErrNotImplemented + return cli.CreateProject(name, "") } func (cli *SOpenStackClient) CreateProject(name, desc string) (*SProject, error) { - region := cli.iregions[0].(*SRegion) - s := cli.client.NewSession(context.Background(), region.Name, "", cli.endpointType, cli.tokenCredential, "") - + s := cli.getDefaultSession("") params := map[string]string{ "name": name, "domain_id": s.GetDomainId(), @@ -368,7 +522,7 @@ func (cli *SOpenStackClient) CreateProject(name, desc string) (*SProject, error) if err != nil { return nil, errors.Wrap(err, "Projects.Create") } - project := SProject{} + project := SProject{client: cli} err = result.Unmarshal(&project) if err != nil { return nil, errors.Wrap(err, "result.Unmarshal") diff --git a/pkg/multicloud/openstack/port.go b/pkg/multicloud/openstack/port.go index 673f6e3b0f..30d9f1271d 100644 --- a/pkg/multicloud/openstack/port.go +++ b/pkg/multicloud/openstack/port.go @@ -15,7 +15,6 @@ package openstack import ( - "fmt" "net/url" "strings" "time" @@ -144,7 +143,7 @@ func (port *SPort) GetICloudInterfaceAddresses() ([]cloudprovider.ICloudInterfac } func (region *SRegion) GetINetworkInterfaces() ([]cloudprovider.ICloudNetworkInterface, error) { - ports, err := region.GetPorts("") + ports, err := region.GetPorts("", "") if err != nil { return nil, err } @@ -160,49 +159,47 @@ func (region *SRegion) GetINetworkInterfaces() ([]cloudprovider.ICloudNetworkInt } func (region *SRegion) GetPort(portId string) (*SPort, error) { - _, resp, err := region.Get("network", "/v2.0/ports/"+portId, "", nil) + resource := "/v2.0/ports/" + portId + resp, err := region.vpcGet(resource) if err != nil { return nil, err } port := &SPort{} - return port, resp.Unmarshal(port, "port") + err = resp.Unmarshal(port, "port") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return port, nil } -func (region *SRegion) GetPorts(macAddress string) ([]SPort, error) { - base := fmt.Sprintf("/v2.0/ports") - params := url.Values{} +func (region *SRegion) GetPorts(macAddress, deviceId string) ([]SPort, error) { + resource, ports := "/v2.0/ports", []SPort{} + query := url.Values{} if len(macAddress) > 0 { - params.Set("mac_address", macAddress) + query.Set("mac_address", macAddress) } - url := fmt.Sprintf("%s?%s", base, params.Encode()) - - ports := []SPort{} - for len(url) > 0 { - _, resp, err := region.List("network", url, "", nil) + if len(deviceId) > 0 { + query.Set("device_id", deviceId) + } + for { + resp, err := region.vpcList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vpcList") } - _ports := []SPort{} - err = resp.Unmarshal(&_ports, "ports") + part := struct { + Ports []SPort + PortsLinks SNextLinks + }{} + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_ports, "ports")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - ports = append(ports, _ports...) - url = "" - if resp.Contains("ports_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "ports_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "ports_links")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } - } + ports = append(ports, part.Ports...) + marker := part.PortsLinks.GetNextMark() + if len(marker) == 0 { + break } + query.Set("marker", marker) } - return ports, nil } diff --git a/pkg/multicloud/openstack/project.go b/pkg/multicloud/openstack/project.go index dd4df9585b..0e36111513 100644 --- a/pkg/multicloud/openstack/project.go +++ b/pkg/multicloud/openstack/project.go @@ -15,20 +15,27 @@ package openstack import ( + "net/url" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/httputils" ) type SProject struct { + client *SOpenStackClient Description string Enabled bool - ID string + Id string Name string } func (p *SProject) GetId() string { - return p.ID + return p.Id } func (p *SProject) GetGlobalId() string { @@ -44,9 +51,18 @@ func (p *SProject) GetName() string { } func (p *SProject) GetStatus() string { + _, err := p.getToken() + if err != nil { + log.Errorf("get project %s token error: %v %T", p.Name, err, err) + return api.EXTERNAL_PROJECT_STATUS_UNKNOWN + } return api.EXTERNAL_PROJECT_STATUS_AVAILABLE } +func (p *SProject) getToken() (mcclient.TokenCredential, error) { + return p.client.getProjectToken(p.Id, p.Name) +} + func (p *SProject) IsEmulated() bool { return false } @@ -54,3 +70,48 @@ func (p *SProject) IsEmulated() bool { func (p *SProject) Refresh() error { return nil } + +type SProjectLinks struct { + Next string + Previous string + Self string +} + +func (link SProjectLinks) GetNextMark() string { + if len(link.Next) == 0 || link.Next == "null" { + return "" + } + next, err := url.Parse(link.Next) + if err != nil { + log.Errorf("parse next link %s error: %v", link.Next, err) + return "" + } + return next.Query().Get("marker") +} + +func (cli *SOpenStackClient) GetProjects() ([]SProject, error) { + resource := "/v3/projects" + projects := []SProject{} + query := url.Values{} + for { + resp, err := cli.iamRequest("", httputils.GET, resource, query, nil) + if err != nil { + return nil, errors.Wrap(err, "iamRequest") + } + part := struct { + Projects []SProject + Links SProjectLinks + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "iamRequest") + } + projects = append(projects, part.Projects...) + marker := part.Links.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) + } + return projects, nil +} diff --git a/pkg/multicloud/openstack/provider/provider.go b/pkg/multicloud/openstack/provider/provider.go index 5f414fce14..2909b21b06 100644 --- a/pkg/multicloud/openstack/provider/provider.go +++ b/pkg/multicloud/openstack/provider/provider.go @@ -102,6 +102,10 @@ func (self *SOpenStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx return output, nil } +func (self *SOpenStackProviderFactory) IsNeedForceAutoCreateProject() bool { + return true +} + func (self *SOpenStackProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) { accountInfo := strings.Split(cfg.Account, "/") if len(accountInfo) < 2 { @@ -200,7 +204,7 @@ func (self *SOpenStackProvider) GetIProjects() ([]cloudprovider.ICloudProject, e return self.client.GetIProjects() } -func (self *SOpenStackProvider) CreateProject(name string) (cloudprovider.ICloudProject, error) { +func (self *SOpenStackProvider) CreateIProject(name string) (cloudprovider.ICloudProject, error) { return self.client.CreateIProject(name) } diff --git a/pkg/multicloud/openstack/quota.go b/pkg/multicloud/openstack/quota.go index 63cb701960..60791bf60a 100644 --- a/pkg/multicloud/openstack/quota.go +++ b/pkg/multicloud/openstack/quota.go @@ -17,11 +17,9 @@ package openstack import ( "fmt" - "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/util/version" ) type QuotaDetail struct { @@ -70,46 +68,44 @@ type SQuota struct { } func (region *SRegion) GetQuota() (*QuotaSet, error) { - _, resp, err := region.Get("compute", fmt.Sprintf("/os-quota-sets/%s/detail", region.client.tokenCredential.GetTenantId()), "", nil) + resource := fmt.Sprintf("/os-quota-sets/%s/detail", region.client.tokenCredential.GetTenantId()) + resp, err := region.ecsGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "ecsGet") } quota := &QuotaSet{} return quota, resp.Unmarshal(quota, "quota_set") } func (region *SRegion) SetQuota(quota *SQuota) error { - _, maxVersion, _ := region.GetVersion("compute") params := map[string]map[string]interface{}{ "quota_set": { "force": "True", }, } - if version.GE(maxVersion, "2.35") { - if quota.Floatingips > 0 { - params["quota_set"]["floating_ips"] = quota.Floatingips - } - - if quota.SecurityGroups > 0 { - params["quota_set"]["security_group"] = quota.SecurityGroups - } - - if quota.SecurityGroupRules > 0 { - params["quota_set"]["security_group_rules"] = quota.SecurityGroupRules - } - - if quota.FixedIps > 0 { - params["quota_set"]["fixed_ips"] = quota.FixedIps - } - - if quota.Networks > 0 { - params["quota_set"]["networks"] = quota.Networks - } - + if quota.Floatingips > 0 { + params["quota_set"]["floating_ips"] = quota.Floatingips } - _, _, err := region.Update("compute", "/os-quota-sets/"+region.client.tokenCredential.GetTenantId(), maxVersion, jsonutils.Marshal(params)) + if quota.SecurityGroups > 0 { + params["quota_set"]["security_group"] = quota.SecurityGroups + } + + if quota.SecurityGroupRules > 0 { + params["quota_set"]["security_group_rules"] = quota.SecurityGroupRules + } + + if quota.FixedIps > 0 { + params["quota_set"]["fixed_ips"] = quota.FixedIps + } + + if quota.Networks > 0 { + params["quota_set"]["networks"] = quota.Networks + } + + resource := "/os-quota-sets/" + region.client.tokenCredential.GetTenantId() + _, err := region.ecsUpdate(resource, params) return err } diff --git a/pkg/multicloud/openstack/region.go b/pkg/multicloud/openstack/region.go index 39f451d749..495547d27a 100644 --- a/pkg/multicloud/openstack/region.go +++ b/pkg/multicloud/openstack/region.go @@ -16,12 +16,10 @@ package openstack import ( "fmt" - "net/http" + "io" "net/url" - "strings" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -37,8 +35,8 @@ type SRegion struct { Name string - izones []cloudprovider.ICloudZone - ivpcs []cloudprovider.ICloudVpc + zones []SZone + vpcs []SVpc storageCache *SStoragecache routers []SRouter @@ -93,26 +91,16 @@ func (region *SRegion) Refresh() error { return nil } +func (region *SRegion) GetMaxVersion(service string) (string, error) { + return region.client.GetMaxVersion(region.Name, service) +} + func (region *SRegion) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) { - params := map[string]map[string]string{ - "network": { - "name": name, - "description": desc, - }, - } - _, resp, err := region.Post("network", "/v2.0/networks", "", jsonutils.Marshal(params)) + vpc, err := region.CreateVpc(name, desc) if err != nil { - return nil, err + return nil, errors.Wrap(err, "CreateVp") } - err = region.fetchInfrastructure() - if err != nil { - return nil, err - } - vpcId, err := resp.GetString("network", "id") - if err != nil { - return nil, err - } - return region.GetIVpcById(vpcId) + return vpc, nil } func (region *SRegion) GetIHostById(id string) (cloudprovider.ICloudHost, error) { @@ -200,30 +188,37 @@ func (region *SRegion) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, } func (region *SRegion) GetIVMById(id string) (cloudprovider.ICloudVM, error) { - return region.GetInstance(id) + instance, err := region.GetInstance(id) + if err != nil { + return nil, errors.Wrapf(err, "GetInstance(%s)", id) + } + return instance, nil } func (region *SRegion) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) { - return region.GetDisk(id) + disk, err := region.GetDisk(id) + if err != nil { + _, err := region.GetInstance(id) + if err == nil { + return &SNovaDisk{region: region, instanceId: id}, nil + } + return nil, errors.Wrapf(err, "GetDisk(%s)", id) + } + return disk, nil } func (region *SRegion) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) { - ivpcs, err := region.GetIVpcs() + vpc, err := region.GetVpc(id) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "GetVpc(%s)", id) } - for i := 0; i < len(ivpcs); i++ { - if ivpcs[i].GetGlobalId() == id { - return ivpcs[i], nil - } - } - return nil, cloudprovider.ErrNotFound + return vpc, nil } func (region *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) { izones, err := region.GetIZones() if err != nil { - return nil, err + return nil, errors.Wrap(err, "GetIZones") } for i := 0; i < len(izones); i++ { if izones[i].GetGlobalId() == id { @@ -234,170 +229,129 @@ func (region *SRegion) GetIZoneById(id string) (cloudprovider.ICloudZone, error) } func (region *SRegion) fetchZones() error { - zone := &SZone{region: region, ZoneName: region.Name, cachedHosts: map[string][]string{}} - _, resp, err := region.List("compute", "/os-availability-zone/detail", "", jsonutils.NewDict()) + if len(region.zones) > 0 { + return nil + } + zones, err := region.GetZones() if err != nil { - return err - } - zones := []SZone{} - if err := resp.Unmarshal(&zones, "availabilityZoneInfo"); err != nil { - return err + return errors.Wrap(err, "GetZones") } + region.zones = []SZone{} for i := 0; i < len(zones); i++ { if zones[i].ZoneName == "internal" { continue } - zone.cachedHosts[zones[i].ZoneName] = []string{} - for hostname, hostInfo := range zones[i].Hosts { - for k := range hostInfo { - if k == "nova-compute" { - zone.cachedHosts[zones[i].ZoneName] = append(zone.cachedHosts[zones[i].ZoneName], hostname) - } - } - } - } - region.izones = []cloudprovider.ICloudZone{zone} - return nil -} - -func (region *SRegion) fetchIVpcs() error { - vpcs, err := region.GetVpcs() - if err != nil { - return err - } - region.ivpcs = []cloudprovider.ICloudVpc{} - for i := 0; i < len(vpcs); i++ { - vpcs[i].region = region - region.ivpcs = append(region.ivpcs, &vpcs[i]) + zones[i].region = region + region.zones = append(region.zones, zones[i]) } return nil } -func (region *SRegion) fetchInfrastructure() error { - if len(region.izones) == 0 { - if err := region.fetchZones(); err != nil { - return err - } +func (region *SRegion) fetchVpcs() error { + if len(region.vpcs) > 0 { + return nil } - if err := region.fetchIVpcs(); err != nil { - return err - } - for i := 0; i < len(region.ivpcs); i++ { - for j := 0; j < len(region.izones); j++ { - zone := region.izones[j].(*SZone) - vpc := region.ivpcs[i].(*SVpc) - wire := SWire{zone: zone, vpc: vpc} - zone.addWire(&wire) - vpc.addWire(&wire) - } + var err error + region.vpcs, err = region.GetVpcs("") + if err != nil { + return errors.Wrap(err, "GetVpcs") } return nil } -func (region *SRegion) Get(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - if strings.HasSuffix(url, "/") { - return nil, nil, cloudprovider.ErrNotFound - } - header, resp, err := region.client.Request("", region.Name, service, "GET", url, microversion, body) - if err != nil { - if jsonErr, ok := err.(*httputils.JSONClientError); ok { - if jsonErr.Code == 404 || strings.HasSuffix(jsonErr.Class, "NotFound") { - return nil, nil, cloudprovider.ErrNotFound - } - } - return nil, nil, err - } - return header, resp, nil +func (region *SRegion) ecsList(resource string, query url.Values) (jsonutils.JSONObject, error) { + return region.client.ecsRequest(region.Name, httputils.GET, resource, query, nil) } -func (region *SRegion) List(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - header, resp, err := region.client.Request("", region.Name, service, "GET", url, microversion, body) - if err != nil { - if jsonErr, ok := err.(*httputils.JSONClientError); ok { - if jsonErr.Code == 404 || strings.HasSuffix(jsonErr.Class, "NotFound") { - return nil, nil, cloudprovider.ErrNotFound - } - } - return nil, nil, err - } - return header, resp, nil +func (region *SRegion) ecsGet(resource string) (jsonutils.JSONObject, error) { + return region.client.ecsRequest(region.Name, httputils.GET, resource, nil, nil) } -func (region *SRegion) Post(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - return region.client.Request("", region.Name, service, "POST", url, microversion, body) +func (region *SRegion) ecsUpdate(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.ecsRequest(region.Name, httputils.PUT, resource, nil, params) } -func (region *SRegion) PostWithProject(projectId string, service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - return region.client.Request(projectId, region.Name, service, "POST", url, microversion, body) +func (region *SRegion) ecsPost(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.ecsRequest(region.Name, httputils.POST, resource, nil, params) } -func (region *SRegion) Update(service, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - return region.client.Request("", region.Name, service, "PUT", url, microversion, body) +func (region *SRegion) ecsDo(projectId, resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.ecsDo(projectId, region.Name, resource, params) } -func (region *SRegion) Delete(service, url string, microversion string) (*http.Response, error) { - return region.client.RawRequest(region.Name, service, "DELETE", url, microversion, nil) +func (region *SRegion) ecsDelete(resource string) (jsonutils.JSONObject, error) { + return region.client.ecsRequest(region.Name, httputils.DELETE, resource, nil, nil) } -func (region *SRegion) CinderList(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - for _, service := range []string{"volumev3", "volumev2", "volume"} { - header, resp, err := region.Get(service, url, microversion, body) - if err == nil || !strings.Contains(err.Error(), "No such service") { - return header, resp, err - } - log.Debugf("failed to list %s by service %s error: %v, try another", url, service, err) - } - return nil, nil, fmt.Errorf("failed to get %s by cinder service", url) +func (region *SRegion) ecsCreate(projectId, resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.ecsCreate(projectId, region.Name, resource, params) } -func (region *SRegion) CinderGet(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - if strings.HasSuffix(url, "/") { - return nil, nil, cloudprovider.ErrNotFound - } - for _, service := range []string{"volumev3", "volumev2", "volume"} { - header, resp, err := region.Get(service, url, microversion, body) - if err == nil || err == cloudprovider.ErrNotFound || !strings.Contains(err.Error(), "No such service") { - return header, resp, err - } - log.Debugf("failed to get %s by service %s error: %v, try another", url, service, err) - } - return nil, nil, fmt.Errorf("failed to get %s by cinder service", url) +func (region *SRegion) vpcList(resource string, query url.Values) (jsonutils.JSONObject, error) { + return region.client.vpcRequest(region.Name, httputils.GET, resource, query, nil) } -func (region *SRegion) CinderCreate(projectId string, url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - for _, service := range []string{"volumev3", "volumev2", "volume"} { - header, resp, err := region.PostWithProject(projectId, service, url, microversion, body) - if err == nil || !strings.Contains(err.Error(), "No such service") { - return header, resp, err - } - log.Debugf("failed to create %s by service %s error: %v, try another", url, service, err) - } - return nil, nil, fmt.Errorf("failed to create %s by cinder service", url) +func (region *SRegion) vpcGet(resource string) (jsonutils.JSONObject, error) { + return region.client.vpcRequest(region.Name, httputils.GET, resource, nil, nil) } -func (region *SRegion) CinderDelete(url string, microversion string) (*http.Response, error) { - if strings.HasSuffix(url, "/") { - return nil, cloudprovider.ErrNotFound - } - for _, service := range []string{"volumev3", "volumev2", "volume"} { - resp, err := region.Delete(service, url, microversion) - if err == nil || !strings.Contains(err.Error(), "No such service") { - return resp, err - } - log.Debugf("failed to delete %s by service %s error: %v, try another", url, service, err) - } - return nil, fmt.Errorf("failed to delete %s by cinder service", url) +func (region *SRegion) vpcUpdate(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.vpcRequest(region.Name, httputils.PUT, resource, nil, params) } -func (region *SRegion) CinderAction(url string, microversion string, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) { - for _, service := range []string{"volumev3", "volumev2", "volume"} { - header, resp, err := region.Post(service, url, microversion, body) - if err == nil || !strings.Contains(err.Error(), "No such service") { - return header, resp, err - } - log.Debugf("failed to operate %s by service %s error: %v, try another", url, service, err) - } - return nil, nil, fmt.Errorf("failed to operate %s by cinder service", url) +func (region *SRegion) vpcPost(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.vpcRequest(region.Name, httputils.POST, resource, nil, params) +} + +func (region *SRegion) vpcDelete(resource string) (jsonutils.JSONObject, error) { + return region.client.vpcRequest(region.Name, httputils.DELETE, resource, nil, nil) +} + +func (region *SRegion) imageList(resource string, query url.Values) (jsonutils.JSONObject, error) { + return region.client.imageRequest(region.Name, httputils.GET, resource, query, nil) +} + +func (region *SRegion) imageGet(resource string) (jsonutils.JSONObject, error) { + return region.client.imageRequest(region.Name, httputils.GET, resource, nil, nil) +} + +func (region *SRegion) imagePost(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.imageRequest(region.Name, httputils.POST, resource, nil, params) +} + +func (region *SRegion) imageDelete(resource string) (jsonutils.JSONObject, error) { + return region.client.imageRequest(region.Name, httputils.DELETE, resource, nil, nil) +} + +func (region *SRegion) imageUpload(url string, body io.Reader) error { + resp, err := region.client.imageUpload(region.Name, url, body) + _, _, err = httputils.ParseResponse("", resp, err, region.client.debug) + return err +} + +// Block Storage +func (region *SRegion) bsList(resource string, query url.Values) (jsonutils.JSONObject, error) { + return region.client.bsRequest(region.Name, httputils.GET, resource, query, nil) +} + +func (region *SRegion) bsGet(resource string) (jsonutils.JSONObject, error) { + return region.client.bsRequest(region.Name, httputils.GET, resource, nil, nil) +} + +func (region *SRegion) bsUpdate(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.bsRequest(region.Name, httputils.PUT, resource, nil, params) +} + +func (region *SRegion) bsPost(resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.bsRequest(region.Name, httputils.POST, resource, nil, params) +} + +func (region *SRegion) bsDelete(resource string) (jsonutils.JSONObject, error) { + return region.client.bsRequest(region.Name, httputils.DELETE, resource, nil, nil) +} + +func (region *SRegion) bsCreate(projectId, resource string, params interface{}) (jsonutils.JSONObject, error) { + return region.client.bsCreate(projectId, region.Name, resource, params) } func (region *SRegion) ProjectId() string { @@ -405,27 +359,32 @@ func (region *SRegion) ProjectId() string { } func (region *SRegion) GetIZones() ([]cloudprovider.ICloudZone, error) { - if region.izones == nil { - if err := region.fetchInfrastructure(); err != nil { - return nil, err - } + err := region.fetchZones() + if err != nil { + return nil, err } - return region.izones, nil -} - -func (region *SRegion) GetVersion(service string) (string, string, error) { - return region.client.getVersion(region.Name, service) + izones := []cloudprovider.ICloudZone{} + for i := range region.zones { + izones = append(izones, ®ion.zones[i]) + } + return izones, nil } func (region *SRegion) GetIVpcs() ([]cloudprovider.ICloudVpc, error) { - if err := region.fetchInfrastructure(); err != nil { - return nil, err + err := region.fetchVpcs() + if err != nil { + return nil, errors.Wrap(err, "fetchVpcs") } - return region.ivpcs, nil + ivpcs := []cloudprovider.ICloudVpc{} + for i := range region.vpcs { + region.vpcs[i].region = region + ivpcs = append(ivpcs, ®ion.vpcs[i]) + } + return ivpcs, nil } func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { - eips, err := region.GetEips() + eips, err := region.GetEips("") if err != nil { return nil, err } @@ -438,7 +397,15 @@ func (region *SRegion) GetIEips() ([]cloudprovider.ICloudEIP, error) { } func (region *SRegion) CreateEIP(eip *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) { - return region.CreateEip(eip) + network, err := region.GetNetwork(eip.NetworkExternalId) + if err != nil { + return nil, errors.Wrapf(err, "GetNetwork(%s)", eip.NetworkExternalId) + } + ieip, err := region.CreateEip(network.NetworkId, eip.NetworkExternalId, eip.IP, eip.ProjectId) + if err != nil { + return nil, errors.Wrap(err, "CreateEip") + } + return ieip, nil } func (region *SRegion) GetIEipById(eipId string) (cloudprovider.ICloudEIP, error) { @@ -522,8 +489,8 @@ func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.I return region.GetSecurityGroup(secgroupId) } -func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, err := region.GetSecurityGroups(name) +func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, err := region.GetSecurityGroups(opts.ProjectId, opts.Name) if err != nil { return nil, err } @@ -538,7 +505,7 @@ func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloud } func (region *SRegion) CreateISecurityGroup(conf *cloudprovider.SecurityGroupCreateInput) (cloudprovider.ICloudSecurityGroup, error) { - return region.CreateSecurityGroup(conf.Name, conf.Desc) + return region.CreateSecurityGroup(conf.ProjectId, conf.Name, conf.Desc) } func (region *SRegion) GetCapabilities() []string { @@ -546,9 +513,10 @@ func (region *SRegion) GetCapabilities() []string { } func (region *SRegion) GetRouters() ([]SRouter, error) { - _, resp, err := region.Get("network", "/v2.0/routers", "", nil) + resource := "/v2.0/routers" + resp, err := region.vpcList(resource, nil) if err != nil { - return nil, errors.Wrap(err, "region.Get routes") + return nil, errors.Wrap(err, "vpcList.routes") } routers := []SRouter{} err = resp.Unmarshal(&routers, "routers") @@ -556,7 +524,7 @@ func (region *SRegion) GetRouters() ([]SRouter, error) { return nil, errors.Wrap(err, "resp.Unmarshal") } for i := 0; i < len(routers); i++ { - ports, err := region.GetPortsByDeviceId(routers[i].ID) + ports, err := region.GetPorts("", routers[i].Id) if err != nil { return nil, errors.Wrap(err, "vpc.region.GetPortsByDeviceId") } @@ -576,15 +544,3 @@ func (region *SRegion) fetchrouters() error { region.routers = routers return nil } - -func (region *SRegion) GetPortsByDeviceId(routerId string) ([]SPort, error) { - query := url.Values{} - query.Set("device_id", routerId) - _, resp, err := region.Get("network", "/v2.0/ports?"+query.Encode(), "", nil) - if err != nil { - return nil, errors.Wrap(err, "region.Get routes") - } - ports := []SPort{} - return ports, resp.Unmarshal(&ports, "ports") - -} diff --git a/pkg/multicloud/openstack/router.go b/pkg/multicloud/openstack/router.go index a623e002aa..5b4c050eb2 100644 --- a/pkg/multicloud/openstack/router.go +++ b/pkg/multicloud/openstack/router.go @@ -16,12 +16,12 @@ package openstack type SExternalFixedIP struct { IPAddress string `json:"ip_address"` - SubnetID string `json:"subnet_id"` + SubnetId string `json:"subnet_id"` } type SExternalGatewayInfo struct { EnableSnat bool `json:"enable_snat"` ExtrernalFiedIps []SExternalFixedIP `json:"external_fixed_ips"` - NetworkID string `json:"network_id"` + NetworkId string `json:"network_id"` } type SConntrackHelper struct { @@ -33,13 +33,13 @@ type SRouter struct { ports []SPort AdminStateUp bool `json:"admin_state_up"` Description string `json:"description"` - FlavorID string `json:"flavor_id"` - ID string `json:"id"` + FlavorId string `json:"flavor_id"` + Id string `json:"id"` Name string `json:"name"` Routes []SRouteEntry `json:"routes"` Status string `json:"status"` - ProjectID string `json:"project_id"` - TenantID string `json:"tenant_id"` + ProjectId string `json:"project_id"` + TenantId string `json:"tenant_id"` Tags []string `json:"tags"` ConntrackHelpers []SConntrackHelper `json:"conntrack_helpers"` ExternalGatewayInfo SExternalGatewayInfo `json:"external_gateway_info"` diff --git a/pkg/multicloud/openstack/routetable.go b/pkg/multicloud/openstack/routetable.go index a4941a1290..8fa18b89b9 100644 --- a/pkg/multicloud/openstack/routetable.go +++ b/pkg/multicloud/openstack/routetable.go @@ -57,7 +57,7 @@ func (self *SRouteTable) GetId() string { } func (self *SRouteTable) GetGlobalId() string { - return self.router.ID + return self.router.Id } func (self *SRouteTable) GetName() string { diff --git a/pkg/multicloud/openstack/securitygroup.go b/pkg/multicloud/openstack/securitygroup.go index 6d3faf6983..466e0c6319 100644 --- a/pkg/multicloud/openstack/securitygroup.go +++ b/pkg/multicloud/openstack/securitygroup.go @@ -16,6 +16,7 @@ package openstack import ( "fmt" + "net/url" "time" "yunion.io/x/jsonutils" @@ -37,17 +38,17 @@ const ( type SSecurityGroupRule struct { Direction string Ethertype string - ID string + Id string PortRangeMax int PortRangeMin int Protocol string - RemoteGroupID string + RemoteGroupId string RemoteIpPrefix string - SecurityGroupID string - ProjectID string + SecurityGroupId string + ProjectId string RevisionNumber int Tags []string - TenantID string + TenantId string CreatedAt time.Time UpdatedAt time.Time Description string @@ -58,62 +59,61 @@ type SSecurityGroup struct { region *SRegion Description string - ID string + Id string Name string SecurityGroupRules []SSecurityGroupRule - ProjectID string + ProjectId string RevisionNumber int CreatedAt time.Time UpdatedAt time.Time Tags []string - TenantID string + TenantId string } func (region *SRegion) GetSecurityGroup(secgroupId string) (*SSecurityGroup, error) { - _, resp, err := region.Get("network", "/v2.0/security-groups/"+secgroupId, "", nil) + resource := "/v2.0/security-groups/" + secgroupId + resp, err := region.vpcGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vpcGet") } secgroup := &SSecurityGroup{region: region} - return secgroup, resp.Unmarshal(secgroup, "security_group") + err = resp.Unmarshal(secgroup, "security_group") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return secgroup, nil } -func (region *SRegion) GetSecurityGroups(name string) ([]SSecurityGroup, error) { - url := "/v2.0/security-groups" - if len(name) > 0 { - url = fmt.Sprintf("%s?name=%s", url, name) - } +func (region *SRegion) GetSecurityGroups(projectId, name string) ([]SSecurityGroup, error) { secgroups := []SSecurityGroup{} - for len(url) > 0 { - _, resp, err := region.List("network", url, "", nil) - if err != nil { - return nil, err - } - _secgroups := []SSecurityGroup{} - err = resp.Unmarshal(&_secgroups, "security_groups") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_secgroups, "security_groups")`) - } - secgroups = append(secgroups, _secgroups...) - url = "" - if resp.Contains("security_groups_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "security_groups_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "security_groups_links")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } - } - } + resource := "/v2.0/security-groups" + query := url.Values{} + if len(name) > 0 { + query.Set("name", name) } - for i := range secgroups { - secgroups[i].region = region + if len(projectId) > 0 { + query.Set("project_id", projectId) + } + for { + resp, err := region.vpcList(resource, query) + if err != nil { + return nil, errors.Wrap(err, "vpcList") + } + part := struct { + SecurityGroups []SSecurityGroup + SecurityGroupsLinks SNextLinks + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + secgroups = append(secgroups, part.SecurityGroups...) + marker := part.SecurityGroupsLinks.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) } - return secgroups, nil } @@ -126,11 +126,11 @@ func (secgroup *SSecurityGroup) GetVpcId() string { } func (secgroup *SSecurityGroup) GetId() string { - return secgroup.ID + return secgroup.Id } func (secgroup *SSecurityGroup) GetGlobalId() string { - return secgroup.ID + return secgroup.Id } func (secgroup *SSecurityGroup) GetDescription() string { @@ -141,17 +141,17 @@ func (secgroup *SSecurityGroup) GetName() string { if len(secgroup.Name) > 0 { return secgroup.Name } - return secgroup.ID + return secgroup.Id } func (secgrouprule *SSecurityGroupRule) toRules() ([]cloudprovider.SecurityRule, error) { rules := []cloudprovider.SecurityRule{} // 暂时忽略IPv6安全组规则,忽略远端也是安全组的规则 - if secgrouprule.Ethertype != "IPv4" || len(secgrouprule.RemoteGroupID) > 0 { - return rules, fmt.Errorf("ethertype: %s remoteGroupId: %s", secgrouprule.Ethertype, secgrouprule.RemoteGroupID) + if secgrouprule.Ethertype != "IPv4" || len(secgrouprule.RemoteGroupId) > 0 { + return rules, fmt.Errorf("ethertype: %s remoteGroupId: %s", secgrouprule.Ethertype, secgrouprule.RemoteGroupId) } rule := cloudprovider.SecurityRule{ - ExternalId: secgrouprule.ID, + ExternalId: secgrouprule.Id, SecurityRule: secrules.SecurityRule{ Direction: secrules.DIR_IN, Action: secrules.SecurityRuleAllow, @@ -197,7 +197,7 @@ func (secgroup *SSecurityGroup) GetRules() ([]cloudprovider.SecurityRule, error) for _, rule := range secgroup.SecurityGroupRules { subRules, err := rule.toRules() if err != nil { - log.Errorf("failed to convert rule %s for secgroup %s(%s) error: %v", rule.ID, secgroup.Name, secgroup.ID, err) + log.Errorf("failed to convert rule %s for secgroup %s(%s) error: %v", rule.Id, secgroup.Name, secgroup.Id, err) continue } rules = append(rules, subRules...) @@ -214,7 +214,7 @@ func (secgroup *SSecurityGroup) IsEmulated() bool { } func (secgroup *SSecurityGroup) Refresh() error { - new, err := secgroup.region.GetSecurityGroup(secgroup.ID) + new, err := secgroup.region.GetSecurityGroup(secgroup.Id) if err != nil { return err } @@ -222,7 +222,8 @@ func (secgroup *SSecurityGroup) Refresh() error { } func (region *SRegion) delSecurityGroupRule(ruleId string) error { - _, err := region.Delete("network", "/v2.0/security-group-rules/"+ruleId, "") + resource := "/v2.0/security-group-rules/" + ruleId + _, err := region.vpcDelete(resource) return err } @@ -252,9 +253,10 @@ func (region *SRegion) addSecurityGroupRules(secgroupId string, rule cloudprovid for _, port := range rule.Ports { params["security_group_rule"]["port_range_max"] = port params["security_group_rule"]["port_range_min"] = port - _, _, err := region.Post("network", "/v2.0/security-group-rules", "", jsonutils.Marshal(params)) + resource := "/v2.0/security-group-rules" + _, err := region.vpcPost(resource, params) if err != nil { - return err + return errors.Wrap(err, "vpcPost") } } return nil @@ -263,36 +265,44 @@ func (region *SRegion) addSecurityGroupRules(secgroupId string, rule cloudprovid params["security_group_rule"]["port_range_min"] = rule.PortStart params["security_group_rule"]["port_range_max"] = rule.PortEnd } - _, _, err := region.Post("network", "/v2.0/security-group-rules", "", jsonutils.Marshal(params)) + _, err := region.vpcPost("/v2.0/security-group-rules", params) return err } func (region *SRegion) DeleteSecurityGroup(secGroupId string) error { - _, err := region.Delete("network", "/v2.0/security-groups/"+secGroupId, "") + resource := "/v2.0/security-groups/" + secGroupId + _, err := region.vpcDelete(resource) return err } func (secgroup *SSecurityGroup) Delete() error { - return secgroup.region.DeleteSecurityGroup(secgroup.ID) + return secgroup.region.DeleteSecurityGroup(secgroup.Id) } -func (region *SRegion) CreateSecurityGroup(name, description string) (*SSecurityGroup, error) { +func (region *SRegion) CreateSecurityGroup(projectId, name, description string) (*SSecurityGroup, error) { params := map[string]map[string]interface{}{ "security_group": { "name": name, "description": description, }, } - _, resp, err := region.Post("network", "/v2.0/security-groups", "", jsonutils.Marshal(params)) + if len(projectId) > 0 { + params["security_group"]["project_id"] = projectId + } + resp, err := region.vpcPost("/v2.0/security-groups", params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "vpcPost") } secgroup := &SSecurityGroup{region: region} - return secgroup, resp.Unmarshal(secgroup, "security_group") + err = resp.Unmarshal(secgroup, "security_group") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return secgroup, nil } func (secgroup *SSecurityGroup) GetProjectId() string { - return secgroup.TenantID + return secgroup.TenantId } func (secgroup *SSecurityGroup) SyncRules(common, inAdds, outAdds, inDels, outDels []cloudprovider.SecurityRule) error { @@ -303,7 +313,7 @@ func (secgroup *SSecurityGroup) SyncRules(common, inAdds, outAdds, inDels, outDe } } for _, r := range append(inAdds, outAdds...) { - err := secgroup.region.addSecurityGroupRules(secgroup.ID, r) + err := secgroup.region.addSecurityGroupRules(secgroup.Id, r) if err != nil { if jsonError, ok := err.(*httputils.JSONClientError); ok { if jsonError.Class == "SecurityGroupRuleExists" { diff --git a/pkg/multicloud/openstack/shell/disk.go b/pkg/multicloud/openstack/shell/disk.go index 522ce29e60..7127ec620e 100644 --- a/pkg/multicloud/openstack/shell/disk.go +++ b/pkg/multicloud/openstack/shell/disk.go @@ -21,11 +21,9 @@ import ( func init() { type DiskListOptions struct { - Category string `help:"Storage type for disk"` - BackendName string `help:"Storage backend eg:lvm, rbd"` } shellutils.R(&DiskListOptions{}, "disk-list", "List disks", func(cli *openstack.SRegion, args *DiskListOptions) error { - disks, err := cli.GetDisks(args.Category, args.BackendName) + disks, err := cli.GetDisks() if err != nil { return err } diff --git a/pkg/multicloud/openstack/shell/eip.go b/pkg/multicloud/openstack/shell/eip.go index 7d80e609bd..1e07e18f81 100644 --- a/pkg/multicloud/openstack/shell/eip.go +++ b/pkg/multicloud/openstack/shell/eip.go @@ -21,9 +21,10 @@ import ( func init() { type EipListOptions struct { + Ip string } shellutils.R(&EipListOptions{}, "eip-list", "List eips", func(cli *openstack.SRegion, args *EipListOptions) error { - eips, err := cli.GetEips() + eips, err := cli.GetEips(args.Ip) if err != nil { return err } @@ -38,4 +39,20 @@ func init() { return cli.DeleteEip(args.ID) }) + type EipCreateOptions struct { + NETWORK string + SUBNET string + Ip string + TenantId string + } + + shellutils.R(&EipCreateOptions{}, "eip-create", "Create eip", func(cli *openstack.SRegion, args *EipCreateOptions) error { + eip, err := cli.CreateEip(args.NETWORK, args.SUBNET, args.Ip, args.TenantId) + if err != nil { + return err + } + printObject(eip) + return nil + }) + } diff --git a/pkg/multicloud/openstack/shell/host.go b/pkg/multicloud/openstack/shell/host.go index d8c6be2f06..7c5f095c99 100644 --- a/pkg/multicloud/openstack/shell/host.go +++ b/pkg/multicloud/openstack/shell/host.go @@ -15,8 +15,6 @@ package shell import ( - "fmt" - "yunion.io/x/onecloud/pkg/multicloud/openstack" "yunion.io/x/onecloud/pkg/util/shellutils" ) @@ -25,7 +23,7 @@ func init() { type HostListOptions struct { } shellutils.R(&HostListOptions{}, "host-list", "List hosts", func(cli *openstack.SRegion, args *HostListOptions) error { - hosts, err := cli.GetIHosts() + hosts, err := cli.GetHosts() if err != nil { return err } @@ -33,24 +31,6 @@ func init() { return nil }) - type HostShowOptions struct { - ID string `help:"Host name"` - ZONE string `help:"Zone name"` - REGION string `help:"Region name"` - } - shellutils.R(&HostShowOptions{}, "host-show", "Show host detail", func(cli *openstack.SRegion, args *HostShowOptions) error { - zone, err := cli.GetIZoneById(fmt.Sprintf("%s/%s/%s", openstack.CLOUD_PROVIDER_OPENSTACK, args.REGION, args.ZONE)) - if err != nil { - return err - } - host, err := zone.GetIHostById(args.ID) - if err != nil { - return err - } - printObject(host) - return nil - }) - type AggregateListOption struct { } @@ -63,4 +43,16 @@ func init() { return nil }) + type HypervisorListOption struct { + } + + shellutils.R(&HypervisorListOption{}, "hypervisor-list", "List os-hypervisor", func(cli *openstack.SRegion, args *HypervisorListOption) error { + hypervisors, err := cli.GetHypervisors() + if err != nil { + return err + } + printList(hypervisors, 0, 0, 0, []string{}) + return nil + }) + } diff --git a/pkg/multicloud/openstack/shell/image.go b/pkg/multicloud/openstack/shell/image.go index 323144b1c1..2b28c3286c 100644 --- a/pkg/multicloud/openstack/shell/image.go +++ b/pkg/multicloud/openstack/shell/image.go @@ -15,6 +15,10 @@ package shell import ( + "os" + + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/multicloud/openstack" "yunion.io/x/onecloud/pkg/util/shellutils" ) @@ -57,10 +61,16 @@ func init() { OsDistro string MinDiskSizeGB int MinRamMb int + FILE string } shellutils.R(&ImageCreateOptions{}, "image-create", "Create image", func(cli *openstack.SRegion, args *ImageCreateOptions) error { - image, err := cli.CreateImage(args.NAME, args.OsType, args.OsDistro, args.MinDiskSizeGB, args.MinRamMb) + file, err := os.Open(args.FILE) + if err != nil { + return errors.Wrap(err, "os.Open") + } + defer file.Close() + image, err := cli.CreateImage(args.NAME, args.OsType, args.OsDistro, args.MinDiskSizeGB, args.MinRamMb, file) if err != nil { return err } diff --git a/pkg/multicloud/openstack/shell/instance.go b/pkg/multicloud/openstack/shell/instance.go index 6b5b146dde..0b990ef2a3 100644 --- a/pkg/multicloud/openstack/shell/instance.go +++ b/pkg/multicloud/openstack/shell/instance.go @@ -37,6 +37,16 @@ func init() { type InstanceOptions struct { ID string `help:"Instance ID"` } + + shellutils.R(&InstanceOptions{}, "instance-network-list", "List instance network", func(cli *openstack.SRegion, args *InstanceOptions) error { + ports, err := cli.GetInstancePorts(args.ID) + if err != nil { + return err + } + printObject(ports) + return nil + }) + shellutils.R(&InstanceOptions{}, "instance-show", "Show instance", func(cli *openstack.SRegion, args *InstanceOptions) error { instance, err := cli.GetInstance(args.ID) if err != nil { @@ -46,6 +56,15 @@ func init() { return nil }) + shellutils.R(&InstanceOptions{}, "instance-metadata", "Show instance metadata", func(cli *openstack.SRegion, args *InstanceOptions) error { + metadata, err := cli.GetInstanceMetadata(args.ID) + if err != nil { + return err + } + printObject(metadata) + return nil + }) + shellutils.R(&InstanceOptions{}, "instance-vnc", "Show instance vnc url", func(cli *openstack.SRegion, args *InstanceOptions) error { url, err := cli.GetInstanceVNCUrl(args.ID) if err != nil { diff --git a/pkg/multicloud/openstack/shell/instancenic.go b/pkg/multicloud/openstack/shell/messages.go similarity index 70% rename from pkg/multicloud/openstack/shell/instancenic.go rename to pkg/multicloud/openstack/shell/messages.go index d66aa5239e..31c61fb2f3 100644 --- a/pkg/multicloud/openstack/shell/instancenic.go +++ b/pkg/multicloud/openstack/shell/messages.go @@ -20,15 +20,15 @@ import ( ) func init() { - type InstanceNicListOptions struct { - Mac string `help:"Mac address for filter nics"` + type MessageListOptions struct { + ResourceId string } - shellutils.R(&InstanceNicListOptions{}, "instancenic-list", "List instance nics", func(cli *openstack.SRegion, args *InstanceNicListOptions) error { - instances, err := cli.GetPorts(args.Mac) + shellutils.R(&MessageListOptions{}, "message-list", "List messages", func(cli *openstack.SRegion, args *MessageListOptions) error { + messages, err := cli.GetMessages(args.ResourceId) if err != nil { return err } - printList(instances, 0, 0, 0, nil) + printList(messages, 0, 0, 0, []string{}) return nil }) } diff --git a/pkg/multicloud/openstack/shell/network.go b/pkg/multicloud/openstack/shell/network.go index 6d3d05403c..7024c248ab 100644 --- a/pkg/multicloud/openstack/shell/network.go +++ b/pkg/multicloud/openstack/shell/network.go @@ -44,4 +44,20 @@ func init() { return nil }) + type NetworkCreateOptions struct { + VPCID string + NAME string + CIDR string + Desc string + ProjectId string + } + + shellutils.R(&NetworkCreateOptions{}, "network-create", "Create network", func(cli *openstack.SRegion, args *NetworkCreateOptions) error { + network, err := cli.CreateNetwork(args.VPCID, args.ProjectId, args.NAME, args.CIDR, args.Desc) + if err != nil { + return err + } + printObject(network) + return nil + }) } diff --git a/pkg/multicloud/openstack/shell/port.go b/pkg/multicloud/openstack/shell/port.go index ab6123c53a..1fa39c0b48 100644 --- a/pkg/multicloud/openstack/shell/port.go +++ b/pkg/multicloud/openstack/shell/port.go @@ -21,10 +21,11 @@ import ( func init() { type PortListOptions struct { - MacAddr string + MacAddr string + DeviceId string } shellutils.R(&PortListOptions{}, "port-list", "List ports", func(cli *openstack.SRegion, args *PortListOptions) error { - ports, err := cli.GetPorts(args.MacAddr) + ports, err := cli.GetPorts(args.MacAddr, args.DeviceId) if err != nil { return err } diff --git a/pkg/multicloud/openstack/shell/project.go b/pkg/multicloud/openstack/shell/project.go index f2016b0107..194b8b78c0 100644 --- a/pkg/multicloud/openstack/shell/project.go +++ b/pkg/multicloud/openstack/shell/project.go @@ -23,7 +23,7 @@ func init() { type ProjectListOptions struct { } shellutils.R(&ProjectListOptions{}, "project-list", "List project", func(cli *openstack.SRegion, args *ProjectListOptions) error { - project, err := cli.GetClient().GetIProjects() + project, err := cli.GetClient().GetProjects() if err != nil { return err } diff --git a/pkg/multicloud/openstack/shell/securitygroup.go b/pkg/multicloud/openstack/shell/securitygroup.go index 4c1d7c2fd3..9c75d06d30 100644 --- a/pkg/multicloud/openstack/shell/securitygroup.go +++ b/pkg/multicloud/openstack/shell/securitygroup.go @@ -21,10 +21,11 @@ import ( func init() { type SecurityGroupListOptions struct { - Name string + ProjectId string + Name string } shellutils.R(&SecurityGroupListOptions{}, "security-group-list", "List security groups", func(cli *openstack.SRegion, args *SecurityGroupListOptions) error { - secgroup, err := cli.GetSecurityGroups(args.Name) + secgroup, err := cli.GetSecurityGroups(args.ProjectId, args.Name) if err != nil { return err } @@ -55,12 +56,13 @@ func init() { }) type SecurityGroupCreateOptions struct { - NAME string `help:"Name of security group"` - Desc string `help:"Description of security group"` + ProjectId string + NAME string `help:"Name of security group"` + Desc string `help:"Description of security group"` } shellutils.R(&SecurityGroupCreateOptions{}, "security-group-create", "Create security group", func(cli *openstack.SRegion, args *SecurityGroupCreateOptions) error { - secgroup, err := cli.CreateSecurityGroup(args.NAME, args.Desc) + secgroup, err := cli.CreateSecurityGroup(args.ProjectId, args.NAME, args.Desc) if err != nil { return err } diff --git a/pkg/multicloud/openstack/shell/storage.go b/pkg/multicloud/openstack/shell/storage.go index 7ea2e90530..17f2d4584e 100644 --- a/pkg/multicloud/openstack/shell/storage.go +++ b/pkg/multicloud/openstack/shell/storage.go @@ -15,23 +15,15 @@ package shell import ( - "fmt" - "yunion.io/x/onecloud/pkg/multicloud/openstack" "yunion.io/x/onecloud/pkg/util/shellutils" ) func init() { type StorageListOptions struct { - REGION string `help:"Region Name"` - ZONE string `help:"Zone Name"` } shellutils.R(&StorageListOptions{}, "storage-list", "List storages", func(cli *openstack.SRegion, args *StorageListOptions) error { - zone, err := cli.GetIZoneById(fmt.Sprintf("%s/%s/%s", openstack.CLOUD_PROVIDER_OPENSTACK, args.REGION, args.ZONE)) - if err != nil { - return err - } - storages, err := zone.GetIStorages() + storages, err := cli.GetStorageTypes() if err != nil { return err } @@ -40,17 +32,10 @@ func init() { }) type CinderServiceListOptions struct { - REGION string `help:"Region Name"` - ZONE string `help:"Zone Name"` } shellutils.R(&CinderServiceListOptions{}, "cinder-service-list", "List cinder services", func(cli *openstack.SRegion, args *CinderServiceListOptions) error { - izone, err := cli.GetIZoneById(fmt.Sprintf("%s/%s/%s", openstack.CLOUD_PROVIDER_OPENSTACK, args.REGION, args.ZONE)) - if err != nil { - return err - } - zone := izone.(*openstack.SZone) - services, err := zone.GetCinderServices() + services, err := cli.GetCinderServices() if err != nil { return err } diff --git a/pkg/multicloud/openstack/shell/vpc.go b/pkg/multicloud/openstack/shell/vpc.go index 1741cb8554..0da4516405 100644 --- a/pkg/multicloud/openstack/shell/vpc.go +++ b/pkg/multicloud/openstack/shell/vpc.go @@ -21,9 +21,10 @@ import ( func init() { type VpcListOptions struct { + TenantId string } shellutils.R(&VpcListOptions{}, "vpc-list", "List vpcs", func(cli *openstack.SRegion, args *VpcListOptions) error { - vpcs, err := cli.GetVpcs() + vpcs, err := cli.GetVpcs(args.TenantId) if err != nil { return err } @@ -31,10 +32,10 @@ func init() { return nil }) - type VpcShowOptions struct { + type VpcIdOptions struct { ID string `help:"ID of vpc"` } - shellutils.R(&VpcShowOptions{}, "vpc-show", "Show vpc", func(cli *openstack.SRegion, args *VpcShowOptions) error { + shellutils.R(&VpcIdOptions{}, "vpc-show", "Show vpc", func(cli *openstack.SRegion, args *VpcIdOptions) error { vpc, err := cli.GetVpc(args.ID) if err != nil { return err @@ -43,4 +44,21 @@ func init() { return nil }) + shellutils.R(&VpcIdOptions{}, "vpc-delete", "Delete vpc", func(cli *openstack.SRegion, args *VpcIdOptions) error { + return cli.DeleteVpc(args.ID) + }) + + type VpcCreateOptions struct { + NAME string + Desc string + } + + shellutils.R(&VpcCreateOptions{}, "vpc-create", "Create vpc", func(cli *openstack.SRegion, args *VpcCreateOptions) error { + vpc, err := cli.CreateVpc(args.NAME, args.Desc) + if err != nil { + return err + } + printObject(vpc) + return nil + }) } diff --git a/pkg/multicloud/openstack/shell/zone.go b/pkg/multicloud/openstack/shell/zone.go index 1eb9261072..92802ea9ba 100644 --- a/pkg/multicloud/openstack/shell/zone.go +++ b/pkg/multicloud/openstack/shell/zone.go @@ -23,7 +23,7 @@ func init() { type ZoneListOptions struct { } shellutils.R(&ZoneListOptions{}, "zone-list", "List zones", func(cli *openstack.SRegion, args *ZoneListOptions) error { - zones, err := cli.GetIZones() + zones, err := cli.GetZones() if err != nil { return err } diff --git a/pkg/multicloud/openstack/snapshot.go b/pkg/multicloud/openstack/snapshot.go index a4201f6246..4164f73bd2 100644 --- a/pkg/multicloud/openstack/snapshot.go +++ b/pkg/multicloud/openstack/snapshot.go @@ -15,7 +15,11 @@ package openstack import ( + "net/url" + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -36,16 +40,16 @@ const ( type SSnapshot struct { region *SRegion - ID string - VolumeID string + Id string + VolumeId string Status string Metadata Metadata Progress string `json:"os-extended-snapshot-attributes:progress"` Name string - UserID string - ProjectID string `json:"os-extended-snapshot-attributes:project_id"` + UserId string + ProjectId string `json:"os-extended-snapshot-attributes:project_id"` //CreatedAt time.Time Size int32 @@ -54,13 +58,15 @@ type SSnapshot struct { } func (region *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) { - _, resp, err := region.CinderGet("/snapshots/"+snapshotId, "", nil) + resource := "/snapshots/" + snapshotId + resp, err := region.bsGet(resource) if err != nil { - return nil, err + return nil, errors.Wrap(err, "bsGet") } snapshot := SSnapshot{region: region} - if err := resp.Unmarshal(&snapshot, "snapshot"); err != nil { - return nil, err + err = resp.Unmarshal(&snapshot, "snapshot") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") } return &snapshot, nil } @@ -85,34 +91,56 @@ func (snapshot *SSnapshot) IsEmulated() bool { } func (snapshot *SSnapshot) Refresh() error { - _snapshot, err := snapshot.region.GetISnapshotById(snapshot.ID) + _snapshot, err := snapshot.region.GetISnapshotById(snapshot.Id) if err != nil { return err } return jsonutils.Update(snapshot, _snapshot) } -func (region *SRegion) GetSnapshots(diskId string) ([]cloudprovider.ICloudSnapshot, error) { - _, resp, err := region.CinderList("/snapshots/detail", "", nil) - if err != nil { - return nil, err +func (region *SRegion) GetSnapshots(diskId string) ([]SSnapshot, error) { + resource := "/snapshots/detail" + query := url.Values{} + query.Set("all_tenants", "true") + if len(diskId) > 0 { + query.Set("volume_id", diskId) } snapshots := []SSnapshot{} - if err := resp.Unmarshal(&snapshots, "snapshots"); err != nil { - return nil, err - } - iSnapshots := []cloudprovider.ICloudSnapshot{} - for i := 0; i < len(snapshots); i++ { - if len(diskId) == 0 || snapshots[i].VolumeID == diskId { - snapshots[i].region = region - iSnapshots = append(iSnapshots, &snapshots[i]) + for { + resp, err := region.bsList(resource, query) + if err != nil { + return nil, errors.Wrap(err, "bsList") } + part := struct { + Snapshots []SSnapshot + SnapshotsLinks SNextLinks + }{} + + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + snapshots = append(snapshots, part.Snapshots...) + marker := part.SnapshotsLinks.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) } - return iSnapshots, nil + return snapshots, nil } func (region *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) { - return region.GetSnapshots("") + snapshots, err := region.GetSnapshots("") + if err != nil { + return nil, err + } + isnapshots := []cloudprovider.ICloudSnapshot{} + for i := range snapshots { + snapshots[i].region = region + isnapshots = append(isnapshots, &snapshots[i]) + } + return isnapshots, nil } func (snapshot *SSnapshot) GetMetadata() *jsonutils.JSONDict { @@ -124,41 +152,43 @@ func (snapshot *SSnapshot) GetSizeMb() int32 { } func (snapshot *SSnapshot) GetDiskId() string { - return snapshot.VolumeID + return snapshot.VolumeId } func (snapshot *SSnapshot) GetId() string { - return snapshot.ID + return snapshot.Id } func (snapshot *SSnapshot) GetGlobalId() string { - return snapshot.ID + return snapshot.Id } func (snapshot *SSnapshot) GetName() string { if len(snapshot.Name) == 0 { - return snapshot.ID + return snapshot.Id } return snapshot.Name } func (snapshot *SSnapshot) Delete() error { - return snapshot.region.DeleteSnapshot(snapshot.ID) + return snapshot.region.DeleteSnapshot(snapshot.Id) } func (snapshot *SSnapshot) GetDiskType() string { - if len(snapshot.VolumeID) > 0 { - if disk, err := snapshot.region.GetDisk(snapshot.VolumeID); err == nil { - if disk.Bootable { - return api.DISK_TYPE_SYS - } + if len(snapshot.VolumeId) > 0 { + disk, err := snapshot.region.GetDisk(snapshot.VolumeId) + if err != nil { + log.Errorf("failed to get snapshot %s disk %s error: %v", snapshot.Name, snapshot.VolumeId, err) + return api.DISK_TYPE_DATA } + return disk.GetDiskType() } return api.DISK_TYPE_DATA } func (region *SRegion) DeleteSnapshot(snapshotId string) error { - _, err := region.CinderDelete("/snapshots/"+snapshotId, "") + resource := "/snapshots/" + snapshotId + _, err := region.bsDelete(resource) return err } @@ -171,14 +201,18 @@ func (region *SRegion) CreateSnapshot(diskId, name, desc string) (*SSnapshot, er "force": true, }, } - _, resp, err := region.CinderCreate("", "/snapshots", "", jsonutils.Marshal(params)) + resp, err := region.bsPost("/snapshots", params) if err != nil { - return nil, err + return nil, errors.Wrap(err, "bsPost") } snapshot := &SSnapshot{region: region} - return snapshot, resp.Unmarshal(snapshot, "snapshot") + err = resp.Unmarshal(snapshot, "snapshot") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return snapshot, nil } func (self *SSnapshot) GetProjectId() string { - return self.ProjectID + return self.ProjectId } diff --git a/pkg/multicloud/openstack/storage.go b/pkg/multicloud/openstack/storage.go index ea6ebe81c5..7a0137e109 100644 --- a/pkg/multicloud/openstack/storage.go +++ b/pkg/multicloud/openstack/storage.go @@ -15,11 +15,13 @@ package openstack import ( + "net/url" "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -66,21 +68,23 @@ func (storage *SStorage) GetIZone() cloudprovider.ICloudZone { } func (storage *SStorage) GetIDisks() ([]cloudprovider.ICloudDisk, error) { - disks, err := storage.zone.region.GetDisks(storage.Name, storage.ExtraSpecs.VolumeBackendName) + disks, err := storage.zone.region.GetDisks() if err != nil { return nil, err } - iDisks := []cloudprovider.ICloudDisk{} + idisks := []cloudprovider.ICloudDisk{} for i := 0; i < len(disks); i++ { - disks[i].storage = storage - iDisks = append(iDisks, &disks[i]) + if disks[i].VolumeType == storage.Name || strings.HasSuffix(disks[i].Host, "#"+storage.ExtraSpecs.VolumeBackendName) { + disks[i].storage = storage + idisks = append(idisks, &disks[i]) + } } - return iDisks, nil + return idisks, nil } func (storage *SStorage) GetStorageType() string { if len(storage.ExtraSpecs.VolumeBackendName) == 0 { - storage.ExtraSpecs.VolumeBackendName = DEFAULT_STORAGE_TYPE + return DEFAULT_STORAGE_TYPE } return storage.ExtraSpecs.VolumeBackendName } @@ -102,10 +106,11 @@ func (storage *SStorage) GetStorageConf() jsonutils.JSONObject { } func (storage *SStorage) GetStatus() string { - if utils.IsInStringArray(storage.GetStorageType(), storage.zone.getAvailableStorages()) { - return api.STORAGE_ONLINE + ok, err := storage.zone.region.IsStorageAvailable(storage.GetStorageType()) + if err != nil || !ok { + return api.STORAGE_OFFLINE } - return api.STORAGE_OFFLINE + return api.STORAGE_ONLINE } func (storage *SStorage) Refresh() error { @@ -147,3 +152,100 @@ func (storage *SStorage) GetMountPoint() string { func (storage *SStorage) IsSysDiskStore() bool { return true } + +func (region *SRegion) GetStorageTypes() ([]SStorage, error) { + resource := "/types" + storages := []SStorage{} + query := url.Values{} + for { + resp, err := region.bsList(resource, query) + if err != nil { + return nil, errors.Wrap(err, "bsReqest") + } + part := struct { + VolumeTypes []SStorage + VolumeTypeLinks SNextLinks + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + storages = append(storages, part.VolumeTypes...) + marker := part.VolumeTypeLinks.GetNextMark() + if len(marker) == 0 { + break + } + query.Set("marker", marker) + } + return storages, nil +} + +type SCinderService struct { + ActiveBackendId string + // cinder-volume + Binary string + DisabledReason string + Frozen string + Host string + ReplicationStatus string + State string + Status string + UpdatedAt time.Time + Zone string +} + +func (region *SRegion) GetCinderServices() ([]SCinderService, error) { + resp, err := region.bsList("/os-services", nil) + if err != nil { + return nil, errors.Wrap(err, "bsList") + } + services := []SCinderService{} + err = resp.Unmarshal(&services, "services") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return services, nil +} + +func (region *SRegion) IsStorageAvailable(storageType string) (bool, error) { + if utils.IsInStringArray(storageType, []string{DEFAULT_STORAGE_TYPE, api.STORAGE_OPENSTACK_NOVA}) { + return true, nil + } + services, err := region.GetCinderServices() + if err != nil { + return false, errors.Wrap(err, "GetCinderServices") + } + for _, service := range services { + if service.Binary == "cinder-volume" && strings.Contains(service.Host, "@") { + hostInfo := strings.Split(service.Host, "@") + if hostInfo[len(hostInfo)-1] == storageType { + if service.State == "up" && service.Status == "enabled" { + return true, nil + } + } + } + } + log.Errorf("storage %s offline", storageType) + return false, nil +} + +type SCapabilities struct { +} + +type SPool struct { + Name string + Capabilities SCapabilities +} + +func (region *SRegion) GetSchedulerStatsPool() ([]SPool, error) { + resp, err := region.bsList("/scheduler-stats/get_pools", nil) + if err != nil { + return nil, errors.Wrap(err, "bsList") + } + pools := []SPool{} + err = resp.Unmarshal(&pools, "pools") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return pools, nil +} diff --git a/pkg/multicloud/openstack/storagecache.go b/pkg/multicloud/openstack/storagecache.go index 9cf3ec1d9c..12441939fe 100644 --- a/pkg/multicloud/openstack/storagecache.go +++ b/pkg/multicloud/openstack/storagecache.go @@ -151,18 +151,14 @@ func (cache *SStoragecache) uploadImage(ctx context.Context, userCred mcclient.T minDiskSizeGB += 1 } - img, err := cache.region.CreateImage(imageName, osType, osDist, int(minDiskSizeGB), int(minRamMb)) + img, err := cache.region.CreateImage(imageName, osType, osDist, int(minDiskSizeGB), int(minRamMb), reader) if err != nil { return "", err } img.storageCache = cache - _, err = cache.region.client.StreamRequest(cache.region.Name, "image", "PUT", fmt.Sprintf("/v2/images/%s/file", img.ID), "", reader) - if err != nil { - return "", err - } - return img.ID, cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second) + return img.Id, cloudprovider.WaitStatus(img, api.CACHED_IMAGE_STATUS_READY, 15*time.Second, 3600*time.Second) } func (cache *SStoragecache) CreateIImage(snapshoutId, imageName, osType, imageDesc string) (cloudprovider.ICloudImage, error) { diff --git a/pkg/multicloud/openstack/vpc.go b/pkg/multicloud/openstack/vpc.go index 7f86895925..442fb302c1 100644 --- a/pkg/multicloud/openstack/vpc.go +++ b/pkg/multicloud/openstack/vpc.go @@ -15,6 +15,8 @@ package openstack import ( + "fmt" + "net/url" "time" "yunion.io/x/jsonutils" @@ -34,18 +36,14 @@ const ( type SVpc struct { multicloud.SVpc - region *SRegion - iwires []cloudprovider.ICloudWire - secgroups []cloudprovider.ICloudSecurityGroup - AdminStateUp bool AvailabilityZoneHints []string AvailabilityZones []string CreatedAt time.Time DnsDomain string - ID string + Id string Ipv4AddressScope string Ipv6AddressScope string L2Adjacency bool @@ -69,23 +67,19 @@ type SVpc struct { SegmentationId string `json:"provider:segmentation_id"` } -func (vpc *SVpc) GetMetadata() *jsonutils.JSONDict { - return nil -} - func (vpc *SVpc) GetId() string { - return vpc.ID + return vpc.Id } func (vpc *SVpc) GetName() string { if len(vpc.Name) > 0 { return vpc.Name } - return vpc.ID + return vpc.Id } func (vpc *SVpc) GetGlobalId() string { - return vpc.ID + return vpc.Id } func (vpc *SVpc) IsEmulated() bool { @@ -114,23 +108,30 @@ func (vpc *SVpc) GetStatus() string { } func (vpc *SVpc) Delete() error { - return vpc.region.DeleteVpc(vpc.ID) + return vpc.region.DeleteVpc(vpc.Id) } func (region *SRegion) DeleteVpc(vpcId string) error { - _, err := region.Delete("network", "/v2.0/networks/"+vpcId, "") + resource := fmt.Sprintf("/v2.0/networks/%s", vpcId) + _, err := region.vpcDelete(resource) return err } func (vpc *SVpc) GetISecurityGroups() ([]cloudprovider.ICloudSecurityGroup, error) { - secgroups, err := vpc.region.GetSecurityGroups("") + err := vpc.region.client.fetchProjects() if err != nil { - return nil, err + return nil, errors.Wrap(err, "fetchProjects") } iSecgroups := []cloudprovider.ICloudSecurityGroup{} - for i := 0; i < len(secgroups); i++ { - secgroups[i].region = vpc.region - iSecgroups = append(iSecgroups, &secgroups[i]) + for _, project := range vpc.region.client.projects { + secgroups, err := vpc.region.GetSecurityGroups(project.Id, "") + if err != nil { + return nil, errors.Wrapf(err, "GetSecurityGroups(%s)", project.Id) + } + for i := 0; i < len(secgroups); i++ { + secgroups[i].region = vpc.region + iSecgroups = append(iSecgroups, &secgroups[i]) + } } return iSecgroups, nil } @@ -166,60 +167,30 @@ func (vpc *SVpc) GetIRouteTables() ([]cloudprovider.ICloudRouteTable, error) { return ret, nil } -func (vpc *SVpc) fetchWires() error { - if len(vpc.region.izones) == 0 { - if err := vpc.region.fetchZones(); err != nil { - return err - } - } - wire := SWire{zone: vpc.region.izones[0].(*SZone), vpc: vpc} - vpc.iwires = []cloudprovider.ICloudWire{&wire} - return nil -} - -func (vpc *SVpc) getWire() *SWire { - if vpc.iwires == nil { - vpc.fetchWires() - } - return vpc.iwires[0].(*SWire) -} - -func (vpc *SVpc) fetchNetworks() error { - networks, err := vpc.region.GetNetworks(vpc.ID) - if err != nil { - return err - } - for i := 0; i < len(networks); i++ { - wire := vpc.getWire() - networks[i].wire = wire - wire.addNetwork(&networks[i]) - } - return nil -} - func (vpc *SVpc) GetIWireById(wireId string) (cloudprovider.ICloudWire, error) { - if vpc.iwires == nil { - err := vpc.fetchNetworks() - if err != nil { - return nil, err - } + iwires, err := vpc.GetIWires() + if err != nil { + return nil, errors.Wrap(err, "GetIWires") } - for i := 0; i < len(vpc.iwires); i++ { - if vpc.iwires[i].GetGlobalId() == wireId { - return vpc.iwires[i], nil + for i := range iwires { + if iwires[i].GetGlobalId() == wireId { + return iwires[i], nil } } return nil, cloudprovider.ErrNotFound } func (vpc *SVpc) GetIWires() ([]cloudprovider.ICloudWire, error) { - if vpc.iwires == nil { - err := vpc.fetchNetworks() - if err != nil { - return nil, err - } + err := vpc.region.fetchZones() + if err != nil { + return nil, errors.Wrap(err, "fetchZones") } - return vpc.iwires, nil + iwire := []cloudprovider.ICloudWire{} + for i := range vpc.region.zones { + wire := &SWire{vpc: vpc, zone: &vpc.region.zones[i]} + iwire = append(iwire, wire) + } + return iwire, nil } func (vpc *SVpc) GetRegion() cloudprovider.ICloudRegion { @@ -227,57 +198,75 @@ func (vpc *SVpc) GetRegion() cloudprovider.ICloudRegion { } func (region *SRegion) GetVpc(vpcId string) (*SVpc, error) { - _, resp, err := region.Get("network", "/v2.0/networks/"+vpcId, "", nil) + vpc := &SVpc{region: region} + resource := fmt.Sprintf("/v2.0/networks/%s", vpcId) + resp, err := region.vpcGet(resource) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "vpcGet(%s)", resource) } - vpc := SVpc{} - return &vpc, resp.Unmarshal(&vpc, "network") + err = resp.Unmarshal(vpc, "network") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return vpc, nil } -func (region *SRegion) GetVpcs() ([]SVpc, error) { - url := "/v2.0/networks" +func (region *SRegion) GetVpcs(projectId string) ([]SVpc, error) { vpcs := []SVpc{} - for len(url) > 0 { - _, resp, err := region.List("network", url, "", nil) + resource := "/v2.0/networks" + query := url.Values{} + if len(projectId) > 0 { + query.Set("tenant_id", projectId) + } + for { + resp, err := region.vpcList(resource, query) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "vpcList.%s", resource) } - _vpcs := []SVpc{} - err = resp.Unmarshal(&_vpcs, "networks") + + part := struct { + Networks []SVpc + NetworksLinks SNextLinks + }{} + + err = resp.Unmarshal(&part) if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&_vpcs, "networks")`) + return nil, errors.Wrap(err, "resp.Unmarshal") } - vpcs = append(vpcs, _vpcs...) - url = "" - if resp.Contains("networks_links") { - nextLink := []SNextLink{} - err = resp.Unmarshal(&nextLink, "networks_links") - if err != nil { - return nil, errors.Wrap(err, `resp.Unmarshal(&nextLink, "networks_links")`) - } - for _, next := range nextLink { - if next.Rel == "next" { - url = next.Href - break - } - } + vpcs = append(vpcs, part.Networks...) + marker := part.NetworksLinks.GetNextMark() + if len(marker) == 0 { + break } + query.Set("marker", marker) } return vpcs, nil } func (vpc *SVpc) Refresh() error { - new, err := vpc.region.GetVpc(vpc.ID) + _vpc, err := vpc.region.GetVpc(vpc.Id) if err != nil { - return err + return errors.Wrapf(err, "GetVpc(%s)", vpc.Id) } - return jsonutils.Update(vpc, new) + return jsonutils.Update(vpc, _vpc) } -func (vpc *SVpc) addWire(wire *SWire) { - if vpc.iwires == nil { - vpc.iwires = make([]cloudprovider.ICloudWire, 0) +func (region *SRegion) CreateVpc(name, desc string) (*SVpc, error) { + params := map[string]map[string]string{ + "network": { + "name": name, + "description": desc, + }, } - vpc.iwires = append(vpc.iwires, wire) + resource := "/v2.0/networks" + resp, err := region.vpcPost(resource, params) + if err != nil { + return nil, errors.Wrap(err, "vpcPost") + } + vpc := &SVpc{region: region} + err = resp.Unmarshal(vpc, "network") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return vpc, nil } diff --git a/pkg/multicloud/openstack/wire.go b/pkg/multicloud/openstack/wire.go index 34646567a5..314ac9c0d5 100644 --- a/pkg/multicloud/openstack/wire.go +++ b/pkg/multicloud/openstack/wire.go @@ -18,7 +18,7 @@ import ( "fmt" "yunion.io/x/jsonutils" - "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -26,8 +26,6 @@ import ( type SWire struct { zone *SZone vpc *SVpc - - inetworks []cloudprovider.ICloudNetwork } func (wire *SWire) GetMetadata() *jsonutils.JSONDict { @@ -70,14 +68,13 @@ func (wire *SWire) GetBandwidth() int { return 10000 } -func (wire *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - networkId, err := wire.zone.region.CreateNetwork(wire.vpc.ID, name, cidr, desc) +func (wire *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + network, err := wire.zone.region.CreateNetwork(wire.vpc.Id, opts.ProjectId, opts.Name, opts.Cidr, opts.Desc) if err != nil { - log.Errorf("CreateNetwork error %s", err) - return nil, err + return nil, errors.Wrap(err, "CreateNetwork") } - wire.inetworks = nil - return wire.GetINetworkById(networkId) + network.wire = wire + return network, nil } func (wire *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, error) { @@ -94,27 +91,14 @@ func (wire *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, e } func (wire *SWire) GetINetworks() ([]cloudprovider.ICloudNetwork, error) { - if wire.inetworks == nil { - err := wire.vpc.fetchNetworks() - if err != nil { - return nil, err - } - } - return wire.inetworks, nil -} - -func (wire *SWire) addNetwork(network *SNetwork) { - if wire.inetworks == nil { - wire.inetworks = []cloudprovider.ICloudNetwork{} - } - find := false - for i := 0; i < len(wire.inetworks); i++ { - if wire.inetworks[i].GetGlobalId() == network.GetGlobalId() { - find = true - break - } - } - if !find { - wire.inetworks = append(wire.inetworks, network) + networks, err := wire.vpc.region.GetNetworks(wire.vpc.Id) + if err != nil { + return nil, errors.Wrapf(err, "GetNetworks(%s)", wire.vpc.Id) } + inetworks := []cloudprovider.ICloudNetwork{} + for i := range networks { + networks[i].wire = wire + inetworks = append(inetworks, &networks[i]) + } + return inetworks, nil } diff --git a/pkg/multicloud/openstack/zone.go b/pkg/multicloud/openstack/zone.go index 0236be3c81..f8327ad51f 100644 --- a/pkg/multicloud/openstack/zone.go +++ b/pkg/multicloud/openstack/zone.go @@ -20,7 +20,6 @@ import ( "time" "yunion.io/x/jsonutils" - "yunion.io/x/log" "yunion.io/x/pkg/errors" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -31,18 +30,6 @@ type ZoneState struct { Available bool } -const ( - HYPERVISORS_VERSION = "2.28" -) - -type SCapabilities struct { -} - -type SPool struct { - Name string - Capabilities SCapabilities -} - type HostState struct { Available bool Active bool @@ -52,18 +39,9 @@ type HostState struct { type SZone struct { region *SRegion - iwires []cloudprovider.ICloudWire - istorages []cloudprovider.ICloudStorage - ZoneName string - cachedHosts map[string][]string - - schedulerPools []SPool - availableStorages []string - unavailableStorages []string - - Hosts map[string]map[string]HostState + hosts []SHypervisor } func (zone *SZone) GetMetadata() *jsonutils.JSONDict { @@ -100,228 +78,129 @@ func (zone *SZone) GetIRegion() cloudprovider.ICloudRegion { } func (zone *SZone) GetIWires() ([]cloudprovider.ICloudWire, error) { - return zone.iwires, nil -} - -func (zone *SZone) fetchSchedulerStatsPool() error { - var err error = nil - zone.schedulerPools, err = zone.region.GetSchedulerStatsPool() + err := zone.region.fetchVpcs() if err != nil { - return errors.Wrap(err, "GetSchedulerStatsPool") + return nil, errors.Wrap(err, "fetchVpcs") } - return nil + iwires := []cloudprovider.ICloudWire{} + for i := range zone.region.vpcs { + wire := &SWire{zone: zone, vpc: &zone.region.vpcs[i]} + iwires = append(iwires, wire) + } + return iwires, nil } -func (region *SRegion) GetSchedulerStatsPool() ([]SPool, error) { - pools := []SPool{} - for _, service := range []string{"volumev3", "volumev2", "volume"} { - _, resp, err := region.List(service, "/scheduler-stats/get_pools", "", nil) - if err == nil { - if err := resp.Unmarshal(&pools, "pools"); err != nil { - return nil, errors.Wrap(err, "resp.Unmarshal(&pools)") - } - return pools, nil - } - } - return nil, fmt.Errorf("failed to find scheduler-stats pool by cinder service") -} - -func (zone *SZone) getSchedulerStatsPool() ([]SPool, error) { - if len(zone.schedulerPools) == 0 { - return zone.schedulerPools, zone.fetchSchedulerStatsPool() - } - return zone.schedulerPools, nil -} - -func (zone *SZone) getStorageByCategory(category string) (*SStorage, error) { - storages, err := zone.GetIStorages() +func (zone *SZone) getStorageByCategory(category, host string) (*SStorage, error) { + storages, err := zone.region.GetStorageTypes() if err != nil { - return nil, err + return nil, errors.Wrap(err, "GetStorageTypes") } - for i := 0; i < len(storages); i++ { - storage, ok := storages[i].(*SStorage) - if ok && strings.ToLower(storage.Name) == strings.ToLower(category) { - return storage, nil + for i := range storages { + if storages[i].Name == category || storages[i].ExtraSpecs.VolumeBackendName == category { + storages[i].zone = zone + return &storages[i], nil } } - return nil, fmt.Errorf("No such storage %s", category) -} - -func (zone *SZone) addWire(wire *SWire) { - if zone.iwires == nil { - zone.iwires = []cloudprovider.ICloudWire{} - } - zone.iwires = append(zone.iwires, wire) -} - -func (zone *SZone) fetchStorages() error { - zone.istorages = []cloudprovider.ICloudStorage{} - - for _, service := range []string{"volumev3", "volumev2", "volume"} { - _, resp, err := zone.region.List(service, "/types", "", nil) - if err == nil { - storages := []SStorage{} - if err := resp.Unmarshal(&storages, "volume_types"); err != nil { - return err - } - for i := 0; i < len(storages); i++ { - storages[i].zone = zone - zone.istorages = append(zone.istorages, &storages[i]) - } - break - } - log.Debugf("failed to get volume types by service %s error: %v, try another", service, err) - } - nova := &SNovaStorage{zone: zone} - zone.istorages = append(zone.istorages, nova) - return fmt.Errorf("failed to find storage types by cinder service") -} - -type SCinderService struct { - ActiveBackendId string - // cinder-volume - Binary string - DisabledReason string - Frozen string - Host string - ReplicationStatus string - State string - Status string - UpdatedAt time.Time - Zone string -} - -func (zone *SZone) GetCinderServices() ([]SCinderService, error) { - _, resp, err := zone.region.CinderList("/os-services", "", nil) - if err != nil { - return nil, errors.Wrap(err, "CinderList") - } - services := []SCinderService{} - err = resp.Unmarshal(&services, "services") - if err != nil { - return nil, errors.Wrap(err, "resp.Unmarshal") - } - return services, nil -} - -func (zone *SZone) fetchCinderSerivces() error { - services, err := zone.GetCinderServices() - if err != nil { - return errors.Wrap(err, "GetCinderServices") - } - zone.availableStorages = []string{DEFAULT_STORAGE_TYPE, api.STORAGE_OPENSTACK_NOVA} - zone.unavailableStorages = []string{} - for _, service := range services { - if service.Binary == "cinder-volume" && strings.Contains(service.Host, "@") { - hostInfo := strings.Split(service.Host, "@") - storage := hostInfo[len(hostInfo)-1] - if service.State == "up" && service.Status == "enabled" { - zone.availableStorages = append(zone.availableStorages, storage) - continue - } - zone.unavailableStorages = append(zone.unavailableStorages, storage) + for i := range storages { + if strings.HasSuffix(host, "#"+storages[i].Name) || strings.HasSuffix(host, "#"+storages[i].ExtraSpecs.VolumeBackendName) { + storages[i].zone = zone + return &storages[i], nil } } - return nil -} - -func (zone *SZone) getAvailableStorages() []string { - if zone.availableStorages == nil { - err := zone.fetchCinderSerivces() - if err != nil { - log.Errorf("fetchCinderSerivces error: %v", err) - } - } - return zone.availableStorages -} - -func (zone *SZone) getUnavailableStorage() []string { - if zone.unavailableStorages == nil { - err := zone.fetchCinderSerivces() - if err != nil { - log.Errorf("fetchCinderSerivces error: %v", err) - } - } - return zone.unavailableStorages + return nil, fmt.Errorf("No such storage [%s]", category) } func (zone *SZone) GetIStorages() ([]cloudprovider.ICloudStorage, error) { - if zone.istorages == nil { - zone.fetchStorages() + storages, err := zone.region.GetStorageTypes() + if err != nil { + return nil, errors.Wrap(err, "GetStorageTypes") } - return zone.istorages, nil + istorages := []cloudprovider.ICloudStorage{} + for i := range storages { + storages[i].zone = zone + istorages = append(istorages, &storages[i]) + } + err = zone.fetchHosts() + if err != nil { + return nil, errors.Wrap(err, "fetchHosts") + } + for i := range zone.hosts { + nova := &SNovaStorage{host: &zone.hosts[i], zone: zone} + istorages = append(istorages, nova) + } + return istorages, nil } func (zone *SZone) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { - if zone.istorages == nil { - zone.fetchStorages() + istorages, err := zone.GetIStorages() + if err != nil { + return nil, errors.Wrap(err, "GetIStorages") } - for i := 0; i < len(zone.istorages); i++ { - if zone.istorages[i].GetGlobalId() == id { - return zone.istorages[i], nil + for i := 0; i < len(istorages); i++ { + if istorages[i].GetGlobalId() == id { + return istorages[i], nil } } return nil, cloudprovider.ErrNotFound } -type SOsHost struct { - Zone string - HostName string - Service string +func (zone *SZone) fetchHosts() error { + if len(zone.hosts) > 0 { + return nil + } + + zone.hosts = []SHypervisor{} + hypervisors, err := zone.region.GetHypervisors() + if err != nil { + return errors.Wrap(err, "GetHypervisors") + } + for i := range hypervisors { + hypervisor := strings.ToLower(hypervisors[i].HypervisorType) + // 过滤vmware的机器 + if strings.Index(hypervisor, "vmware") != -1 { + continue + } + zone.hosts = append(zone.hosts, hypervisors[i]) + } + return nil + } func (zone *SZone) GetIHosts() ([]cloudprovider.ICloudHost, error) { - ihosts := []cloudprovider.ICloudHost{} - hosts := []SHost{} - - // 尽可能的优先使用 os-hypervisor, 里面的信息更全些, 实在不行再使用 os-host - _, resp, err := zone.region.List("compute", "/os-hypervisors/detail", "", nil) - if err == nil { - if err := resp.Unmarshal(&hosts, "hypervisors"); err != nil { - return nil, err - } - for i := 0; i < len(hosts); i++ { - // 过滤vmware的机器 - hypervisor := strings.ToLower(hosts[i].HypervisorType) - if strings.Index(hypervisor, "vmware") != -1 { - continue - } - hosts[i].zone = zone - ihosts = append(ihosts, &hosts[i]) - } - return ihosts, nil - } - _, resp, err = zone.region.List("compute", "/os-hosts", "", nil) + err := zone.fetchHosts() if err != nil { - return nil, err + return nil, errors.Wrap(err, "fetchHosts") } - - _hosts := []SOsHost{} - - if err := resp.Unmarshal(&_hosts, "hosts"); err != nil { - return nil, err - } - for i := 0; i < len(_hosts); i++ { - if _hosts[i].Service == "compute" { - host := SHost{HostName: _hosts[i].HostName, Zone: _hosts[i].Zone, zone: zone} - ihosts = append(ihosts, &host) - } + ihosts := []cloudprovider.ICloudHost{} + for i := range zone.hosts { + zone.hosts[i].zone = zone + ihosts = append(ihosts, &zone.hosts[i]) } return ihosts, nil } func (zone *SZone) GetIHostById(id string) (cloudprovider.ICloudHost, error) { - host := &SHost{zone: zone} - _, resp, err := zone.region.Get("compute", "/os-hypervisors/"+id, "", nil) - if err == nil { - return host, resp.Unmarshal(&host, "hypervisor") - } - - host.HostName = id - host.Resource = []map[string]SResource{} - _, resp, err = zone.region.Get("compute", "/os-hosts/"+id, "", nil) + ihosts, err := zone.GetIHosts() if err != nil { - return nil, err + return nil, errors.Wrap(err, "GetIHosts") } - return host, resp.Unmarshal(&(host.Resource), "host") + for i := range ihosts { + if ihosts[i].GetGlobalId() == id { + return ihosts[i], nil + } + } + return nil, cloudprovider.ErrNotFound +} + +func (region *SRegion) GetZones() ([]SZone, error) { + zones := []SZone{} + resp, err := region.ecsList("os-availability-zone/detail", nil) + if err != nil { + return nil, errors.Wrap(err, "ecsList.os-availability-zone") + } + err = resp.Unmarshal(&zones, "availabilityZoneInfo") + if err != nil { + return nil, errors.Wrap(err, "resp.Unmarshal") + } + return zones, nil } diff --git a/pkg/multicloud/qcloud/region.go b/pkg/multicloud/qcloud/region.go index f96c217b1b..0ed8862dbc 100644 --- a/pkg/multicloud/qcloud/region.go +++ b/pkg/multicloud/qcloud/region.go @@ -934,8 +934,8 @@ func (self *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICl return self.GetSecurityGroupDetails(secgroupId) } -func (self *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, total, err := self.GetSecurityGroups(vpcId, name, 0, 0) +func (self *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, total, err := self.GetSecurityGroups(opts.VpcId, opts.Name, 0, 0) if err != nil { return nil, err } diff --git a/pkg/multicloud/qcloud/wire.go b/pkg/multicloud/qcloud/wire.go index f154c23c63..8407a9623d 100644 --- a/pkg/multicloud/qcloud/wire.go +++ b/pkg/multicloud/qcloud/wire.go @@ -86,8 +86,8 @@ func (self *SWire) GetNetworkById(networkId string) *SNetwork { return nil } -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - networkId, err := self.zone.region.CreateNetwork(self.zone.Zone, self.vpc.VpcId, name, cidr, desc) +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + networkId, err := self.zone.region.CreateNetwork(self.zone.Zone, self.vpc.VpcId, opts.Name, opts.Cidr, opts.Desc) if err != nil { log.Errorf("CreateNetwork error %s", err) return nil, err diff --git a/pkg/multicloud/ucloud/region.go b/pkg/multicloud/ucloud/region.go index 7707c4e278..32e470fe03 100644 --- a/pkg/multicloud/ucloud/region.go +++ b/pkg/multicloud/ucloud/region.go @@ -200,8 +200,8 @@ func (self *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.ICl return self.GetSecurityGroupById(secgroupId) } -func (self *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, err := self.GetSecurityGroups("", "", name) +func (self *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, err := self.GetSecurityGroups("", "", opts.Name) if err != nil { return nil, err } diff --git a/pkg/multicloud/ucloud/wire.go b/pkg/multicloud/ucloud/wire.go index e9a6e88ffc..43cd735066 100644 --- a/pkg/multicloud/ucloud/wire.go +++ b/pkg/multicloud/ucloud/wire.go @@ -94,8 +94,8 @@ func (self *SWire) GetINetworkById(netid string) (cloudprovider.ICloudNetwork, e } // https://docs.ucloud.cn/api/vpc2.0-api/create_subnet -func (self *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - return self.region.CreateNetwork(self.vpc.GetId(), name, cidr, desc) +func (self *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + return self.region.CreateNetwork(self.vpc.GetId(), opts.Name, opts.Cidr, opts.Desc) } func (self *SWire) addNetwork(network *SNetwork) { diff --git a/pkg/multicloud/zstack/network.go b/pkg/multicloud/zstack/network.go index ad1755efd6..9021c8279a 100644 --- a/pkg/multicloud/zstack/network.go +++ b/pkg/multicloud/zstack/network.go @@ -199,12 +199,11 @@ func (network *SNetwork) GetIpMask() int8 { } func (network *SNetwork) GetIsPublic() bool { - // return network.IsDefault - return false + return true } func (self *SNetwork) GetPublicScope() rbacutils.TRbacScope { - return rbacutils.ScopeProject + return rbacutils.ScopeSystem } func (network *SNetwork) GetServerType() string { diff --git a/pkg/multicloud/zstack/region.go b/pkg/multicloud/zstack/region.go index de18d35001..ea08900e22 100644 --- a/pkg/multicloud/zstack/region.go +++ b/pkg/multicloud/zstack/region.go @@ -362,8 +362,8 @@ func (region *SRegion) GetISecurityGroupById(secgroupId string) (cloudprovider.I return region.GetSecurityGroup(secgroupId) } -func (region *SRegion) GetISecurityGroupByName(vpcId string, name string) (cloudprovider.ICloudSecurityGroup, error) { - secgroups, err := region.GetSecurityGroups("", "", name) +func (region *SRegion) GetISecurityGroupByName(opts *cloudprovider.SecurityGroupFilterOptions) (cloudprovider.ICloudSecurityGroup, error) { + secgroups, err := region.GetSecurityGroups("", "", opts.Name) if err != nil { return nil, err } diff --git a/pkg/multicloud/zstack/wire.go b/pkg/multicloud/zstack/wire.go index 1ffb989126..cf5a843392 100644 --- a/pkg/multicloud/zstack/wire.go +++ b/pkg/multicloud/zstack/wire.go @@ -126,8 +126,8 @@ func (wire *SWire) GetBandwidth() int { return 10000 } -func (wire *SWire) CreateINetwork(name string, cidr string, desc string) (cloudprovider.ICloudNetwork, error) { - network, err := wire.vpc.region.CreateNetwork(name, cidr, wire.UUID, desc) +func (wire *SWire) CreateINetwork(opts *cloudprovider.SNetworkCreateOptions) (cloudprovider.ICloudNetwork, error) { + network, err := wire.vpc.region.CreateNetwork(opts.Name, opts.Cidr, wire.UUID, opts.Desc) if err != nil { return nil, err }