From 1eaf06d0516963144d354430258e12c1823f2545 Mon Sep 17 00:00:00 2001 From: tangbin Date: Wed, 28 Oct 2020 07:56:49 +0800 Subject: [PATCH 1/2] sync elb support classic vpc --- pkg/compute/models/loadbalancers.go | 69 ++++++++++++++++++++- pkg/compute/regiondrivers/huawei.go | 2 +- pkg/compute/regiondrivers/managedvirtual.go | 2 +- pkg/compute/regiondrivers/openstack.go | 2 +- pkg/multicloud/qcloud/loadbalancer.go | 10 ++- 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index 91ec3a2939..9abbdc3d87 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -803,6 +803,53 @@ func (man *SLoadbalancerManager) getLoadbalancersByRegion(region *SCloudregion, return lbs, nil } +func (man *SLoadbalancerManager) getLoadbalancersByExternalIds(externalIds []string) ([]SLoadbalancer, error) { + lbs := []SLoadbalancer{} + q := man.Query() + q = q.In("external_id", externalIds) + q = q.IsFalse("pending_deleted") + if err := db.FetchModelObjects(man, q, &lbs); err != nil { + log.Errorf("failed to get lbs for region: %#v error: %v", externalIds, err) + return nil, err + } + return lbs, nil +} + +func (man *SLoadbalancerManager) getLocalLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, lbs []cloudprovider.ICloudLoadbalancer) ([]SLoadbalancer, error) { + part1, err := man.getLoadbalancersByRegion(region, provider) + if err != nil { + return nil, err + } + + localLbs := map[string]SLoadbalancer{} + for i := range part1 { + localLbs[part1[i].Id] = part1[i] + } + + externalIds := []string{} + for i := range lbs { + externalIds = append(externalIds, lbs[i].GetGlobalId()) + } + + if len(externalIds) > 0 { + part2, err := man.getLoadbalancersByExternalIds(externalIds) + if err != nil { + return nil, err + } + + for i := range part2 { + localLbs[part2[i].Id] = part2[i] + } + } + + ret := []SLoadbalancer{} + for id, _ := range localLbs { + ret = append(ret, localLbs[id]) + } + + return ret, nil +} + func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, lbs []cloudprovider.ICloudLoadbalancer, syncRange *SSyncRange) ([]SLoadbalancer, []cloudprovider.ICloudLoadbalancer, compare.SyncResult) { syncOwnerId := provider.GetOwnerId() @@ -813,7 +860,7 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred remoteLbs := []cloudprovider.ICloudLoadbalancer{} syncResult := compare.SyncResult{} - dbLbs, err := man.getLoadbalancersByRegion(region, provider) + dbLbs, err := man.getLocalLoadbalancers(ctx, userCred, provider, region, remoteLbs) if err != nil { syncResult.Error(err) return nil, nil, syncResult @@ -846,7 +893,7 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred } } for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancer(ctx, userCred, commonext[i], syncOwnerId, provider) + err = commondb[i].SyncWithCloudLoadbalancer(ctx, userCred, commonext[i], syncOwnerId, provider, region) if err != nil { syncResult.UpdateError(err) } else { @@ -912,6 +959,14 @@ func (man *SLoadbalancerManager) newFromCloudLoadbalancer(ctx context.Context, u lbNetworkIds := getExtLbNetworkIds(extLb, lb.ManagerId) lb.NetworkId = strings.Join(lbNetworkIds, ",") + // classic vpc + if extLb.GetNetworkType() == api.LB_NETWORK_TYPE_CLASSIC { + if vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(ctx, provider, region); err == nil && vpc != nil { + lb.VpcId = vpc.GetId() + } + } + + // vpc if vpcId := extLb.GetVpcId(); len(vpcId) > 0 { if vpc, err := db.FetchByExternalIdAndManagerId(VpcManager, vpcId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Equals("manager_id", provider.Id) @@ -1081,7 +1136,7 @@ func (self *SLoadbalancer) SyncLoadbalancerEip(ctx context.Context, userCred mcc return result } -func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred mcclient.TokenCredential, extLb cloudprovider.ICloudLoadbalancer, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider) error { +func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred mcclient.TokenCredential, extLb cloudprovider.ICloudLoadbalancer, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, region *SCloudregion) error { lockman.LockObject(ctx, lb) defer lockman.ReleaseObject(ctx, lb) @@ -1100,6 +1155,14 @@ func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred } syncVirtualResourceMetadata(ctx, userCred, lb, extLb) + // classic vpc + if extLb.GetNetworkType() == api.LB_NETWORK_TYPE_CLASSIC { + if vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(ctx, provider, region); err == nil && vpc != nil { + lb.VpcId = vpc.GetId() + } + } + + // vpc if vpcId := extLb.GetVpcId(); len(vpcId) > 0 { if vpc, err := db.FetchByExternalIdAndManagerId(VpcManager, vpcId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Equals("manager_id", provider.Id) diff --git a/pkg/compute/regiondrivers/huawei.go b/pkg/compute/regiondrivers/huawei.go index 66f3f72127..d74b106f24 100644 --- a/pkg/compute/regiondrivers/huawei.go +++ b/pkg/compute/regiondrivers/huawei.go @@ -1927,7 +1927,7 @@ func (self *SHuaWeiRegionDriver) RequestCreateLoadbalancer(ctx context.Context, } } - if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil, lb.GetCloudprovider()); err != nil { + if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil, lb.GetCloudprovider(), lb.GetRegion()); err != nil { return nil, errors.Wrap(err, "Huawei.RequestCreateLoadbalancer.SyncWithCloudLoadbalancer") } lbbgs, err := iLoadbalancer.GetILoadBalancerBackendGroups() diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index 4b72bebcb5..9a98982a08 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -189,7 +189,7 @@ func (self *SManagedVirtualizationRegionDriver) RequestCreateLoadbalancer(ctx co if err := db.SetExternalId(lb, userCred, iLoadbalancer.GetGlobalId()); err != nil { return nil, err } - if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil, lb.GetCloudprovider()); err != nil { + if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil, lb.GetCloudprovider(), lb.GetRegion()); err != nil { return nil, err } //公网lb,需要同步public ip diff --git a/pkg/compute/regiondrivers/openstack.go b/pkg/compute/regiondrivers/openstack.go index dbbc959721..752b222137 100644 --- a/pkg/compute/regiondrivers/openstack.go +++ b/pkg/compute/regiondrivers/openstack.go @@ -178,7 +178,7 @@ func (self *SOpenStackRegionDriver) RequestCreateLoadbalancer(ctx context.Contex return nil, errors.Wrap(fmt.Errorf("status error"), "check status") } - if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil, lb.GetCloudprovider()); err != nil { + if err := lb.SyncWithCloudLoadbalancer(ctx, userCred, iLoadbalancer, nil, lb.GetCloudprovider(), lb.GetRegion()); err != nil { return nil, err } diff --git a/pkg/multicloud/qcloud/loadbalancer.go b/pkg/multicloud/qcloud/loadbalancer.go index b9048b3e4f..1223aa0586 100644 --- a/pkg/multicloud/qcloud/loadbalancer.go +++ b/pkg/multicloud/qcloud/loadbalancer.go @@ -312,10 +312,18 @@ func (self *SLoadbalancer) GetAddressType() string { } func (self *SLoadbalancer) GetNetworkType() string { - return api.LB_NETWORK_TYPE_VPC + if len(self.VpcID) > 0 { + return api.LB_NETWORK_TYPE_VPC + } else { + return api.LB_NETWORK_TYPE_CLASSIC + } } func (self *SLoadbalancer) GetNetworkIds() []string { + if len(self.SubnetId) == 0 { + return []string{} + } + return []string{self.SubnetId} } From 399539b92dcb8c506e5c6f2705324cdb2196e9c6 Mon Sep 17 00:00:00 2001 From: tangbin Date: Wed, 28 Oct 2020 08:46:11 +0800 Subject: [PATCH 2/2] elb sync bugfix --- .../models/loadbalancercachedcertificates.go | 2 +- pkg/compute/models/loadbalancercertificates.go | 13 ++++++++++--- pkg/compute/models/loadbalancers.go | 6 +++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pkg/compute/models/loadbalancercachedcertificates.go b/pkg/compute/models/loadbalancercachedcertificates.go index e45b6c0864..ffac98502e 100644 --- a/pkg/compute/models/loadbalancercachedcertificates.go +++ b/pkg/compute/models/loadbalancercachedcertificates.go @@ -317,7 +317,7 @@ func (man *SCachedLoadbalancerCertificateManager) newFromCloudLoadbalancerCertif if err != nil { switch err { case sql.ErrNoRows: - localcert, err := LoadbalancerCertificateManager.CreateCertificate(ctx, userCred, lbcert.Name, extCertificate.GetPublickKey(), extCertificate.GetPrivateKey(), extCertificate.GetFingerprint()) + localcert, err := LoadbalancerCertificateManager.CreateCertificate(ctx, userCred, lbcert.Name, extCertificate) if err != nil { return nil, fmt.Errorf("newFromCloudLoadbalancerCertificate CreateCertificate %s", err) } diff --git a/pkg/compute/models/loadbalancercertificates.go b/pkg/compute/models/loadbalancercertificates.go index 243bd8550f..7e54419221 100644 --- a/pkg/compute/models/loadbalancercertificates.go +++ b/pkg/compute/models/loadbalancercertificates.go @@ -32,6 +32,7 @@ import ( api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/validators" + "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" @@ -382,14 +383,15 @@ func (man *SLoadbalancerCertificateManager) InitializeData() error { return nil } -func (man *SLoadbalancerCertificateManager) CreateCertificate(ctx context.Context, userCred mcclient.TokenCredential, name string, publicKey string, privateKey, fingerprint string) (*SLoadbalancerCertificate, error) { +func (man *SLoadbalancerCertificateManager) CreateCertificate(ctx context.Context, userCred mcclient.TokenCredential, name string, extCert cloudprovider.ICloudLoadbalancerCertificate) (*SLoadbalancerCertificate, error) { + fingerprint := extCert.GetFingerprint() if len(fingerprint) == 0 { return nil, fmt.Errorf("CreateCertificate fingerprint can not be empty") } data := jsonutils.NewDict() - data.Set("certificate", jsonutils.NewString(publicKey)) - data.Set("private_key", jsonutils.NewString(privateKey)) + data.Set("certificate", jsonutils.NewString(extCert.GetPublickKey())) + data.Set("private_key", jsonutils.NewString(extCert.GetPrivateKey())) data.Set("name", jsonutils.NewString(name)) data.Set("fingerprint", jsonutils.NewString(fingerprint)) q := man.Query().Equals("fingerprint", fingerprint).Asc("created_at").IsFalse("pending_deleted") @@ -410,6 +412,11 @@ func (man *SLoadbalancerCertificateManager) CreateCertificate(ctx context.Contex cert.ProjectId = userCred.GetProjectId() cert.ProjectSrc = string(apis.OWNER_SOURCE_CLOUD) + // other information's + cert.CommonName = extCert.GetCommonName() + cert.SubjectAlternativeNames = extCert.GetSubjectAlternativeNames() + cert.NotAfter = extCert.GetExpireTime() + err = man.TableSpec().Insert(ctx, cert) if err != nil { return nil, err diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index 9abbdc3d87..879a5d0a94 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -815,7 +815,7 @@ func (man *SLoadbalancerManager) getLoadbalancersByExternalIds(externalIds []str return lbs, nil } -func (man *SLoadbalancerManager) getLocalLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, lbs []cloudprovider.ICloudLoadbalancer) ([]SLoadbalancer, error) { +func (man *SLoadbalancerManager) getLocalLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, lbs []cloudprovider.ICloudLoadbalancer) ([]SLoadbalancer, error) { part1, err := man.getLoadbalancersByRegion(region, provider) if err != nil { return nil, err @@ -960,7 +960,7 @@ func (man *SLoadbalancerManager) newFromCloudLoadbalancer(ctx context.Context, u lb.NetworkId = strings.Join(lbNetworkIds, ",") // classic vpc - if extLb.GetNetworkType() == api.LB_NETWORK_TYPE_CLASSIC { + if extLb.GetNetworkType() == api.LB_NETWORK_TYPE_CLASSIC { if vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(ctx, provider, region); err == nil && vpc != nil { lb.VpcId = vpc.GetId() } @@ -1156,7 +1156,7 @@ func (lb *SLoadbalancer) SyncWithCloudLoadbalancer(ctx context.Context, userCred syncVirtualResourceMetadata(ctx, userCred, lb, extLb) // classic vpc - if extLb.GetNetworkType() == api.LB_NETWORK_TYPE_CLASSIC { + if extLb.GetNetworkType() == api.LB_NETWORK_TYPE_CLASSIC { if vpc, err := VpcManager.GetOrCreateVpcForClassicNetwork(ctx, provider, region); err == nil && vpc != nil { lb.VpcId = vpc.GetId() }