Merge pull request #12792 from ioito/hotfix/qx-optimized-sync

fix(region): optimized cloudaccount sync resources

pkg/apis/compute/cloudaccount.go

@@ -500,6 +500,6 @@ type SyncRangeInput struct {
 	Host   []string `json:"host"`
 
 	// 按资源类型同步，可输入多个
-	// enmu: compute, loadbalancer, objectstore, rds, cache, nat, nas, waf, mongodb, es, kafka, app, container, quota, intervpcnetwork, cdn, dnszone
-	Resources []string `json:"resources" choices:"compute|loadbalancer|objectstore|rds|cache|nat|nas|waf|mongodb|es|kafka|app|container"`
+	// enmu: compute, network, loadbalancer, objectstore, rds, cache, nat, nas, waf, mongodb, es, kafka, app, container
+	Resources []string `json:"resources" choices:"compute|network|loadbalancer|objectstore|rds|cache|nat|nas|waf|mongodb|es|kafka|app|container"`
 }

pkg/cloudprovider/resources.go

@@ -394,7 +394,7 @@ type ICloudNic interface {
 	GetMAC() string
 	InClassicNetwork() bool
 	GetDriver() string
-	GetINetwork() ICloudNetwork
+	GetINetworkId() string
 
 	// GetSubAddress returns non-primary/secondary/alias ipv4 addresses of
 	// the network interface
@@ -415,12 +415,12 @@ type DummyICloudNic struct{}
 
 var _ ICloudNic = DummyICloudNic{}
 
-func (d DummyICloudNic) GetId() string              { panic(errors.ErrNotImplemented) }
-func (d DummyICloudNic) GetIP() string              { panic(errors.ErrNotImplemented) }
-func (d DummyICloudNic) GetMAC() string             { panic(errors.ErrNotImplemented) }
-func (d DummyICloudNic) InClassicNetwork() bool     { panic(errors.ErrNotImplemented) }
-func (d DummyICloudNic) GetDriver() string          { panic(errors.ErrNotImplemented) }
-func (d DummyICloudNic) GetINetwork() ICloudNetwork { panic(errors.ErrNotImplemented) }
+func (d DummyICloudNic) GetId() string          { panic(errors.ErrNotImplemented) }
+func (d DummyICloudNic) GetIP() string          { panic(errors.ErrNotImplemented) }
+func (d DummyICloudNic) GetMAC() string         { panic(errors.ErrNotImplemented) }
+func (d DummyICloudNic) InClassicNetwork() bool { panic(errors.ErrNotImplemented) }
+func (d DummyICloudNic) GetDriver() string      { panic(errors.ErrNotImplemented) }
+func (d DummyICloudNic) GetINetworkId() string  { panic(errors.ErrNotImplemented) }
 func (d DummyICloudNic) GetSubAddress() ([]string, error) {
 	return nil, nil
 }

pkg/compute/models/cloudsync.go

@@ -1752,10 +1752,12 @@ func syncPublicCloudProviderInfo(
 	}
 
 	if cloudprovider.IsSupportCompute(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) {
-		// 需要先同步vpc，避免私有云eip找不到network
-		syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK) {
+			// 需要先同步vpc，避免私有云eip找不到network
+			syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
 
-		syncRegionEips(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+			syncRegionEips(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
+		}
 		// sync snapshot policies before sync disks
 		syncRegionSnapshotPolicies(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
 

pkg/compute/models/disks.go

@@ -1464,7 +1464,6 @@ func (self *SDisk) syncWithCloudDisk(ctx context.Context, userCred mcclient.Toke
 	if provider.GetFactory().IsSupportPrepaidResources() && len(guests) == 1 && guests[0].IsPrepaidRecycle() {
 		recycle = true
 	}
-	extDisk.Refresh()
 
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
 		// self.Name = extDisk.GetName()

pkg/compute/models/guests.go

@@ -2466,9 +2466,7 @@ func (self *SGuest) syncWithCloudVM(ctx context.Context, userCred mcclient.Token
 		recycle = true
 	}
 
-	// metaData := extVM.GetMetadata()
 	diff, err := db.UpdateWithLock(ctx, self, func() error {
-		extVM.Refresh()
 		if options.NameSyncResources.Contains(self.Keyword()) && !recycle {
 			newName, _ := db.GenerateAlterName(self, extVM.GetName())
 			if len(newName) > 0 && newName != self.Name && extVM.GetName() != extVM.GetHostname() {
@@ -2930,8 +2928,8 @@ type sAddGuestnic struct {
 }
 
 func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host *SHost, ipList []string, index int) (*SNetwork, error) {
-	vnet := vnic.GetINetwork()
-	if vnet == nil {
+	vnetId := vnic.GetINetworkId()
+	if len(vnetId) == 0 {
 		if vnic.InClassicNetwork() {
 			region, _ := host.GetRegion()
 			cloudprovider := host.GetCloudprovider()
@@ -2958,7 +2956,7 @@ func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host
 		// find network by IP
 		return host.getNetworkOfIPOnHost(ip)
 	}
-	localNetObj, err := db.FetchByExternalIdAndManagerId(NetworkManager, vnet.GetGlobalId(), func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
+	localNetObj, err := db.FetchByExternalIdAndManagerId(NetworkManager, vnetId, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
 		vpc := VpcManager.Query().SubQuery()
 		wire := WireManager.Query().SubQuery()
 		return q.Join(wire, sqlchemy.Equals(q.Field("wire_id"), wire.Field("id"))).
@@ -2966,7 +2964,7 @@ func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host
 			Filter(sqlchemy.Equals(vpc.Field("manager_id"), host.ManagerId))
 	})
 	if err != nil {
-		return nil, fmt.Errorf("Cannot find network of external_id %s: %v", vnet.GetGlobalId(), err)
+		return nil, fmt.Errorf("Cannot find network of external_id %s: %v", vnetId, err)
 	}
 	localNet := localNetObj.(*SNetwork)
 	return localNet, nil

pkg/multicloud/aliyun/instancenic.go

@@ -69,20 +69,8 @@ func (self *SInstanceNic) GetDriver() string {
 	return "virtio"
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	vswitchId := self.instance.VpcAttributes.VSwitchId
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		return nil
-	}
-	for i := 0; i < len(wires); i += 1 {
-		wire := wires[i].(*SWire)
-		net := wire.getNetworkById(vswitchId)
-		if net != nil {
-			return net
-		}
-	}
-	return nil
+func (self *SInstanceNic) GetINetworkId() string {
+	return self.instance.VpcAttributes.VSwitchId
 }
 
 func (self *SInstanceNic) GetSubAddress() ([]string, error) {

pkg/multicloud/apsara/disk.go

@@ -17,6 +17,7 @@ package apsara
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -272,6 +273,7 @@ func (self *SRegion) CreateDisk(zoneId string, category string, name string, siz
 }
 
 func (self *SRegion) getDisk(diskId string) (*SDisk, error) {
+	debug.PrintStack()
 	disks, total, err := self.GetDisks("", "", "", []string{diskId}, 0, 1)
 	if err != nil {
 		return nil, err

pkg/multicloud/apsara/instance.go

@@ -243,8 +243,6 @@ func (self *SInstance) GetIDisks() ([]cloudprovider.ICloudDisk, error) {
 
 	sort.Sort(byAttachedTime(disks))
 
-	log.Debugf("%s", jsonutils.Marshal(&disks))
-
 	idisks := make([]cloudprovider.ICloudDisk, len(disks))
 	for i := 0; i < len(disks); i += 1 {
 		store, err := self.host.zone.getStorageByCategory(disks[i].Category)

pkg/multicloud/apsara/instancenic.go

@@ -47,18 +47,6 @@ func (self *SInstanceNic) GetDriver() string {
 	return "virtio"
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	vswitchId := self.instance.VpcAttributes.VSwitchId
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		return nil
-	}
-	for i := 0; i < len(wires); i += 1 {
-		wire := wires[i].(*SWire)
-		net := wire.getNetworkById(vswitchId)
-		if net != nil {
-			return net
-		}
-	}
-	return nil
+func (self *SInstanceNic) GetINetworkId() string {
+	return self.instance.VpcAttributes.VSwitchId
 }

pkg/multicloud/aws/instancenic.go

@@ -56,20 +56,8 @@ func (self *SInstanceNic) GetDriver() string {
 	return "virtio"
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	networkId := self.instance.VpcAttributes.NetworkId
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		return nil
-	}
-	for i := 0; i < len(wires); i += 1 {
-		wire := wires[i].(*SWire)
-		net := wire.getNetworkById(networkId)
-		if net != nil {
-			return net
-		}
-	}
-	return nil
+func (self *SInstanceNic) GetINetworkId() string {
+	return self.instance.VpcAttributes.NetworkId
 }
 
 func (self *SInstanceNic) getEc2Client() *ec2.EC2 {

pkg/multicloud/azure/classic_instancenic.go

@@ -15,7 +15,8 @@
 package azure
 
 import (
-	"yunion.io/x/log"
+	"strings"
+
 	"yunion.io/x/pkg/util/netutils"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -58,17 +59,6 @@ func (self *SClassicInstanceNic) InClassicNetwork() bool {
 	return true
 }
 
-func (self *SClassicInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		log.Errorf("GetINetwork error: %v", err)
-		return nil
-	}
-	for i := 0; i < len(wires); i++ {
-		wire := wires[i].(*SClassicWire)
-		if network := wire.getNetworkById(self.ID); network != nil {
-			return network
-		}
-	}
-	return nil
+func (self *SClassicInstanceNic) GetINetworkId() string {
+	return strings.ToLower(self.ID)
 }

pkg/multicloud/azure/instancenic.go

@@ -114,15 +114,11 @@ func (self *SInstanceNic) assignSecurityGroup(secgroupId string) error {
 	return self.updateSecurityGroup(secgroupId)
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
+func (self *SInstanceNic) GetINetworkId() string {
 	if len(self.Properties.IPConfigurations) > 0 {
-		network, err := self.instance.host.zone.region.GetNetwork(self.Properties.IPConfigurations[0].Properties.Subnet.ID)
-		if err != nil {
-			return nil
-		}
-		return network
+		return strings.ToLower(self.Properties.IPConfigurations[0].Properties.Subnet.ID)
 	}
-	return nil
+	return ""
 }
 
 func (self *SRegion) GetNetworkInterface(interfaceId string) (*SInstanceNic, error) {

pkg/multicloud/cloudpods/instancenic.go

@@ -68,12 +68,8 @@ func (self *SInstanceNic) UnassignAddress(IpAddrs []string) error {
 	return cloudprovider.ErrNotImplemented
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	network, err := self.ins.host.zone.region.GetNetwork(self.NetworkId)
-	if err != nil {
-		return nil
-	}
-	return network
+func (self *SInstanceNic) GetINetworkId() string {
+	return self.NetworkId
 }
 
 func (self *SInstance) GetINics() ([]cloudprovider.ICloudNic, error) {

pkg/multicloud/ctyun/instancenic.go

@@ -15,7 +15,6 @@
 package ctyun
 
 import (
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -61,14 +60,8 @@ func (self *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	network, err := self.instance.host.zone.region.GetNetwork(self.NetID)
-	if err != nil {
-		log.Errorf("SInstanceNic.GetINetwork %s", err)
-		return nil
-	}
-
-	return network
+func (self *SInstanceNic) GetINetworkId() string {
+	return self.NetID
 }
 
 func (self *SRegion) GetNics(vmId string) ([]SInstanceNic, error) {

pkg/multicloud/ecloud/instancenic.go

@@ -17,8 +17,6 @@ package ecloud
 import (
 	"time"
 
-	"yunion.io/x/log"
-
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
@@ -84,20 +82,6 @@ func (in *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (in *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	net, err := in.instance.host.zone.region.GetNetworkById("", "", in.NetworkId)
-	if err != nil {
-		log.Errorf("unable to get Network %s", in.NetworkId)
-		return nil
-	}
-	vpc, err := in.instance.host.zone.region.getVpcByRouterId(in.RouterId)
-	if err != nil {
-		log.Errorf("unable to get Vpc by routerId %s", in.RouterId)
-	}
-	wire := &SWire{
-		vpc:  vpc,
-		zone: in.instance.host.zone,
-	}
-	net.wire = wire
-	return net
+func (in *SInstanceNic) GetINetworkId() string {
+	return in.NetworkId
 }

pkg/multicloud/esxi/vnic.go

@@ -68,6 +68,6 @@ func (nic *SVirtualNIC) InClassicNetwork() bool {
 	return false
 }
 
-func (nic *SVirtualNIC) GetINetwork() cloudprovider.ICloudNetwork {
-	return nil
+func (nic *SVirtualNIC) GetINetworkId() string {
+	return ""
 }

pkg/multicloud/google/instancenic.go

@@ -55,15 +55,11 @@ func (nic *SNetworkInterface) InClassicNetwork() bool {
 	return false
 }
 
-func (nic *SNetworkInterface) GetINetwork() cloudprovider.ICloudNetwork {
+func (nic *SNetworkInterface) GetINetworkId() string {
 	vpc := &SVpc{region: nic.instance.host.zone.region}
 	err := nic.instance.host.zone.region.GetBySelfId(nic.Subnetwork, vpc)
 	if err != nil {
-		return nil
+		return ""
 	}
-	networks, _ := vpc.getWire().GetINetworks()
-	for i := range networks {
-		return networks[i]
-	}
-	return nil
+	return vpc.Id
 }

pkg/multicloud/hcso/instancenic.go

@@ -83,26 +83,14 @@ func (self *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
+func (self *SInstanceNic) GetINetworkId() string {
 	instanceId := self.instance.GetId()
 	subnets, err := self.instance.host.zone.region.getSubnetIdsByInstanceId(instanceId)
 	if err != nil || len(subnets) == 0 {
 		log.Errorf("getSubnetIdsByInstanceId error: %s", err.Error())
-		return nil
+		return ""
 	}
-
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		return nil
-	}
-	for i := 0; i < len(wires); i += 1 {
-		wire := wires[i].(*SWire)
-		net := wire.getNetworkById(subnets[0])
-		if net != nil {
-			return net
-		}
-	}
-	return nil
+	return subnets[0]
 }
 
 func (self *SRegion) getSubnetIdsByInstanceId(instanceId string) ([]string, error) {

pkg/multicloud/huawei/instancenic.go

@@ -83,26 +83,15 @@ func (self *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
+func (self *SInstanceNic) GetINetworkId() string {
 	instanceId := self.instance.GetId()
 	subnets, err := self.instance.host.zone.region.getSubnetIdsByInstanceId(instanceId)
 	if err != nil || len(subnets) == 0 {
 		log.Errorf("getSubnetIdsByInstanceId error: %s", err.Error())
-		return nil
+		return ""
 	}
 
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		return nil
-	}
-	for i := 0; i < len(wires); i += 1 {
-		wire := wires[i].(*SWire)
-		net := wire.getNetworkById(subnets[0])
-		if net != nil {
-			return net
-		}
-	}
-	return nil
+	return subnets[0]
 }
 
 func (self *SRegion) getSubnetIdsByInstanceId(instanceId string) ([]string, error) {

pkg/multicloud/jdcloud/instancenic.go

@@ -17,8 +17,6 @@ package jdcloud
 import (
 	"github.com/jdcloud-api/jdcloud-sdk-go/services/vm/models"
 
-	"yunion.io/x/log"
-
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
@@ -49,11 +47,6 @@ func (in *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (in *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	net, err := in.instance.host.zone.region.GetNetworkById(in.SubnetId)
-	if err != nil {
-		log.Errorf("unable to get network %s: %v", in.SubnetId, err)
-		return nil
-	}
-	return net
+func (in *SInstanceNic) GetINetworkId() string {
+	return in.SubnetId
 }

pkg/multicloud/openstack/instancenic.go

@@ -17,9 +17,7 @@ package openstack
 import (
 	"fmt"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/util/netutils"
 	"yunion.io/x/pkg/util/regutils"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -87,24 +85,11 @@ func (nic *SInstancePort) InClassicNetwork() bool {
 	return false
 }
 
-func (nic *SInstancePort) GetINetwork() cloudprovider.ICloudNetwork {
+func (nic *SInstancePort) GetINetworkId() string {
 	for i := range nic.FixedIps {
 		if regutils.MatchIPAddr(nic.FixedIps[i].IpAddress) {
-			network, err := nic.region.GetNetwork(nic.FixedIps[i].SubnetId)
-			if err != nil {
-				log.Errorf("failed to found network by %s error: %v", nic.FixedIps[i].SubnetId, err)
-				return nil
-			}
-			for _, pool := range network.AllocationPools {
-				start, _ := netutils.NewIPV4Addr(pool.Start)
-				end, _ := netutils.NewIPV4Addr(pool.End)
-				ipPool := netutils.NewIPV4AddrRange(start, end)
-				ip, _ := netutils.NewIPV4Addr(nic.FixedIps[i].IpAddress)
-				ipPool.Contains(ip)
-				network.AllocationPools = []AllocationPool{pool}
-				return network
-			}
+			return nic.FixedIps[i].SubnetId
 		}
 	}
-	return nil
+	return ""
 }

pkg/multicloud/qcloud/instancenic.go

@@ -49,18 +49,6 @@ func (self *SInstanceNic) InClassicNetwork() bool {
 	return self.classic
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
-	networkId := self.instance.VirtualPrivateCloud.SubnetId
-	wires, err := self.instance.host.GetIWires()
-	if err != nil {
-		return nil
-	}
-	for i := 0; i < len(wires); i++ {
-		wire := wires[i].(*SWire)
-		net := wire.getNetworkById(networkId)
-		if net != nil {
-			return net
-		}
-	}
-	return nil
+func (self *SInstanceNic) GetINetworkId() string {
+	return self.instance.VirtualPrivateCloud.SubnetId
 }

pkg/multicloud/ucloud/instancenic.go

@@ -46,13 +46,11 @@ func (self *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (self *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
+func (self *SInstanceNic) GetINetworkId() string {
 	for _, ip := range self.instance.IPSet {
 		if ip.IP == self.ipAddr {
-			network, _ := self.instance.host.zone.region.getNetwork(ip.SubnetID)
-			return network
+			return ip.SubnetID
 		}
 	}
-
-	return nil
+	return ""
 }

pkg/multicloud/zstack/instancenic.go

@@ -59,27 +59,16 @@ func (nic *SInstanceNic) InClassicNetwork() bool {
 	return false
 }
 
-func (nic *SInstanceNic) GetINetwork() cloudprovider.ICloudNetwork {
+func (nic *SInstanceNic) GetINetworkId() string {
 	networks, err := nic.instance.host.zone.region.GetNetworks(nic.instance.host.zone.UUID, "", nic.L3NetworkUUID, "")
 	if err != nil {
 		log.Errorf("failed to found networks for nic %v error: %v", jsonutils.Marshal(nic).String(), err)
-		return nil
+		return ""
 	}
 	for i := 0; i < len(networks); i++ {
 		if networks[i].Contains(nic.IP) {
-			l3Network, err := nic.instance.host.zone.region.GetL3Network(networks[i].L3NetworkUUID)
-			if err != nil {
-				log.Errorf("failed to found l3Network for network %v error: %v", jsonutils.Marshal(networks[i]).String(), err)
-				return nil
-			}
-			wire, err := nic.instance.host.zone.region.GetWire(l3Network.L2NetworkUUID)
-			if err != nil {
-				log.Errorf("failed to found wire for l3Network %v error: %v", jsonutils.Marshal(l3Network).String(), err)
-				return nil
-			}
-			networks[i].wire = wire
-			return &networks[i]
+			return networks[i].L3NetworkUUID
 		}
 	}
-	return nil
+	return ""
 }