From fee218c59cd38d6dfb12bb5c7df83bad27d4f91d Mon Sep 17 00:00:00 2001 From: rainzm Date: Sat, 12 Dec 2020 17:00:27 +0800 Subject: [PATCH] feat(region): larger suggest network and more complete vlanid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 抓取标准 PortGroup 的 vlanid，以前只有分布式 PortGroup 的 vlanid 2. 提供的建议网络会尽可能的大 --- pkg/compute/models/cloudaccounts.go | 321 +++++++++------------------- pkg/multicloud/esxi/manager.go | 236 +++++++++++--------- pkg/multicloud/esxi/network.go | 155 ++++++++++++++ 3 files changed, 387 insertions(+), 325 deletions(-) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 98eb67bf71..256f7af6b5 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -357,29 +357,28 @@ func (scm *SCloudaccountManager) AllowPerformPrepareNets(_ context.Context, user return db.IsAdminAllowPerform(userCred, scm, "prepare-nets") } -type sHostVMIp struct { - hostIps map[string]string - vms []esxi.SSimpleVM - prefix string +type sNetworkInfo struct { + esxi.SNetworkInfo + prefix string } -func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, client *esxi.SESXiClient, input api.CloudaccountPerformPrepareNetsInput) ([]sHostVMIp, error) { +func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, client *esxi.SESXiClient, + input api.CloudaccountPerformPrepareNetsInput) ([]sNetworkInfo, error) { caName := input.Name wireLevel := input.WireLevelForVmware - ret := make([]sHostVMIp, 0) + ret := make([]sNetworkInfo, 0) if len(wireLevel) == 0 { wireLevel = api.CLOUD_ACCOUNT_WIRE_LEVEL_VCENTER } switch wireLevel { case api.CLOUD_ACCOUNT_WIRE_LEVEL_VCENTER: - hostIps, simpleVms, err := client.HostVmIPs(ctx) + nInfo, err := client.HostVmIPs(ctx) if err != nil { - return ret, errors.Wrap(err, "unable to fetch ips of hosts and vms") + return nil, errors.Wrap(err, "unable to fetch ips of hosts and vms") } - ret = append(ret, sHostVMIp{ - hostIps: hostIps, - vms: simpleVms, - prefix: caName, + ret = append(ret, sNetworkInfo{ + SNetworkInfo: nInfo, + prefix: caName, }) case api.CLOUD_ACCOUNT_WIRE_LEVEL_DATACENTER: dcs, err := client.GetDatacenters() @@ -387,14 +386,13 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien return ret, errors.Wrap(err, "GetDatacenters") } for _, dc := range dcs { - hostIps, simpleVms, err := client.HostVmIPsInDc(ctx, dc) + nInfo, err := client.HostVmIPsInDc(ctx, dc) if err != nil { return ret, errors.Wrapf(err, "unable to fetch ips of hosts and vms for dc %q", dc.GetName()) } - ret = append(ret, sHostVMIp{ - hostIps: hostIps, - vms: simpleVms, - prefix: fmt.Sprintf("%s/%s", caName, dc.GetName()), + ret = append(ret, sNetworkInfo{ + SNetworkInfo: nInfo, + prefix: fmt.Sprintf("%s/%s", caName, dc.GetName()), }) } case api.CLOUD_ACCOUNT_WIRE_LEVEL_CLUSTER: @@ -408,14 +406,13 @@ func (scm *SCloudaccountManager) hostVMIPsPrepareNets(ctx context.Context, clien return nil, errors.Wrapf(err, "unable to ListCluster for dc %q", dc.GetName()) } for _, cluster := range clusters { - hostIps, simpleVms, err := client.HostVmIPsInCluster(ctx, cluster) + nInfo, err := client.HostVmIPsInCluster(ctx, cluster) if err != nil { return ret, errors.Wrapf(err, "unable to fetch ips of hosts and vms for dc %q cluster %q", dc.GetName(), cluster.GetName()) } - ret = append(ret, sHostVMIp{ - hostIps: hostIps, - vms: simpleVms, - prefix: fmt.Sprintf("%s/%s/%s", caName, dc.GetName(), cluster.GetName()), + ret = append(ret, sNetworkInfo{ + SNetworkInfo: nInfo, + prefix: fmt.Sprintf("%s/%s/%s", caName, dc.GetName(), cluster.GetName()), }) } } @@ -478,7 +475,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre zoneids = append(zoneids, id) } // fetch all wire candidate - wires, err := scm.fetchWires(zoneids, domainId) + wires, err := scm.fetchWires(userCred, zoneids, domainId) if err != nil { return output, errors.Wrap(err, "unable to fetch wires") } @@ -521,7 +518,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre } // hack client := iregion.(*esxi.SESXiClient) - hostVMIPs, err := scm.hostVMIPsPrepareNets(ctx, client, input) + nInfos, err := scm.hostVMIPsPrepareNets(ctx, client, input) if err != nil { return output, err } @@ -534,13 +531,11 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre } networks[i] = nets } - output.CAWireNets = make([]api.CAWireNet, 0, len(hostVMIPs)) - for _, hv := range hostVMIPs { + output.CAWireNets = make([]api.CAWireNet, 0, len(nInfos)) + for _, ni := range nInfos { var ( - wireNet api.CAWireNet - hostIps = hv.hostIps - simpleVms = hv.vms - prefix = hv.prefix + wireNet api.CAWireNet + hostIps = ni.HostIps ) // key of ipHosts is host's ip @@ -590,7 +585,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre } else { wireNet.SuggestedWire = api.CAWireConf{ ZoneIds: zoneids, - Name: prefix + "-wire", + Name: ni.prefix + "-wire", Description: fmt.Sprintf("Auto created Wire for VMware account %q", input.Name), } } @@ -615,7 +610,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre confs := make([]api.CANetConf, len(sConfs)) for i := range confs { confs[i].CASimpleNetConf = sConfs[i] - confs[i].Name = fmt.Sprintf("%s-host-network-%d", prefix, i+1) + confs[i].Name = fmt.Sprintf("%s-host-network-%d", ni.prefix, i+1) } wireNet.HostSuggestedNetworks = confs } @@ -628,214 +623,96 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre return output, err } } - - type vm struct { - FakeID int - IP netutils.IPV4Addr - Name string - VlanID int32 + type simpleNet struct { + Id string + Vlan int32 } - vms := make([]vm, 0, len(simpleVms)) - var nip netutils.IPV4Addr - guestMap := make(map[int]*api.CAGuestNet) - for i := range simpleVms { - id := i - if len(simpleVms[i].IPVlans) == 0 { - if _, ok := guestMap[id]; !ok { - guestMap[id] = &api.CAGuestNet{ - Name: simpleVms[i].Name, - IPNets: []api.CAIPNet{}, - } - } - } - for _, ipVlan := range simpleVms[i].IPVlans { - nip, err = netutils.NewIPV4Addr(ipVlan.IP) - if err != nil { - return output, err - } - vms = append(vms, vm{ - FakeID: i, - IP: nip, - Name: simpleVms[i].Name, - VlanID: ipVlan.VlanID, - }) - } - } - // sort vms via vm's ip - sort.Slice(vms, func(i, j int) bool { - return vms[i].IP < vms[j].IP - }) - - type sExcludeNet struct { - StartIp netutils.IPV4Addr - EndIp netutils.IPV4Addr - Id string - } - excludeNets := make([]sExcludeNet, 0, len(allNets)+len(wireNet.HostSuggestedNetworks)) + existedNetMap := make(map[netutils.IPV4Addr]simpleNet, len(allNets)) for i := range allNets { - startIp, _ := netutils.NewIPV4Addr(allNets[i].GuestIpStart) - endIp, _ := netutils.NewIPV4Addr(allNets[i].GuestIpEnd) - excludeNets = append(excludeNets, sExcludeNet{ - StartIp: startIp, - EndIp: endIp, - Id: allNets[i].GetId(), - }) - } - for i := range wireNet.HostSuggestedNetworks { - startIp, _ := netutils.NewIPV4Addr(wireNet.HostSuggestedNetworks[i].GuestIpStart) - endIp, _ := netutils.NewIPV4Addr(wireNet.HostSuggestedNetworks[i].GuestIpEnd) - excludeNets = append(excludeNets, sExcludeNet{ - StartIp: startIp, - EndIp: endIp, - }) - } - // sort excludeNets via their GuestIpStart - sort.Slice(excludeNets, func(i, j int) bool { - return excludeNets[i].StartIp < excludeNets[j].StartIp - }) - - svNets := make([]api.CASimpleNetConf, 0, 5) - var vmi, neti int - - Loop: - for neti = 0; neti < len(excludeNets); { - if vmi >= len(vms) { - break - } - startIp := excludeNets[neti].StartIp - endIp := excludeNets[neti].EndIp - switch { - case vms[vmi].IP > endIp: - neti++ - case vms[vmi].IP >= startIp: - for vms[vmi].IP <= endIp { - id := vms[vmi].FakeID - if _, ok := guestMap[id]; !ok { - guestMap[id] = &api.CAGuestNet{ - Name: vms[vmi].Name, - IPNets: []api.CAIPNet{}, - } - } - guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{ - IP: vms[vmi].IP.String(), - SuitableNetwork: excludeNets[neti].Id, - VlanID: vms[vmi].VlanID, - }) - vmi += 1 - if vmi == len(vms) { - break Loop - } - } - neti++ - default: - for vms[vmi].IP < startIp { - suggestStartIp := vms[vmi].IP - suggestEndIp := vms[vmi].IP - endLimitIp := suggestStartIp.NetAddr(24) + 255 - vlanId := vms[vmi].VlanID - for { - id := vms[vmi].FakeID - if _, ok := guestMap[id]; !ok { - guestMap[id] = &api.CAGuestNet{ - Name: vms[vmi].Name, - IPNets: []api.CAIPNet{}, - } - } - guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{ - IP: vms[vmi].IP.String(), - VlanID: vms[vmi].VlanID, - }) - vmi++ - if vmi == len(vms) { - break - } - if suggestEndIp == endLimitIp { - break - } - if vms[vmi].IP > suggestEndIp+1 { - break - } - if vms[vmi].IP >= startIp { - break - } - if vms[vmi].VlanID != vlanId { - break - } - suggestEndIp = vms[vmi].IP - } - svNets = append(svNets, api.CASimpleNetConf{ - GuestIpStart: suggestStartIp.String(), - GuestIpEnd: suggestEndIp.String(), - VlanID: vlanId, - GuestIpMask: 24, - GuestGateway: (suggestStartIp.NetAddr(24) + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)).String(), - }) - if vmi == len(vms) { - break Loop - } - } + ipStart, _ := netutils.NewIPV4Addr(allNets[i].GuestIpStart) + ipEnd, _ := netutils.NewIPV4Addr(allNets[i].GuestIpEnd) + for ip := ipStart; ip <= ipEnd; ip++ { + existedNetMap[ip] = simpleNet{Id: allNets[i].Id, Vlan: int32(allNets[i].VlanId)} } } - for vmi < len(vms) { - suggestStartIp := vms[vmi].IP - endLimitIp := suggestStartIp.NetAddr(24) + 255 - suggestEndIp := suggestStartIp - vlanId := vms[vmi].VlanID - for { - id := vms[vmi].FakeID - if _, ok := guestMap[id]; !ok { - guestMap[id] = &api.CAGuestNet{ - Name: vms[vmi].Name, - IPNets: []api.CAIPNet{}, - } + guests := make([]api.CAGuestNet, len(ni.VMs)) + for i := range guests { + guests[i].Name = ni.VMs[i].Name + for _, ipvlan := range ni.VMs[i].IPVlans { + var suitableNetId string + sn, ok := existedNetMap[ipvlan.IP] + if ok { + suitableNetId = sn.Id } - guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{ - IP: vms[vmi].IP.String(), + guests[i].IPNets = append(guests[i].IPNets, api.CAIPNet{ + IP: ipvlan.IP.String(), + VlanID: ipvlan.VlanId, + SuitableNetwork: suitableNetId, }) - vmi++ - if vmi == len(vms) { - break - } - if suggestEndIp == endLimitIp { - break - } - if vms[vmi].IP > suggestEndIp+1 { - break - } - if vms[vmi].VlanID != vlanId { - break - } - suggestEndIp = vms[vmi].IP } - svNets = append(svNets, api.CASimpleNetConf{ - GuestIpStart: suggestStartIp.String(), - GuestIpEnd: suggestEndIp.String(), - VlanID: vlanId, - GuestIpMask: 24, - GuestGateway: (suggestStartIp.NetAddr(24) + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)).String(), - }) } + wireNet.Guests = guests - for _, guest := range guestMap { - wireNet.Guests = append(wireNet.Guests, *guest) - } - - if len(svNets) > 0 { - confs := make([]api.CANetConf, len(svNets)) - for i := range confs { - confs[i].CASimpleNetConf = svNets[i] - confs[i].Name = fmt.Sprintf("%s-guest-network-%d", prefix, i+1) + for vlan, ips := range ni.VlanIps { + for i := 0; i < len(ips); i++ { + ip := ips[i] + if _, ok := existedNetMap[ip]; ok { + continue + } + net := ip.NetAddr(24) + netLimitLow := net + 1 + netLimitUp := net + 254 + // find startip + startIp := ip - 1 + for ; startIp >= netLimitLow; startIp-- { + if _, ok := existedNetMap[startIp]; ok { + break + } + if _, ok := ni.IPPool.Get(startIp); ok { + break + } + } + endIp := ip + 1 + for ; endIp <= netLimitUp; endIp++ { + if _, ok := existedNetMap[endIp]; ok { + break + } + if proc, ok := ni.IPPool.Get(endIp); ok { + if proc.VlanId == vlan { + // find one in ips + i++ + continue + } + break + } + } + slen := len(wireNet.GuestSuggestedNetworks) + wireNet.GuestSuggestedNetworks = append(wireNet.GuestSuggestedNetworks, api.CANetConf{ + Name: fmt.Sprintf("%s-guest-network-%d", ni.prefix, slen+1), + Description: "", + CASimpleNetConf: api.CASimpleNetConf{ + VlanID: vlan, + GuestIpStart: (startIp + 1).String(), + GuestIpEnd: (endIp - 1).String(), + GuestIpMask: 24, + GuestGateway: (net + netutils.IPV4Addr(options.Options.DefaultNetworkGatewayAddressEsxi)). + String(), + }, + }) + // Avoid assigning already assigned ip subnet + existedNetMap[startIp+1] = simpleNet{} + existedNetMap[endIp-1] = simpleNet{} } - wireNet.GuestSuggestedNetworks = confs } output.CAWireNets = append(output.CAWireNets, wireNet) } return output, nil } -func (manager *SCloudaccountManager) fetchWires(zoneIds []string, domainId string) ([]SWire, error) { +func (manager *SCloudaccountManager) fetchWires(userCred mcclient.TokenCredential, zoneIds []string, domainId string) ([]SWire, error) { q := WireManager.Query().Equals("domain_id", domainId).In("zone_id", zoneIds) + q = WireManager.FilterByOwner(q, userCred, rbacutils.ScopeDomain) wires := make([]SWire, 0, 1) err := db.FetchModelObjects(WireManager, q, &wires) return wires, err diff --git a/pkg/multicloud/esxi/manager.go b/pkg/multicloud/esxi/manager.go index fb5151bf82..17a9a89b77 100644 --- a/pkg/multicloud/esxi/manager.go +++ b/pkg/multicloud/esxi/manager.go @@ -19,6 +19,7 @@ import ( "fmt" "net/url" "reflect" + "sort" "strings" "sync" @@ -105,7 +106,8 @@ type SESXiClient struct { client *govmomi.Client context context.Context - datacenters []*SDatacenter + datacenters []*SDatacenter + networkQueryMap *sync.Map } func NewESXiClient(cfg *ESXiClientConfig) (*SESXiClient, error) { @@ -682,14 +684,15 @@ func (cli *SESXiClient) MoveDisk(ctx context.Context, src, dst string, isForce b } var ( - SIMPLE_HOST_PROPS = []string{"name", "config.network", "vm"} - SIMPLE_VM_PROPS = []string{"name", "guest.net", "config.template", "config.hardware.device"} - SIMPLE_DVPG_PROPS = []string{"key", "config.defaultPortConfig"} + SIMPLE_HOST_PROPS = []string{"name", "config.network", "vm"} + SIMPLE_VM_PROPS = []string{"name", "guest.net", "config.template", "config.hardware.device"} + SIMPLE_DVPG_PROPS = []string{"key", "config.defaultPortConfig", "config.distributedVirtualSwitch"} + SIMPLE_NETWORK_PROPS = []string{"name"} ) type SIPVlan struct { - IP string - VlanID int32 + IP netutils.IPV4Addr + VlanId int32 } type SSimpleVM struct { @@ -710,54 +713,74 @@ func (cli *SESXiClient) scanAllDvPortgroups() ([]*SDistributedVirtualPortgroup, return dvpgs, nil } -func (cli *SESXiClient) dvpgKeyVlanMap() (*sync.Map, error) { - dvpgs, err := cli.scanAllDvPortgroups() +func (cli *SESXiClient) scanAllNetworks() ([]mo.Network, error) { + var monets []mo.Network + err := cli.scanAllMObjects(SIMPLE_NETWORK_PROPS, &monets) if err != nil { - return nil, err + return nil, errors.Wrap(err, "scanAllMObjects") } - ret := new(sync.Map) - for i := range dvpgs { - key := dvpgs[i].getMODVPortgroup().Key - vlanid := dvpgs[i].GetVlanId() - ret.Store(key, vlanid) - } - return ret, nil + return monets, nil } -func (cli *SESXiClient) HostVmIPsInDc(ctx context.Context, dc *SDatacenter) (map[string]string, []SSimpleVM, error) { +func (cli *SESXiClient) networkName(refV string) (string, error) { + if cli.networkQueryMap == nil { + nets, err := cli.scanAllNetworks() + if err != nil { + return "", err + } + cli.networkQueryMap = &sync.Map{} + for i := range nets { + cli.networkQueryMap.Store(nets[i].Reference().Value, nets[i].Name) + } + } + iter, ok := cli.networkQueryMap.Load(refV) + if !ok { + return "", nil + } + return iter.(string), nil +} + +func (cli *SESXiClient) HostVmIPsInDc(ctx context.Context, dc *SDatacenter) (SNetworkInfo, error) { var hosts []mo.HostSystem err := cli.scanMObjects(dc.object.Entity().Self, HOST_SYSTEM_PROPS, &hosts) if err != nil { - return nil, nil, errors.Wrap(err, "scanMObjects") + return SNetworkInfo{}, errors.Wrap(err, "scanMObjects") } return cli.hostVMIPs(ctx, hosts) } -func (cli *SESXiClient) HostVmIPsInCluster(ctx context.Context, cluster *SCluster) (map[string]string, []SSimpleVM, - error) { +func (cli *SESXiClient) HostVmIPsInCluster(ctx context.Context, cluster *SCluster) (SNetworkInfo, error) { var hosts []mo.HostSystem err := cli.scanMObjects(cluster.object.Entity().Self, HOST_SYSTEM_PROPS, &hosts) if err != nil { - return nil, nil, errors.Wrap(err, "scanMObjects") + return SNetworkInfo{}, errors.Wrap(err, "scanMObjects") } return cli.hostVMIPs(ctx, hosts) } -func (cli *SESXiClient) hostVMIPs(ctx context.Context, hosts []mo.HostSystem) (map[string]string, []SSimpleVM, error) { - dvpgKeyVlanMap, err := cli.dvpgKeyVlanMap() +type SNetworkInfo struct { + HostIps map[string]string + VMs []SSimpleVM + VlanIps map[int32][]netutils.IPV4Addr + IPPool SIPPool +} + +func (cli *SESXiClient) hostVMIPs(ctx context.Context, hosts []mo.HostSystem) (SNetworkInfo, error) { + ret := SNetworkInfo{} + dvpgMap, err := cli.getDVPGMap() if err != nil { - return nil, nil, errors.Wrap(err, "unable to get dvpgKeyVlanMap") + return ret, errors.Wrap(err, "unable to get dvpgKeyVlanMap") } group, ctx := errgroup.WithContext(ctx) - collection := make([][]SSimpleVM, len(hosts)) + collection := make([]SNetworkInfo, len(hosts)) for i := range hosts { j := i group.Go(func() error { - vmIps, err := cli.vmIPs(&hosts[j], dvpgKeyVlanMap) + nInfo, err := cli.vmIPs(&hosts[j], dvpgMap) if err != nil { return err } - collection[j] = vmIps + collection[j] = nInfo return nil }) } @@ -771,72 +794,56 @@ func (cli *SESXiClient) hostVMIPs(ctx context.Context, hosts []mo.HostSystem) (m } err = group.Wait() if err != nil { - return nil, nil, err + return ret, err } // length - length := 0 - for i := range collection { - length += len(collection[i]) + if len(collection) == 0 { + ret.HostIps = hostIps + return ret, nil } - svms := make([]SSimpleVM, 0, length) - for i := range collection { - for j := range collection[i] { - svms = append(svms, collection[i][j]) - } - } - return hostIps, svms, nil + ni := cli.mergeNetworInfo(collection) + ni.HostIps = hostIps + return ni, nil } -func (cli *SESXiClient) HostVmIPs(ctx context.Context) (map[string]string, []SSimpleVM, error) { - dvpgKeyVlanMap, err := cli.dvpgKeyVlanMap() - if err != nil { - return nil, nil, errors.Wrap(err, "unable to get dvpgKeyVlanMap") +func (cli *SESXiClient) mergeNetworInfo(nInfos []SNetworkInfo) SNetworkInfo { + var vmsLen, vlanIpLen, ipPoolLen int + for i := range nInfos { + vmsLen += len(nInfos[i].VMs) + vlanIpLen += len(nInfos[i].VlanIps) + ipPoolLen += nInfos[i].IPPool.Len() } + ret := SNetworkInfo{ + VMs: make([]SSimpleVM, 0, vmsLen), + VlanIps: make(map[int32][]netutils.IPV4Addr, vlanIpLen), + IPPool: NewIPPool(ipPoolLen), + } + for i := range nInfos { + ret.VMs = append(ret.VMs, nInfos[i].VMs...) + for vlan, ips := range nInfos[i].VlanIps { + ret.VlanIps[vlan] = append(ret.VlanIps[vlan], ips...) + } + ret.IPPool.Merge(&nInfos[i].IPPool) + } + for _, ips := range ret.VlanIps { + sort.Slice(ips, func(i, j int) bool { + return ips[i] < ips[j] + }) + } + return ret +} + +func (cli *SESXiClient) HostVmIPs(ctx context.Context) (SNetworkInfo, error) { var hosts []mo.HostSystem - err = cli.scanAllMObjects(SIMPLE_HOST_PROPS, &hosts) + err := cli.scanAllMObjects(SIMPLE_HOST_PROPS, &hosts) if err != nil { - return nil, nil, errors.Wrap(err, "scanAllMObjects") + return SNetworkInfo{}, errors.Wrap(err, "scanAllMObjects") } - group, ctx := errgroup.WithContext(ctx) - collection := make([][]SSimpleVM, len(hosts)) - for i := range hosts { - j := i - group.Go(func() error { - vmIps, err := cli.vmIPs(&hosts[j], dvpgKeyVlanMap) - if err != nil { - return err - } - collection[j] = vmIps - return nil - }) - } - - hostIps := make(map[string]string, len(hosts)) - for i := range hosts { - // find ip - host := &SHost{SManagedObject: newManagedObject(cli, &hosts[i], nil)} - ip := host.GetAccessIp() - hostIps[host.GetName()] = ip - } - err = group.Wait() - if err != nil { - return nil, nil, err - } - // length - length := 0 - for i := range collection { - length += len(collection[i]) - } - svms := make([]SSimpleVM, 0, length) - for i := range collection { - for j := range collection[i] { - svms = append(svms, collection[i][j]) - } - } - return hostIps, svms, nil + return cli.hostVMIPs(ctx, hosts) } -func (cli *SESXiClient) macVlanMap(movm mo.VirtualMachine, dvpgKeyVlanMap *sync.Map) map[string]int32 { +func (cli *SESXiClient) macVlanMap(mohost *mo.HostSystem, movm *mo.VirtualMachine, dvpgMap sVPGMap) map[string]int32 { + vpgMap := cli.getVPGMap(mohost) ret := make(map[string]int32, 2) for _, device := range movm.Config.Hardware.Device { bcard, ok := device.(types.BaseVirtualEthernetCard) @@ -845,34 +852,52 @@ func (cli *SESXiClient) macVlanMap(movm mo.VirtualMachine, dvpgKeyVlanMap *sync. } card := bcard.GetVirtualEthernetCard() mac := card.MacAddress - dvpgBackInfo, ok := card.Backing.(*types.VirtualEthernetCardDistributedVirtualPortBackingInfo) - if !ok { + switch bk := card.Backing.(type) { + case *types.VirtualEthernetCardDistributedVirtualPortBackingInfo: + key := bk.Port.PortgroupKey + proc, ok := dvpgMap.Get(key) + if !ok { + log.Errorf("dvpg %s not found in key-vlanid map", key) + ret[mac] = 0 + continue + } + ret[mac] = proc.vlanId + case *types.VirtualEthernetCardNetworkBackingInfo: + netName, err := cli.networkName(bk.Network.Value) + if err != nil { + log.Errorf("get getNetworkName of %q: %v", bk.Network.Value, err) + ret[mac] = 0 + continue + } + key := fmt.Sprintf("%s-%s", mohost.Reference().Value, netName) + proc, ok := vpgMap.Get(key) + if !ok { + log.Errorf("vpg %s not found in key-vlanid map", key) + ret[mac] = 0 + continue + } + ret[mac] = proc.vlanId + default: ret[mac] = 0 - continue } - dvpgKey := dvpgBackInfo.Port.PortgroupKey - value, ok := dvpgKeyVlanMap.Load(dvpgKey) - if !ok { - log.Errorf("dvpg %s not found in key-vlanid map", dvpgKey) - ret[mac] = 0 - continue - } - vlanid := value.(int32) - ret[mac] = vlanid } return ret } -func (cli *SESXiClient) vmIPs(host *mo.HostSystem, dvpgKeyVlanMap *sync.Map) ([]SSimpleVM, error) { +func (cli *SESXiClient) vmIPs(host *mo.HostSystem, vpgMap sVPGMap) (SNetworkInfo, error) { + nInfo := SNetworkInfo{ + VMs: make([]SSimpleVM, 0, len(host.Vm)), + VlanIps: make(map[int32][]netutils.IPV4Addr), + IPPool: NewIPPool(), + } if len(host.Vm) == 0 { - return []SSimpleVM{}, nil + return nInfo, nil } var vms []mo.VirtualMachine err := cli.references2Objects(host.Vm, SIMPLE_VM_PROPS, &vms) if err != nil { - return nil, errors.Wrap(err, "references2Objects") + return nInfo, errors.Wrap(err, "references2Objects") } - ret := make([]SSimpleVM, 0, len(vms)) for i := range vms { vm := vms[i] if vm.Config == nil || vm.Config.Template { @@ -881,7 +906,7 @@ func (cli *SESXiClient) vmIPs(host *mo.HostSystem, dvpgKeyVlanMap *sync.Map) ([] if vm.Guest == nil { continue } - macVlanMap := cli.macVlanMap(vm, dvpgKeyVlanMap) + macVlanMap := cli.macVlanMap(host, &vm, vpgMap) guestIps := make([]SIPVlan, 0) for _, net := range vm.Guest.Net { if len(net.Network) == 0 { @@ -899,14 +924,19 @@ func (cli *SESXiClient) vmIPs(host *mo.HostSystem, dvpgKeyVlanMap *sync.Map) ([] if netutils.IsLinkLocal(ipaddr) { continue } + vlan := macVlanMap[mac] guestIps = append(guestIps, SIPVlan{ - IP: ip, - VlanID: macVlanMap[mac], + IP: ipaddr, + VlanId: vlan, + }) + nInfo.VlanIps[vlan] = append(nInfo.VlanIps[vlan], ipaddr) + nInfo.IPPool.Insert(ipaddr, SIPProc{ + VlanId: vlan, }) break } } - ret = append(ret, SSimpleVM{vm.Name, guestIps}) + nInfo.VMs = append(nInfo.VMs, SSimpleVM{vm.Name, guestIps}) } - return ret, nil + return nInfo, nil } diff --git a/pkg/multicloud/esxi/network.go b/pkg/multicloud/esxi/network.go index 409f1180e8..d3b25f4fcd 100644 --- a/pkg/multicloud/esxi/network.go +++ b/pkg/multicloud/esxi/network.go @@ -15,11 +15,15 @@ package esxi import ( + "fmt" + "sync" + "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/netutils" ) type IVMNetwork interface { @@ -261,3 +265,154 @@ func (net *SDistributedVirtualPortgroup) AddHostToDVS(host *SHost) (err error) { } return err } + +type SVirtualSwitch struct { + Name string + Id string + Distributed bool + Hosts []string + Vlans map[int32]SVirtualLan +} + +type SVirtualLan struct { + Vlan int32 + Ips []netutils.IPV4Addr +} + +func (h *SHost) getVirtualSwitchs() []SVirtualSwitch { + hs := h.getHostSystem() + if hs.Config.Network == nil { + return nil + } + vss := make([]SVirtualSwitch, 0, len(hs.Config.Network.Vswitch)) + vsMap := make(map[string]*SVirtualSwitch) + for i := range hs.Config.Network.Vswitch { + ivs := hs.Config.Network.Vswitch[i] + vsId := fmt.Sprintf("%s-%s", h.GetId(), ivs.Name) + vsName := ivs.Name + vs := SVirtualSwitch{ + Name: vsName, + Id: vsId, + Distributed: false, + } + vss = append(vss, vs) + vsMap[ivs.Name] = &vss[len(vss)-1] + } + for i := range hs.Config.Network.Portgroup { + ipg := hs.Config.Network.Portgroup[i] + vlan, vsName := ipg.Spec.VlanId, ipg.Spec.VswitchName + ivs := vsMap[vsName] + ivs.Vlans[vlan] = SVirtualLan{ + Vlan: vlan, + } + } + return vss +} + +func (cli *SESXiClient) getVPGMap(mohost *mo.HostSystem) sVPGMap { + sm := newVPGMap() + if mohost.Config.Network == nil { + return sm + } + for i := range mohost.Config.Network.Portgroup { + ipg := mohost.Config.Network.Portgroup[i] + key := fmt.Sprintf("%s-%s", mohost.Reference().Value, ipg.Spec.Name) + vlan := ipg.Spec.VlanId + sm.Insert(key, sVPGProc{ + vlanId: vlan, + }) + // TODO: fill VSId + } + return sm +} + +func (cli *SESXiClient) getDVPGMap() (sVPGMap, error) { + sm := newVPGMap() + dvpgs, err := cli.scanAllDvPortgroups() + if err != nil { + return sm, err + } + for i := range dvpgs { + key := dvpgs[i].getMODVPortgroup().Key + vlanid := dvpgs[i].GetVlanId() + sm.Insert(key, sVPGProc{ + vlanId: vlanid, + }) + // TODO: fill VSId + } + return sm, nil +} + +func (cli *SESXiClient) getVirtualSwitchs() []SVirtualSwitch { + // TODO + return nil +} + +func newVPGMap() sVPGMap { + return sVPGMap{m: &sync.Map{}} +} + +type sVPGMap struct { + m *sync.Map +} + +func (vm *sVPGMap) Insert(key string, proc sVPGProc) { + vm.m.Store(key, proc) +} + +func (vm *sVPGMap) Get(key string) (sVPGProc, bool) { + v, ok := vm.m.Load(key) + if !ok { + return sVPGProc{}, ok + } + r := v.(sVPGProc) + return r, ok +} + +type sVPGProc struct { + vlanId int32 + dvId string +} + +type SIPProc struct { + VSId string + VlanId int32 +} + +type SIPPool struct { + p map[netutils.IPV4Addr]SIPProc +} + +func NewIPPool(length ...int) SIPPool { + if len(length) > 0 { + return SIPPool{p: make(map[netutils.IPV4Addr]SIPProc, length[0])} + } + return SIPPool{p: make(map[netutils.IPV4Addr]SIPProc)} +} + +func (p *SIPPool) Has(ip netutils.IPV4Addr) bool { + _, ok := p.p[ip] + return ok +} + +func (p *SIPPool) Get(ip netutils.IPV4Addr) (SIPProc, bool) { + r, ok := p.p[ip] + return r, ok +} + +func (p *SIPPool) Insert(ip netutils.IPV4Addr, proc SIPProc) { + p.p[ip] = proc +} + +func (p *SIPPool) Merge(op *SIPPool) { + for ip, proc := range op.p { + if p.Has(ip) { + continue + } + p.Insert(ip, proc) + } +} + +func (p *SIPPool) Len() int { + return len(p.p) +}