From ed484f4e21c8d059d4f7ed87e33652053d267b1a Mon Sep 17 00:00:00 2001 From: rainzm Date: Mon, 24 Aug 2020 15:46:43 +0800 Subject: [PATCH] refactor(region): optimize preparenet of cloudaccount MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 从vcenter中获取host和vm的信息时，只需要获取网络相关的字段就可以了。 2. 每个host的vms，单独启用一个goroutine。 3. 优化了处理vms ip的逻辑，现在的时间复杂度是O(n)。 --- pkg/compute/models/cloudaccounts.go | 238 ++++++++++++++++++---------- pkg/multicloud/esxi/ip.go | 92 +++++++++++ pkg/multicloud/esxi/manager.go | 82 ++++++++++ pkg/multicloud/esxi/shell/ip.go | 55 +++++++ 4 files changed, 385 insertions(+), 82 deletions(-) create mode 100644 pkg/multicloud/esxi/ip.go create mode 100644 pkg/multicloud/esxi/shell/ip.go diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index a233502b55..8bd2c619f3 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -49,6 +49,7 @@ import ( "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/multicloud/esxi" "yunion.io/x/onecloud/pkg/multicloud/esxi/vcenter" "yunion.io/x/onecloud/pkg/util/choices" "yunion.io/x/onecloud/pkg/util/httputils" @@ -418,9 +419,11 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre if err != nil { return output, errors.Wrap(err, "provider.GetOnPremiseIRegion") } - hosts, err := iregion.GetIHosts() + // hack + client := iregion.(*esxi.SESXiClient) + hostIps, simpleVms, err := client.HostVmIPs(ctx) if err != nil { - return output, errors.Wrap(err, "unable to get hosts") + return output, errors.Wrap(err, "unable to fetch ips of hosts and vms") } // fetch networks networks := make([][]SNetwork, len(wires)) @@ -431,20 +434,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre } networks[i] = nets } - // fetch host ips + // key of ipHosts is host's ip - ipHosts := make(map[netutils.IPV4Addr]cloudprovider.ICloudHost, len(hosts)) - for i := range hosts { - ip := hosts[i].GetAccessIp() - if len(ip) == 0 { - log.Errorf("unable to get accessip of host %q", hosts[i].GetName()) - continue - } + ipHosts := make(map[netutils.IPV4Addr]string, len(hostIps)) + for name, ip := range hostIps { addr, err := netutils.NewIPV4Addr(ip) if err != nil { return output, err } - ipHosts[addr] = hosts[i] + ipHosts[addr] = name } // Find suitable wire and the network containing the Host IP in suitable wire. var ( @@ -491,15 +489,15 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre // Give the suggested network configuration for the Host IP that does not have a corresponding suitable network. noNetHostIP := make([]netutils.IPV4Addr, 0, len(ipHosts)) - for hip, host := range ipHosts { + for ip, name := range ipHosts { rnet := api.CAHostNet{ - Name: host.GetName(), - IP: host.GetAccessIp(), + Name: name, + IP: ip.String(), } - if net, ok := suitableNetworks[hip]; ok { + if net, ok := suitableNetworks[ip]; ok { rnet.SuitableNetwork = net.GetId() } else { - noNetHostIP = append(noNetHostIP, hip) + noNetHostIP = append(noNetHostIP, ip) } output.Hosts = append(output.Hosts, rnet) } @@ -516,7 +514,7 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre // Find the suitable network containing the VM IP in Project 'input.Project', and if not, give the corresponding suggested network configuration in this project. project := input.ProjectId - nets := []*SNetwork{} + excludeNets := []*SNetwork{} var allNets []SNetwork if suitableWire != nil { allNets, err = suitableWire.getNetworks(userCred, rbacutils.ScopeSystem) @@ -525,85 +523,161 @@ func (scm *SCloudaccountManager) PerformPrepareNets(ctx context.Context, userCre } for i := range allNets { if allNets[i].ProjectId == project { - nets = append(nets, &allNets[i]) + excludeNets = append(excludeNets, &allNets[i]) } } } - // find suitable network - vms := make([]cloudprovider.ICloudVM, 0, len(hosts)) - for _, host := range hosts { - vs, err := host.GetIVMs() - if err != nil { - return output, errors.Wrapf(err, "unable to get VMs of host %q", host.GetName()) - } - vms = append(vms, vs...) + + type vm struct { + FakeID int + IP netutils.IPV4Addr + Name string } - ipVMs := make(map[netutils.IPV4Addr]int, len(vms)) - for i, vm := range vms { - nics, err := vm.GetINics() - if err != nil { - return output, errors.Wrapf(err, "unable get nics of vm %q", vm.GetName()) - } - for _, nic := range nics { - ipStr := nic.GetIP() - if len(ipStr) == 0 { - continue - } - ip, err := netutils.NewIPV4Addr(ipStr) - if err != nil { - return output, errors.Wrapf(err, "unable to new IPV4Addr for ip %q", nic.GetIP()) - } - ipVMs[ip] = i - } - output.Guests = append(output.Guests, api.CAGuestNet{ - Name: vm.GetName(), - IPNets: make([]api.CAIPNet, 0, 1), - }) - } - suitableNetworks = make(map[netutils.IPV4Addr]*SNetwork) - if len(nets) != 0 { - for vip := range ipVMs { - for i := range nets { - ipRange := nets[i].GetIPRange() - if !ipRange.Contains(vip) { - continue + vms := make([]vm, 0, len(simpleVms)) + var nip netutils.IPV4Addr + guestMap := make(map[int]*api.CAGuestNet) + for i := range simpleVms { + id := i + if len(simpleVms[i].IPs) == 0 { + if _, ok := guestMap[id]; !ok { + guestMap[id] = &api.CAGuestNet{ + Name: simpleVms[i].Name, + IPNets: []api.CAIPNet{}, } - suitableNetworks[vip] = nets[i] - break } } - } - - noNetVMIPs := make([]netutils.IPV4Addr, 0, len(ipVMs)) - for vip, i := range ipVMs { - ipnet := api.CAIPNet{ - IP: vip.String(), + for _, ip := range simpleVms[i].IPs { + nip, err = netutils.NewIPV4Addr(ip) + if err != nil { + return output, err + } + vms = append(vms, vm{ + FakeID: i, + IP: nip, + Name: simpleVms[i].Name, + }) } - if net, ok := suitableNetworks[vip]; ok { - ipnet.SuitableNetwork = net.GetId() - } else { - noNetVMIPs = append(noNetVMIPs, vip) - } - output.Guests[i].IPNets = append(output.Guests[ipVMs[vip]].IPNets, ipnet) } + // sort vms via vm's ip + sort.Slice(vms, func(i, j int) bool { + return vms[i].IP < vms[j].IP + }) - // excludedIRs is to prevent network conflicts - excludedIRs := make([]netutils.IPV4AddrRange, len(allNets)) - for i := range excludedIRs { - excludedIRs[i] = allNets[i].GetIPRange() + excludeNets = make([]*SNetwork, 0, len(allNets)+len(output.HostSuggestedNetworks)) + for i := range allNets { + excludeNets = append(excludeNets, &allNets[i]) } for i := range output.HostSuggestedNetworks { - startIPStr := output.HostSuggestedNetworks[i].GuestIpStart - endIPStr := output.HostSuggestedNetworks[i].GuestIpEnd - startIP, _ := netutils.NewIPV4Addr(startIPStr) - endIP, _ := netutils.NewIPV4Addr(endIPStr) - excludedIRs = append(excludedIRs, netutils.NewIPV4AddrRange(startIP, endIP)) + startipStr := output.HostSuggestedNetworks[i].GuestIpStart + endipStr := output.HostSuggestedNetworks[i].GuestIpEnd + excludeNets = append(excludeNets, &SNetwork{GuestIpEnd: endipStr, GuestIpStart: startipStr}) } - if len(noNetVMIPs) > 0 { - sConfs := scm.suggestVMNetwors(noNetVMIPs, excludedIRs) - confs := make([]api.CANetConf, len(sConfs)) + // sort excludeNets via their GuestIpStart + sort.Slice(excludeNets, func(i, j int) bool { + return excludeNets[i].GuestIpStart < excludeNets[j].GuestIpStart + }) + + svNets := make([]netutils.IPV4AddrRange, 0, 5) + var vmi, neti int + + lastEndIp := netutils.IPV4Addr(0) +Loop: + for neti = 0; neti < len(excludeNets); { + startIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpStart) + endIp, _ := netutils.NewIPV4Addr(excludeNets[neti].GuestIpEnd) + switch { + case vms[vmi].IP > endIp: + lastEndIp = endIp + neti++ + case vms[vmi].IP >= startIp: + for vms[vmi].IP <= endIp { + id := vms[vmi].FakeID + if _, ok := guestMap[id]; !ok { + guestMap[id] = &api.CAGuestNet{ + Name: vms[vmi].Name, + IPNets: []api.CAIPNet{}, + } + } + guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{ + IP: vms[vmi].IP.String(), + SuitableNetwork: excludeNets[neti].Id, + }) + vmi += 1 + if vmi == len(vms) { + break Loop + } + } + lastEndIp = endIp + neti++ + default: + for vms[vmi].IP < startIp { + suggestStartIp := vms[vmi].IP.NetAddr(24) + 1 + if suggestStartIp <= lastEndIp { + suggestStartIp = lastEndIp + 1 + } + if suggestStartIp >= startIp { + break + } + suggestEndIp := suggestStartIp.NetAddr(24) + 255 + if suggestEndIp >= startIp { + suggestEndIp = startIp - 1 + } + svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp)) + for vmi < len(vms) && vms[vmi].IP <= suggestEndIp { + id := vms[vmi].FakeID + if _, ok := guestMap[id]; !ok { + guestMap[id] = &api.CAGuestNet{ + Name: vms[vmi].Name, + IPNets: []api.CAIPNet{}, + } + } + guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{ + IP: vms[vmi].IP.String(), + }) + vmi++ + } + if vmi == len(vms) { + break Loop + } + } + } + } + + for vmi < len(vms) { + suggestStartIp := vms[vmi].IP.NetAddr(24) + 1 + if suggestStartIp <= lastEndIp { + suggestStartIp = lastEndIp + 1 + } + suggestEndIp := suggestStartIp.NetAddr(24) + 255 + svNets = append(svNets, netutils.NewIPV4AddrRange(suggestStartIp, suggestEndIp)) + for vmi < len(vms) && vms[vmi].IP <= suggestEndIp { + id := vms[vmi].FakeID + if _, ok := guestMap[id]; !ok { + guestMap[id] = &api.CAGuestNet{ + Name: vms[vmi].Name, + IPNets: []api.CAIPNet{}, + } + } + guestMap[id].IPNets = append(guestMap[id].IPNets, api.CAIPNet{ + IP: vms[vmi].IP.String(), + }) + vmi++ + } + } + + for _, guest := range guestMap { + output.Guests = append(output.Guests, *guest) + } + + if len(svNets) > 0 { + confs := make([]api.CANetConf, len(svNets)) for i := range confs { - confs[i].CASimpleNetConf = sConfs[i] + confs[i].CASimpleNetConf = api.CASimpleNetConf{ + GuestIpStart: svNets[i].StartIp().String(), + GuestIpEnd: svNets[i].EndIp().String(), + GuestIpMask: 24, + GuestGateway: (svNets[i].StartIp().NetAddr(24) + 1).String(), + } confs[i].Name = fmt.Sprintf("%s-guest-network-%d", input.Name, i+1) } output.GuestSuggestedNetworks = confs diff --git a/pkg/multicloud/esxi/ip.go b/pkg/multicloud/esxi/ip.go new file mode 100644 index 0000000000..44863462d9 --- /dev/null +++ b/pkg/multicloud/esxi/ip.go @@ -0,0 +1,92 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esxi + +import ( + "github.com/vmware/govmomi/vim25/mo" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/regutils" +) + +var HOST_PROPS = []string{"name", "config.network", "vm"} + +var VM_PROPS = []string{"name", "guest.net", "config.template"} + +func (cli *SESXiClient) AllHostIP() (map[string]string, []mo.HostSystem, error) { + var hosts []mo.HostSystem + err := cli.scanAllMObjects(HOST_PROPS, &hosts) + if err != nil { + return nil, nil, errors.Wrap(err, "scanAllMObjects") + } + ret := make(map[string]string, len(hosts)) + for i := range hosts { + // find ip + host := &SHost{SManagedObject: newManagedObject(cli, &hosts[i], nil)} + ip := host.GetAccessIp() + ret[host.GetName()] = ip + } + return ret, hosts, nil +} + +func (cli *SESXiClient) VMIP(host mo.HostSystem) (map[string][]string, error) { + var vms []mo.VirtualMachine + err := cli.references2Objects(host.Vm, VM_PROPS, &vms) + if err != nil { + return nil, errors.Wrap(err, "references2Objects") + } + ret := make(map[string][]string, len(vms)) + for i := range vms { + vm := vms[i] + if vm.Config.Template { + continue + } + guestIps := make([]string, 0) + for _, net := range vm.Guest.Net { + for _, ip := range net.IpAddress { + if regutils.MatchIP4Addr(ip) { + guestIps = append(guestIps, ip) + } + } + } + ret[vm.Name] = guestIps + } + return ret, nil +} + +func (cli *SESXiClient) VMIP2() (map[string][]string, error) { + var vms []mo.VirtualMachine + err := cli.scanAllMObjects(VM_PROPS, &vms) + if err != nil { + return nil, errors.Wrap(err, "scanAllMObjects") + } + ret := make(map[string][]string, len(vms)) + for i := range vms { + vm := vms[i] + if vm.Config.Template { + continue + } + guestIps := make([]string, 0) + for _, net := range vm.Guest.Net { + for _, ip := range net.IpAddress { + if regutils.MatchIP4Addr(ip) { + guestIps = append(guestIps, ip) + } + } + } + ret[vm.Name] = guestIps + } + return ret, nil +} diff --git a/pkg/multicloud/esxi/manager.go b/pkg/multicloud/esxi/manager.go index 073c83ad67..362ebee75a 100644 --- a/pkg/multicloud/esxi/manager.go +++ b/pkg/multicloud/esxi/manager.go @@ -31,10 +31,12 @@ import ( "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + "golang.org/x/sync/errgroup" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/regutils" "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" @@ -597,3 +599,83 @@ func (cli *SESXiClient) MoveDisk(ctx context.Context, src, dst string, isForce b } return task.Wait(ctx) } + +var ( + SIMPLE_HOST_PROPS = []string{"name", "config.network", "vm"} + SIMPLE_VM_PROPS = []string{"name", "guest.net", "config.template"} +) + +type SSimpleVM struct { + Name string + IPs []string +} + +func (cli *SESXiClient) HostVmIPs(ctx context.Context) (map[string]string, []SSimpleVM, error) { + var hosts []mo.HostSystem + err := cli.scanAllMObjects(SIMPLE_HOST_PROPS, &hosts) + if err != nil { + return nil, nil, errors.Wrap(err, "scanAllMObjects") + } + group, ctx := errgroup.WithContext(ctx) + collection := make([][]SSimpleVM, len(hosts)) + for i := range hosts { + j := i + group.Go(func() error { + vmIps, err := cli.vmIPs(&hosts[j]) + if err != nil { + return err + } + collection[j] = vmIps + return nil + }) + } + + hostIps := make(map[string]string, len(hosts)) + for i := range hosts { + // find ip + host := &SHost{SManagedObject: newManagedObject(cli, &hosts[i], nil)} + ip := host.GetAccessIp() + hostIps[host.GetName()] = ip + } + err = group.Wait() + if err != nil { + return nil, nil, err + } + // length + length := 0 + for i := range collection { + length += len(collection[i]) + } + svms := make([]SSimpleVM, 0, length) + for i := range collection { + for j := range collection[i] { + svms = append(svms, collection[i][j]) + } + } + return hostIps, svms, nil +} + +func (cli *SESXiClient) vmIPs(host *mo.HostSystem) ([]SSimpleVM, error) { + var vms []mo.VirtualMachine + err := cli.references2Objects(host.Vm, VM_PROPS, &vms) + if err != nil { + return nil, errors.Wrap(err, "references2Objects") + } + ret := make([]SSimpleVM, 0, len(vms)) + for i := range vms { + vm := vms[i] + if vm.Config.Template { + continue + } + guestIps := make([]string, 0) + for _, net := range vm.Guest.Net { + for _, ip := range net.IpAddress { + if regutils.MatchIP4Addr(ip) { + guestIps = append(guestIps, ip) + } + } + } + ret = append(ret, SSimpleVM{vm.Name, guestIps}) + } + return ret, nil +} diff --git a/pkg/multicloud/esxi/shell/ip.go b/pkg/multicloud/esxi/shell/ip.go new file mode 100644 index 0000000000..027eb4e15f --- /dev/null +++ b/pkg/multicloud/esxi/shell/ip.go @@ -0,0 +1,55 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shell + +import ( + "fmt" + + "yunion.io/x/onecloud/pkg/multicloud/esxi" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + type IPOption struct { + Host bool + } + shellutils.R(&IPOption{}, "ip-all", "List all ip", func(cli *esxi.SESXiClient, args *IPOption) error { + hostIps, _, err := cli.AllHostIP() + if err != nil { + return err + } + for name, ip := range hostIps { + fmt.Printf("name: %s, ip: %s\n", name, ip) + } + // for i := range hosts { + // fmt.Printf("host %s: \n", hosts[i].Name) + // vmips, err := cli.VMIP(hosts[i]) + // if err != nil { + // return err + // } + // for name, ip := range vmips { + // fmt.Printf("\tname: %s, ip: %s\n", name, ip) + // } + // } + vmips, err := cli.VMIP2() + if err != nil { + return err + } + for name, ip := range vmips { + fmt.Printf("\tname: %s, ip: %s\n", name, ip) + } + return nil + }) +}