From 2ea855a4254cae974de056ae73a4cb7d42f393ab Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Tue, 1 Sep 2020 10:51:41 +0800 Subject: [PATCH] fix: vmware esxi host detects vnic portgroup info (#7649) Co-authored-by: Qiu Jian --- pkg/cloudprovider/resources.go | 1 + pkg/compute/models/hosts.go | 12 +++- pkg/multicloud/esxi/devtools.go | 21 +++++-- pkg/multicloud/esxi/host.go | 103 +++++++++++++++++++++++++++----- pkg/multicloud/esxi/hostnic.go | 26 ++++++++ 5 files changed, 140 insertions(+), 23 deletions(-) diff --git a/pkg/cloudprovider/resources.go b/pkg/cloudprovider/resources.go index 3a01912c63..d70a01353e 100644 --- a/pkg/cloudprovider/resources.go +++ b/pkg/cloudprovider/resources.go @@ -487,6 +487,7 @@ type ICloudHostNetInterface interface { GetIpAddr() string GetMtu() int32 GetNicType() string + GetBridge() string } type ICloudLoadbalancer interface { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index fd4420214b..63d5348d16 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -4638,7 +4638,15 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T enables = append(enables, extNics[i]) } } else { - // do nothing, in sync + // in sync, sync interface and bridge + hw := host.getHostwireOfIdAndMac(netIfs[i].WireId, netIfs[i].Mac) + if hw != nil && (hw.Bridge != extNics[i].GetBridge() || hw.Interface != extNics[i].GetDevice()) { + db.Update(hw, func() error { + hw.Interface = extNics[i].GetDevice() + hw.Bridge = extNics[i].GetBridge() + return nil + }) + } } } else { reserveIp := false @@ -4689,7 +4697,7 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T // always try reserved pool extNic := adds[i].netif err = host.addNetif(ctx, userCred, extNic.GetMac(), "", extNic.GetIpAddr(), 0, extNic.GetNicType(), extNic.GetIndex(), - extNic.IsLinkUp(), int16(extNic.GetMtu()), false, "", "", true, true) + extNic.IsLinkUp(), int16(extNic.GetMtu()), false, extNic.GetDevice(), extNic.GetBridge(), true, true) if err != nil { result.AddError(err) } else { diff --git a/pkg/multicloud/esxi/devtools.go b/pkg/multicloud/esxi/devtools.go index 550f8ebc77..79f191d2e2 100644 --- a/pkg/multicloud/esxi/devtools.go +++ b/pkg/multicloud/esxi/devtools.go @@ -20,6 +20,7 @@ import ( "github.com/vmware/govmomi/vim25/types" + "yunion.io/x/log" "yunion.io/x/pkg/errors" ) @@ -144,15 +145,25 @@ func NewUSBController(key *int32) types.BaseVirtualDevice { return &device } -func NewVNICDev(host *SHost, mac, driver string, vlanId int32, key, ctlKey, index int32) (types.BaseVirtualDevice, error) { +func NewVNICDev(host *SHost, mac, driver string, bridge string, vlanId int32, key, ctlKey, index int32) (types.BaseVirtualDevice, error) { desc := types.Description{Label: fmt.Sprintf("Network adapter %d", index+1), Summary: "VM Network"} - inet, err := host.FindNetworkByVlanID(vlanId) - if err != nil { - return nil, errors.Wrap(err, "SHost.FindNetworkByVlanID") + var inet IVMNetwork + var err error + if (vlanId == 0 || vlanId == 1) && len(bridge) > 0 { + inet, err = host.findDVPGById(bridge) + if err != nil { + log.Errorf("fail to find dvportgroup by name %s: %s", bridge, err) + } + } + if inet == nil { + inet, err = host.FindNetworkByVlanID(vlanId) + if err != nil { + log.Errorf("fail to find network by vlanid %d: %s", vlanId, err) + } } if inet == nil || reflect.ValueOf(inet).IsNil() { - return nil, errors.Error(fmt.Sprintf("VLAN %d not found", vlanId)) + return nil, errors.Error(fmt.Sprintf("Brige %s VLAN %d not found", bridge, vlanId)) } var ( diff --git a/pkg/multicloud/esxi/host.go b/pkg/multicloud/esxi/host.go index 602ed46614..a146e47b5e 100644 --- a/pkg/multicloud/esxi/host.go +++ b/pkg/multicloud/esxi/host.go @@ -31,6 +31,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/utils" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -335,6 +336,24 @@ func (self *SHost) getNicInfo(debug bool) []SHostNicInfo { return self.nicInfo } +func mask2len(mask string) int8 { + maskAddr, _ := netutils.NewIPV4Addr(mask) + return netutils.Mask2Len(maskAddr) +} + +func isVnicAdmin(nic types.HostVirtualNic, defRouteDev string, defGateway string) bool { + if nic.Spec.Portgroup == "Management Network" || nic.Spec.Portgroup == "Service Console" || nic.Device == defRouteDev { + return true + } + if nic.Spec.IpRouteSpec != nil { + hostRouteConf := nic.Spec.IpRouteSpec.IpRouteConfig.GetHostIpRouteConfig() + if hostRouteConf != nil && len(hostRouteConf.DefaultGateway) > 0 && hostRouteConf.DefaultGateway == defGateway { + return true + } + } + return false +} + func (self *SHost) fetchNicInfo(debug bool) []SHostNicInfo { moHost := self.getHostSystem() @@ -365,22 +384,67 @@ func (self *SHost) fetchNicInfo(debug bool) []SHostNicInfo { if len(moHost.Config.Network.ConsoleVnic) > 0 { vnics = append(vnics, moHost.Config.Network.ConsoleVnic...) } - findMaster := false + // find default route vnic + defRouteDevs := make([]string, 0) + for _, r := range moHost.Config.Network.RouteTableInfo.IpRoute { + if r.Network == "0.0.0.0" && r.PrefixLength == 0 && !utils.IsInStringArray(r.DeviceName, defRouteDevs) { + // default route + defRouteDevs = append(defRouteDevs, r.DeviceName) + } + } + var defRouteDev string + if len(defRouteDevs) > 0 { + defRouteDev = defRouteDevs[0] + } + var defGateway string + if moHost.Config.Network.IpRouteConfig.GetHostIpRouteConfig() != nil { + defGateway = moHost.Config.Network.IpRouteConfig.GetHostIpRouteConfig().DefaultGateway + } + // findMaster := false for _, nic := range vnics { mac := netutils.FormatMacAddr(nic.Spec.Mac) pnic := findHostNicByMac(nicInfoList, mac) if pnic != nil { - findMaster = true + // findMaster = true pnic.IpAddr = nic.Spec.Ip.IpAddress - if nic.Spec.Portgroup == "Management Network" || nic.Spec.Portgroup == "Service Console" { + pnic.IpAddrPrefixLen = mask2len(nic.Spec.Ip.SubnetMask) + if nic.Spec.Ip.IpV6Config != nil && len(nic.Spec.Ip.IpV6Config.IpV6Address) > 0 { + pnic.IpAddr6 = nic.Spec.Ip.IpV6Config.IpV6Address[0].IpAddress + pnic.IpAddr6PrefixLen = int8(nic.Spec.Ip.IpV6Config.IpV6Address[0].PrefixLength) + } + if isVnicAdmin(nic, defRouteDev, defGateway) { pnic.NicType = api.NIC_TYPE_ADMIN } pnic.LinkUp = true pnic.Mtu = nic.Spec.Mtu + if nic.Spec.DistributedVirtualPort != nil { + pnic.DVPortGroup = nic.Spec.DistributedVirtualPort.PortgroupKey + } + } else { + info := SHostNicInfo{} + info.Dev = nic.Device + info.Driver = "vmkernel" + info.Mac = mac + info.Index = int8(len(nicInfoList)) + info.LinkUp = true + info.IpAddr = nic.Spec.Ip.IpAddress + info.IpAddrPrefixLen = mask2len(nic.Spec.Ip.SubnetMask) + if nic.Spec.Ip.IpV6Config != nil && len(nic.Spec.Ip.IpV6Config.IpV6Address) > 0 { + info.IpAddr6 = nic.Spec.Ip.IpV6Config.IpV6Address[0].IpAddress + info.IpAddr6PrefixLen = int8(nic.Spec.Ip.IpV6Config.IpV6Address[0].PrefixLength) + } + info.Mtu = nic.Spec.Mtu + if isVnicAdmin(nic, defRouteDev, defGateway) { + info.NicType = api.NIC_TYPE_ADMIN + } + if nic.Spec.DistributedVirtualPort != nil { + info.DVPortGroup = nic.Spec.DistributedVirtualPort.PortgroupKey + } + nicInfoList = append(nicInfoList, info) } } - if !findMaster && len(nicInfoList) > 0 { + /*if !findMaster && len(nicInfoList) > 0 { // no match pnic found for master nic // choose the first pnic pnic := &nicInfoList[0] @@ -394,17 +458,9 @@ func (self *SHost) fetchNicInfo(debug bool) []SHostNicInfo { } } if len(pnic.IpAddr) == 0 { - // find default route vnic - defRouteDev := make([]string, 0) - for _, r := range moHost.Config.Network.RouteTableInfo.IpRoute { - if r.Network == "0.0.0.0" && r.PrefixLength == 0 { - // default route - defRouteDev = append(defRouteDev, r.DeviceName) - } - } if len(defRouteDev) == 1 { for _, nic := range vnics { - if nic.Device == defRouteDev[0] { + if nic.Device == defRouteDev { pnic.NicType = api.NIC_TYPE_ADMIN pnic.IpAddr = nic.Spec.Ip.IpAddress pnic.LinkUp = true @@ -416,7 +472,7 @@ func (self *SHost) fetchNicInfo(debug bool) []SHostNicInfo { log.Errorf("find default route interfaces fail: %s", defRouteDev) } } - } + }*/ return nicInfoList } @@ -819,6 +875,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, params SCreat for _, nic := range nics { index, _ := nic.Int("index") mac, _ := nic.GetString("mac") + bridge, _ := nic.GetString("bridge") driver := "e1000" if nic.Contains("driver") { driver, _ = nic.GetString("driver") @@ -830,7 +887,7 @@ func (self *SHost) DoCreateVM(ctx context.Context, ds *SDatastore, params SCreat if nic.Contains("vlan") { vlanId, _ = nic.Int("vlan") } - dev, err := NewVNICDev(self, mac, driver, int32(vlanId), 4000, 100, int32(index)) + dev, err := NewVNICDev(self, mac, driver, bridge, int32(vlanId), 4000, 100, int32(index)) if err != nil { return nil, errors.Wrap(err, "NewVNICDev") } @@ -888,6 +945,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData for _, nic := range nics { index, _ := nic.Int("index") mac, _ := nic.GetString("mac") + bridge, _ := nic.GetString("bridge") driver := "e1000" if nic.Contains("driver") { driver, _ = nic.GetString("driver") @@ -899,7 +957,7 @@ func (host *SHost) CloneVM(ctx context.Context, from *SVirtualMachine, ds *SData if nic.Contains("vlan") { vlanId, _ = nic.Int("vlan") } - dev, err := NewVNICDev(host, mac, driver, int32(vlanId), 4000, 100, int32(index)) + dev, err := NewVNICDev(host, mac, driver, bridge, int32(vlanId), 4000, 100, int32(index)) if err != nil { return nil, errors.Wrap(err, "NewVNICDev") } @@ -1330,6 +1388,19 @@ func (host *SHost) findNovlanDVPG() (*SDistributedVirtualPortgroup, error) { return nil, nil } +func (host *SHost) findDVPGById(id string) (*SDistributedVirtualPortgroup, error) { + nets, err := host.datacenter.GetNetworks() + if err != nil { + return nil, errors.Wrap(err, "SHost.datacenter.GetNetworks") + } + for _, net := range nets { + if dvpg, ok := net.(*SDistributedVirtualPortgroup); ok && dvpg.GetId() == id { + return dvpg, nil + } + } + return nil, nil +} + func (host *SHost) findVlanDVPG(vlanId int32) (*SDistributedVirtualPortgroup, error) { nets, err := host.datacenter.GetNetworks() if err != nil { diff --git a/pkg/multicloud/esxi/hostnic.go b/pkg/multicloud/esxi/hostnic.go index 4b6885178e..737cdc32ee 100644 --- a/pkg/multicloud/esxi/hostnic.go +++ b/pkg/multicloud/esxi/hostnic.go @@ -25,8 +25,14 @@ type SHostNicInfo struct { Index int8 LinkUp bool IpAddr string + IpAddr6 string Mtu int32 NicType string + + DVPortGroup string + + IpAddrPrefixLen int8 + IpAddr6PrefixLen int8 } func (nic *SHostNicInfo) GetDevice() string { @@ -56,6 +62,18 @@ func (nic *SHostNicInfo) GetIpAddr() string { return nic.IpAddr } +func (nic *SHostNicInfo) GetIpAddrPrefixLen() int8 { + return nic.IpAddrPrefixLen +} + +func (nic *SHostNicInfo) GetIpAddr6() string { + return nic.IpAddr6 +} + +func (nic *SHostNicInfo) GetIpAddr6PrefixLen() int8 { + return nic.IpAddr6PrefixLen +} + func (nic *SHostNicInfo) GetMtu() int32 { return nic.Mtu } @@ -63,3 +81,11 @@ func (nic *SHostNicInfo) GetMtu() int32 { func (nic *SHostNicInfo) GetNicType() string { return nic.NicType } + +func (nic *SHostNicInfo) GetDVPortGroup() string { + return nic.DVPortGroup +} + +func (nic *SHostNicInfo) GetBridge() string { + return nic.DVPortGroup +}