From 152b00d2d5d5d33f5cc10375ce9cbc7e961278ec Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Fri, 24 May 2024 12:20:39 +0800 Subject: [PATCH] fix: support host local network (#20349) Co-authored-by: Qiu Jian --- pkg/apis/compute/api.go | 3 +- pkg/apis/compute/host.go | 2 +- pkg/apis/compute/network.go | 7 +- pkg/apis/compute/network_const.go | 27 +++- pkg/apis/compute/vpc.go | 3 +- pkg/baremetal/manager.go | 8 +- pkg/baremetal/pxe/dhcp.go | 2 +- pkg/baremetal/pxe/pxe.go | 3 +- pkg/cloudcommon/cmdline/parser.go | 4 +- pkg/compute/guestdrivers/baremetals.go | 6 +- .../guestdrivers/cloudpods-baremetals.go | 6 +- pkg/compute/guestdrivers/kvm.go | 4 + pkg/compute/guestdrivers/pod.go | 4 +- pkg/compute/guestdrivers/virtualization.go | 6 +- pkg/compute/models/cloudaccount_vmware.go | 2 +- pkg/compute/models/guestdrivers.go | 2 +- pkg/compute/models/hosts.go | 10 +- pkg/compute/models/networks.go | 12 +- pkg/compute/models/wires.go | 10 +- pkg/hostman/host_services.go | 2 +- pkg/hostman/hostinfo/container.go | 78 ++++++++++ pkg/hostman/hostinfo/finalize_network.go | 137 ++++++++++++++++++ pkg/hostman/hostinfo/hostinfo.go | 69 ++------- .../algorithm/predicates/network_predicate.go | 12 +- .../predicates/network_schedtag_predicate.go | 6 +- 25 files changed, 309 insertions(+), 116 deletions(-) create mode 100644 pkg/hostman/hostinfo/container.go create mode 100644 pkg/hostman/hostinfo/finalize_network.go diff --git a/pkg/apis/compute/api.go b/pkg/apis/compute/api.go index 2b90f0b2e6..68d5f61004 100644 --- a/pkg/apis/compute/api.go +++ b/pkg/apis/compute/api.go @@ -94,11 +94,12 @@ type NetworkConfig struct { BwLimit int `json:"bw_limit"` Vip bool `json:"vip"` Reserved bool `json:"reserved"` - NetType string `json:"net_type"` NumQueues int `json:"num_queues"` RxTrafficLimit int64 `json:"rx_traffic_limit"` TxTrafficLimit int64 `json:"tx_traffic_limit"` + NetType TNetworkType `json:"net_type"` + IsDefault bool `json:"is_default"` // sriov nic diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index e5a8c8147c..ceb5a56232 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -600,7 +600,7 @@ type HostEnableNetifInput struct { AllocDir string `json:"alloc_dir"` - NetType string `json:"net_type"` + NetType TNetworkType `json:"net_type"` Reserve *bool `json:"reserve"` diff --git a/pkg/apis/compute/network.go b/pkg/apis/compute/network.go index 463d990925..9cdccdfead 100644 --- a/pkg/apis/compute/network.go +++ b/pkg/apis/compute/network.go @@ -253,9 +253,9 @@ type NetworkCreateInput struct { Vpc string `json:"vpc"` // description: server type - // enum: guest,baremetal,pxe,ipmi + // enum: guest,baremetal,pxe,ipmi,hostlocal // default: guest - ServerType string `json:"server_type"` + ServerType TNetworkType `json:"server_type"` // 是否加入自动分配地址池 IsAutoAlloc *bool `json:"is_auto_alloc"` @@ -420,8 +420,9 @@ type NetworkTryCreateNetworkInput struct { Ip string `json:"ip"` Mask int `json:"mask"` - ServerType string `json:"server_type"` IsOnPremise bool `json:"is_on_premise"` + + ServerType TNetworkType `json:"server_type"` } type NetworkSyncInput struct { diff --git a/pkg/apis/compute/network_const.go b/pkg/apis/compute/network_const.go index d26da63ab3..a289b9b461 100644 --- a/pkg/apis/compute/network_const.go +++ b/pkg/apis/compute/network_const.go @@ -20,18 +20,21 @@ import ( "yunion.io/x/cloudmux/pkg/apis/compute" ) +type TNetworkType string + const ( // # DEFAULT_BANDWIDTH = options.default_bandwidth // in mbps, maximal is 100Tbps MAX_BANDWIDTH = 100000000 - NETWORK_TYPE_GUEST = compute.NETWORK_TYPE_GUEST - NETWORK_TYPE_BAREMETAL = "baremetal" - NETWORK_TYPE_CONTAINER = "container" - NETWORK_TYPE_PXE = "pxe" - NETWORK_TYPE_IPMI = "ipmi" - NETWORK_TYPE_EIP = "eip" + NETWORK_TYPE_GUEST = TNetworkType(compute.NETWORK_TYPE_GUEST) + NETWORK_TYPE_BAREMETAL = TNetworkType("baremetal") + NETWORK_TYPE_CONTAINER = TNetworkType("container") + NETWORK_TYPE_PXE = TNetworkType("pxe") + NETWORK_TYPE_IPMI = TNetworkType("ipmi") + NETWORK_TYPE_EIP = TNetworkType("eip") + NETWORK_TYPE_HOSTLOCAL = TNetworkType("hostlocal") STATIC_ALLOC = "static" @@ -59,13 +62,14 @@ const ( ) var ( - ALL_NETWORK_TYPES = []string{ + ALL_NETWORK_TYPES = []TNetworkType{ NETWORK_TYPE_GUEST, NETWORK_TYPE_BAREMETAL, NETWORK_TYPE_CONTAINER, NETWORK_TYPE_PXE, NETWORK_TYPE_IPMI, NETWORK_TYPE_EIP, + NETWORK_TYPE_HOSTLOCAL, } REGIONAL_NETWORK_PROVIDERS = []string{ @@ -82,6 +86,15 @@ var ( } ) +func IsInNetworkTypes(netType TNetworkType, types []TNetworkType) bool { + for _, t := range types { + if t == netType { + return true + } + } + return false +} + type IPAllocationDirection string const ( diff --git a/pkg/apis/compute/vpc.go b/pkg/apis/compute/vpc.go index 1952d57c04..6d5f2962cd 100644 --- a/pkg/apis/compute/vpc.go +++ b/pkg/apis/compute/vpc.go @@ -158,9 +158,10 @@ type NetworkTopologyOutput struct { GuestIpStart string `json:"guest_ip_start"` GuestIpEnd string `json:"guest_ip_end"` GuestIpMask int8 `json:"guest_ip_mask"` - ServerType string `json:"server_type"` VlanId int `json:"vlan_id"` + ServerType TNetworkType `json:"server_type"` + GetNetworkAddressesOutput // Address []SNetworkUsedAddress `json:"address"` } diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index aebf288f5d..29a57e9e8d 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -1298,7 +1298,7 @@ func (b *SBaremetalInstance) InitAdminNetif( cliMac net.HardwareAddr, wireId string, nicType compute.TNicType, - netType string, + netType api.TNetworkType, isDoImport bool, importIpAddr string, ) error { @@ -1357,7 +1357,7 @@ func (b *SBaremetalInstance) attachWire(mac net.HardwareAddr, wireId string, nic return modules.Hosts.PerformAction(session, b.GetId(), "add-netif", params) } -func (b *SBaremetalInstance) postAttachWire(mac net.HardwareAddr, nicType compute.TNicType, netType string, ipAddr string) error { +func (b *SBaremetalInstance) postAttachWire(mac net.HardwareAddr, nicType compute.TNicType, netType api.TNetworkType, ipAddr string) error { if ipAddr == "" { switch nicType { case api.NIC_TYPE_IPMI: @@ -1379,7 +1379,7 @@ func (b *SBaremetalInstance) postAttachWire(mac net.HardwareAddr, nicType comput return b.SaveDesc(desc) } -func (b *SBaremetalInstance) enableWire(mac net.HardwareAddr, ipAddr string, nicType compute.TNicType, netType string) (jsonutils.JSONObject, error) { +func (b *SBaremetalInstance) enableWire(mac net.HardwareAddr, ipAddr string, nicType compute.TNicType, netType api.TNetworkType) (jsonutils.JSONObject, error) { session := b.manager.GetClientSession() params := jsonutils.NewDict() params.Add(jsonutils.NewString(mac.String()), "mac") @@ -1394,7 +1394,7 @@ func (b *SBaremetalInstance) enableWire(mac net.HardwareAddr, ipAddr string, nic params.Add(jsonutils.NewString("stepup"), "alloc_dir") // alloc bottom up } if len(netType) > 0 { - params.Add(jsonutils.NewString(netType), "net_type") + params.Add(jsonutils.NewString(string(netType)), "net_type") } log.Infof("enable net if params: %s", params.String()) return modules.Hosts.PerformAction(session, b.GetId(), "enable-netif", params) diff --git a/pkg/baremetal/pxe/dhcp.go b/pkg/baremetal/pxe/dhcp.go index 5484021d73..f87115a19b 100644 --- a/pkg/baremetal/pxe/dhcp.go +++ b/pkg/baremetal/pxe/dhcp.go @@ -299,7 +299,7 @@ func (req *dhcpRequest) findNetworkConf(session *mcclient.ClientSession, filterU idx := 0 for i := range ret.Data { netType, _ := ret.Data[i].GetString("server_type") - if netType == api.NETWORK_TYPE_PXE { + if netType == string(api.NETWORK_TYPE_PXE) { idx = i break } diff --git a/pkg/baremetal/pxe/pxe.go b/pkg/baremetal/pxe/pxe.go index 43e485b13e..dc328c2539 100644 --- a/pkg/baremetal/pxe/pxe.go +++ b/pkg/baremetal/pxe/pxe.go @@ -21,6 +21,7 @@ import ( "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/jsonutils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/dhcp" @@ -93,7 +94,7 @@ type IBaremetalInstance interface { GetIPMINic(cliMac net.HardwareAddr) *types.SNic GetPXEDHCPConfig(arch uint16) (*dhcp.ResponseConfig, error) GetDHCPConfig(cliMac net.HardwareAddr) (*dhcp.ResponseConfig, error) - InitAdminNetif(cliMac net.HardwareAddr, wireId string, nicType compute.TNicType, netType string, isDoImport bool, ipAddr string) error + InitAdminNetif(cliMac net.HardwareAddr, wireId string, nicType compute.TNicType, netType computeapi.TNetworkType, isDoImport bool, ipAddr string) error RegisterNetif(cliMac net.HardwareAddr, wireId string) error GetTFTPResponse() string } diff --git a/pkg/cloudcommon/cmdline/parser.go b/pkg/cloudcommon/cmdline/parser.go index 0e4a3c966a..fd66049a60 100644 --- a/pkg/cloudcommon/cmdline/parser.go +++ b/pkg/cloudcommon/cmdline/parser.go @@ -337,8 +337,8 @@ func ParseNetworkConfig(desc string, idx int) (*compute.NetworkConfig, error) { if err != nil { return nil, errors.Wrap(err, "parse tx-traffic-limit") } - } else if utils.IsInStringArray(p, compute.ALL_NETWORK_TYPES) { - netConfig.NetType = p + } else if compute.IsInNetworkTypes(compute.TNetworkType(p), compute.ALL_NETWORK_TYPES) { + netConfig.NetType = compute.TNetworkType(p) } else { netConfig.Network = p } diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index 6da2defb0c..fd04758d99 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -175,8 +175,8 @@ func (self *SBaremetalGuestDriver) GetNamedNetworkConfiguration(guest *models.SG return net, nil, "", false, nil } -func (self *SBaremetalGuestDriver) GetRandomNetworkTypes() []string { - return []string{api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_GUEST} +func (self *SBaremetalGuestDriver) GetRandomNetworkTypes() []api.TNetworkType { + return []api.TNetworkType{api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_GUEST} } func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { @@ -191,7 +191,7 @@ func (self *SBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ct netTypes := drv.GetRandomNetworkTypes() if len(netConfig.NetType) > 0 { - netTypes = []string{netConfig.NetType} + netTypes = []api.TNetworkType{api.TNetworkType(netConfig.NetType)} } var wirePattern *regexp.Regexp if len(netConfig.Wire) > 0 { diff --git a/pkg/compute/guestdrivers/cloudpods-baremetals.go b/pkg/compute/guestdrivers/cloudpods-baremetals.go index 01464ff956..8477a4cd0b 100644 --- a/pkg/compute/guestdrivers/cloudpods-baremetals.go +++ b/pkg/compute/guestdrivers/cloudpods-baremetals.go @@ -175,8 +175,8 @@ func (self *SCloudpodsBaremetalGuestDriver) GetNamedNetworkConfiguration(guest * return net, nil, "", false, nil } -func (self *SCloudpodsBaremetalGuestDriver) GetRandomNetworkTypes() []string { - return []string{api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_GUEST} +func (self *SCloudpodsBaremetalGuestDriver) GetRandomNetworkTypes() []api.TNetworkType { + return []api.TNetworkType{api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_GUEST} } func (self *SCloudpodsBaremetalGuestDriver) Attach2RandomNetwork(guest *models.SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *models.SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]models.SGuestnetwork, error) { @@ -191,7 +191,7 @@ func (self *SCloudpodsBaremetalGuestDriver) Attach2RandomNetwork(guest *models.S netTypes := drv.GetRandomNetworkTypes() if len(netConfig.NetType) > 0 { - netTypes = []string{netConfig.NetType} + netTypes = []api.TNetworkType{netConfig.NetType} } var wirePattern *regexp.Regexp if len(netConfig.Wire) > 0 { diff --git a/pkg/compute/guestdrivers/kvm.go b/pkg/compute/guestdrivers/kvm.go index 33983ea68c..1482ada950 100644 --- a/pkg/compute/guestdrivers/kvm.go +++ b/pkg/compute/guestdrivers/kvm.go @@ -1273,3 +1273,7 @@ func (kvm *SKVMGuestDriver) ValidateGuestHotChangeConfigInput(ctx context.Contex } return confs, nil } + +func (kvm *SKVMGuestDriver) GetRandomNetworkTypes() []api.TNetworkType { + return []api.TNetworkType{api.NETWORK_TYPE_GUEST, api.NETWORK_TYPE_HOSTLOCAL} +} diff --git a/pkg/compute/guestdrivers/pod.go b/pkg/compute/guestdrivers/pod.go index df505dd08e..900be4c076 100644 --- a/pkg/compute/guestdrivers/pod.go +++ b/pkg/compute/guestdrivers/pod.go @@ -509,8 +509,8 @@ func (p *SPodDriver) RequestRebuildRootDisk(ctx context.Context, guest *models.S return p.newUnsupportOperationError("rebuild root") } -func (p *SPodDriver) GetRandomNetworkTypes() []string { - return []string{api.NETWORK_TYPE_CONTAINER, api.NETWORK_TYPE_GUEST} +func (p *SPodDriver) GetRandomNetworkTypes() []api.TNetworkType { + return []api.TNetworkType{api.NETWORK_TYPE_CONTAINER, api.NETWORK_TYPE_GUEST, api.NETWORK_TYPE_HOSTLOCAL} } func (p *SPodDriver) IsSupportGuestClone() bool { diff --git a/pkg/compute/guestdrivers/virtualization.go b/pkg/compute/guestdrivers/virtualization.go index 3a02413b85..cc3f509e48 100644 --- a/pkg/compute/guestdrivers/virtualization.go +++ b/pkg/compute/guestdrivers/virtualization.go @@ -79,8 +79,8 @@ func (self *SVirtualizedGuestDriver) GetNamedNetworkConfiguration(guest *models. return net, nicConfs, api.IPAllocationStepdown, false, nil } -func (self *SVirtualizedGuestDriver) GetRandomNetworkTypes() []string { - return []string{api.NETWORK_TYPE_GUEST} +func (self *SVirtualizedGuestDriver) GetRandomNetworkTypes() []api.TNetworkType { + return []api.TNetworkType{api.NETWORK_TYPE_GUEST} } func (self *SVirtualizedGuestDriver) wireAvaiableForGuest(guest *models.SGuest, wire *models.SWire) (bool, error) { @@ -106,7 +106,7 @@ func (self *SVirtualizedGuestDriver) Attach2RandomNetwork(guest *models.SGuest, } netTypes := driver.GetRandomNetworkTypes() if len(netConfig.NetType) > 0 { - netTypes = []string{netConfig.NetType} + netTypes = []api.TNetworkType{api.TNetworkType(netConfig.NetType)} } var sriovWires []string diff --git a/pkg/compute/models/cloudaccount_vmware.go b/pkg/compute/models/cloudaccount_vmware.go index dd77f82ca3..53ca3403cf 100644 --- a/pkg/compute/models/cloudaccount_vmware.go +++ b/pkg/compute/models/cloudaccount_vmware.go @@ -287,7 +287,7 @@ func (account *SCloudaccount) createNetworks(ctx context.Context, zoneId string, // NETWORK_TYPE_GUEST = "guest" // NETWORK_TYPE_BAREMETAL = "baremetal" -func (account *SCloudaccount) createNetwork(ctx context.Context, wireId, networkType string, net CANetConf) error { +func (account *SCloudaccount) createNetwork(ctx context.Context, wireId string, networkType api.TNetworkType, net CANetConf) error { network := &SNetwork{} network.Name = net.Name if hint, err := NetworkManager.NewIfnameHint(net.Name); err != nil { diff --git a/pkg/compute/models/guestdrivers.go b/pkg/compute/models/guestdrivers.go index 5a07420e09..78d6474ef7 100644 --- a/pkg/compute/models/guestdrivers.go +++ b/pkg/compute/models/guestdrivers.go @@ -73,7 +73,7 @@ type IGuestDriver interface { GetNamedNetworkConfiguration(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig) (*SNetwork, []SNicConfig, api.IPAllocationDirection, bool, error) Attach2RandomNetwork(guest *SGuest, ctx context.Context, userCred mcclient.TokenCredential, host *SHost, netConfig *api.NetworkConfig, pendingUsage quotas.IQuota) ([]SGuestnetwork, error) - GetRandomNetworkTypes() []string + GetRandomNetworkTypes() []api.TNetworkType GetStorageTypes() []string ChooseHostStorage(host *SHost, guest *SGuest, diskConfig *api.DiskConfig, storageIds []string) (*SStorage, error) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index a0a2f4d9ca..7d1850cbcd 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -3799,7 +3799,7 @@ func (manager *SHostManager) ValidateCreateData( wire := wireObj.(*SWire) lockman.LockObject(ctx, wire) defer lockman.ReleaseObject(ctx, wire) - net, err := wire.GetCandidatePrivateNetwork(ctx, userCred, userCred, NetworkManager.AllowScope(userCred), false, []string{api.NETWORK_TYPE_PXE, api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_GUEST}) + net, err := wire.GetCandidatePrivateNetwork(ctx, userCred, userCred, NetworkManager.AllowScope(userCred), false, []api.TNetworkType{api.NETWORK_TYPE_PXE, api.NETWORK_TYPE_BAREMETAL, api.NETWORK_TYPE_GUEST}) if err != nil { return input, httperrors.NewGeneralError(err) } @@ -4939,7 +4939,7 @@ func (h *SHost) PerformEnableNetif( } func (h *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCredential, netif *SNetInterface, - network, ipAddr, allocDir string, netType string, reserve, requireDesignatedIp bool) error { + network, ipAddr, allocDir string, netType api.TNetworkType, reserve, requireDesignatedIp bool) error { bn := netif.GetHostNetwork() if bn != nil { log.Debugf("Netif has been attach2network? %s", jsonutils.Marshal(bn)) @@ -4982,11 +4982,11 @@ func (h *SHost) EnableNetif(ctx context.Context, userCred mcclient.TokenCredenti return fmt.Errorf("Network %s not reacheable on mac %s", network, netif.Mac) } } else { - var netTypes []string + var netTypes []api.TNetworkType if len(netType) > 0 && netType != api.NETWORK_TYPE_BAREMETAL { - netTypes = []string{netType, api.NETWORK_TYPE_BAREMETAL} + netTypes = []api.TNetworkType{netType, api.NETWORK_TYPE_BAREMETAL} } else { - netTypes = []string{api.NETWORK_TYPE_BAREMETAL} + netTypes = []api.TNetworkType{api.NETWORK_TYPE_BAREMETAL} } net, err = wire.GetCandidatePrivateNetwork(ctx, userCred, userCred, NetworkManager.AllowScope(userCred), false, netTypes) if err != nil { diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 353dc4dd3f..9d89600919 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -127,7 +127,7 @@ type SNetwork struct { // 服务器类型 // example: server - ServerType string `width:"16" charset:"ascii" default:"guest" nullable:"true" list:"user" create:"optional"` + ServerType api.TNetworkType `width:"16" charset:"ascii" default:"guest" nullable:"true" list:"user" create:"optional"` // 分配策略 AllocPolicy string `width:"16" charset:"ascii" nullable:"true" get:"user" update:"user" create:"optional"` @@ -734,7 +734,7 @@ func (snet *SNetwork) SyncWithCloudNetwork(ctx context.Context, userCred mcclien snet.GuestIpEnd = extNet.GetIpEnd() snet.GuestIpMask = extNet.GetIpMask() snet.GuestGateway = extNet.GetGateway() - snet.ServerType = extNet.GetServerType() + snet.ServerType = api.TNetworkType(extNet.GetServerType()) snet.GuestIp6Start = extNet.GetIp6Start() snet.GuestIp6End = extNet.GetIp6End() @@ -799,7 +799,7 @@ func (manager *SNetworkManager) newFromCloudNetwork(ctx context.Context, userCre net.GuestIpEnd = extNet.GetIpEnd() net.GuestIpMask = extNet.GetIpMask() net.GuestGateway = extNet.GetGateway() - net.ServerType = extNet.GetServerType() + net.ServerType = api.TNetworkType(extNet.GetServerType()) net.GuestIp6Start = extNet.GetIp6Start() net.GuestIp6End = extNet.GetIp6End() net.GuestIp6Mask = extNet.GetIp6Mask() @@ -988,7 +988,7 @@ func (manager *SNetworkManager) TotalPortCount( providers []string, brands []string, cloudEnv string, rangeObjs []db.IStandaloneModel, policyResult rbacutils.SPolicyResult, -) map[string]NetworkPortStat { +) map[api.TNetworkType]NetworkPortStat { nets := make([]SNetwork, 0) err := manager.totalPortCountQ( ctx, @@ -1001,7 +1001,7 @@ func (manager *SNetworkManager) TotalPortCount( if err != nil { log.Errorf("TotalPortCount: %v", err) } - ret := make(map[string]NetworkPortStat) + ret := make(map[api.TNetworkType]NetworkPortStat) for _, net := range nets { var stat NetworkPortStat var allStat NetworkPortStat @@ -1702,7 +1702,7 @@ func (manager *SNetworkManager) validateEnsureZoneVpc(ctx context.Context, userC func (manager *SNetworkManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, input api.NetworkCreateInput) (api.NetworkCreateInput, error) { if input.ServerType == "" { input.ServerType = api.NETWORK_TYPE_GUEST - } else if !utils.IsInStringArray(input.ServerType, api.ALL_NETWORK_TYPES) { + } else if !api.IsInNetworkTypes(input.ServerType, api.ALL_NETWORK_TYPES) { return input, httperrors.NewInputParameterError("Invalid server_type: %s", input.ServerType) } diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index a4326b7cfc..fe77a3440e 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -1004,7 +1004,7 @@ func (swire *SWire) getPrivateNetworks(ctx context.Context, userCred mcclient.To return nets, nil } -func (swire *SWire) GetCandidatePrivateNetwork(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope, isExit bool, serverTypes []string) (*SNetwork, error) { +func (swire *SWire) GetCandidatePrivateNetwork(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope, isExit bool, serverTypes []api.TNetworkType) (*SNetwork, error) { nets, err := swire.getPrivateNetworks(ctx, userCred, ownerId, scope) if err != nil { return nil, err @@ -1012,7 +1012,7 @@ func (swire *SWire) GetCandidatePrivateNetwork(ctx context.Context, userCred mcc return ChooseCandidateNetworks(nets, isExit, serverTypes), nil } -func (swire *SWire) GetCandidateAutoAllocNetwork(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope, isExit bool, serverTypes []string) (*SNetwork, error) { +func (swire *SWire) GetCandidateAutoAllocNetwork(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope, isExit bool, serverTypes []api.TNetworkType) (*SNetwork, error) { nets, err := swire.getAutoAllocNetworks(ctx, userCred, ownerId, scope) if err != nil { return nil, err @@ -1072,14 +1072,14 @@ func chooseNetworkByAddressCount(nets []*SNetwork) (*SNetwork, *SNetwork) { return minSel, maxSel } -func ChooseCandidateNetworks(nets []SNetwork, isExit bool, serverTypes []string) *SNetwork { +func ChooseCandidateNetworks(nets []SNetwork, isExit bool, serverTypes []api.TNetworkType) *SNetwork { matchingNets := make([]*SNetwork, 0) notMatchingNets := make([]*SNetwork, 0) for _, s := range serverTypes { net := chooseCandidateNetworksByNetworkType(nets, isExit, s) if net != nil { - if utils.IsInStringArray(net.ServerType, serverTypes) { + if api.IsInNetworkTypes(net.ServerType, serverTypes) { matchingNets = append(matchingNets, net) } else { notMatchingNets = append(notMatchingNets, net) @@ -1098,7 +1098,7 @@ func ChooseCandidateNetworks(nets []SNetwork, isExit bool, serverTypes []string) return nil } -func chooseCandidateNetworksByNetworkType(nets []SNetwork, isExit bool, serverType string) *SNetwork { +func chooseCandidateNetworksByNetworkType(nets []SNetwork, isExit bool, serverType api.TNetworkType) *SNetwork { matchingNets := make([]*SNetwork, 0) notMatchingNets := make([]*SNetwork, 0) diff --git a/pkg/hostman/host_services.go b/pkg/hostman/host_services.go index 2388bdfc2c..6869afc950 100644 --- a/pkg/hostman/host_services.go +++ b/pkg/hostman/host_services.go @@ -77,7 +77,7 @@ func (host *SHostService) RunService() { hostutils.Init() hostInstance := hostinfo.Instance() - if err := hostInstance.Init(); err != nil { + if err := hostInstance.Init(app.GetContext()); err != nil { log.Fatalf("Host instance init error: %v", err) } diff --git a/pkg/hostman/hostinfo/container.go b/pkg/hostman/hostinfo/container.go new file mode 100644 index 0000000000..3f5885cfc9 --- /dev/null +++ b/pkg/hostman/hostinfo/container.go @@ -0,0 +1,78 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinfo + +import ( + "context" + "path" + "time" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + hostapi "yunion.io/x/onecloud/pkg/apis/host" + "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/pod" + "yunion.io/x/onecloud/pkg/util/pod/cadvisor" + "yunion.io/x/onecloud/pkg/util/pod/stats" +) + +func (h *SHostInfo) initCRI() error { + cri, err := pod.NewCRI(h.GetContainerRuntimeEndpoint(), 3*time.Second) + if err != nil { + return errors.Wrapf(err, "New CRI by endpoint %q", h.GetContainerRuntimeEndpoint()) + } + ver, err := cri.Version(context.Background()) + if err != nil { + return errors.Wrap(err, "get runtime version") + } + log.Infof("Init container runtime: %s", ver) + h.cri = cri + return nil +} + +func (h *SHostInfo) initContainerCPUMap(topo *hostapi.HostTopology) error { + statefile := path.Join(options.HostOptions.ServersPath, "container_cpu_map") + cm, err := pod.NewHostContainerCPUMap(topo, statefile) + if err != nil { + return errors.Wrap(err, "NewHostContainerCPUMap") + } + h.containerCPUMap = cm + return nil +} + +func (h *SHostInfo) startContainerStatsProvider(cri pod.CRI) error { + ca, err := cadvisor.New(nil, "/opt/cloud/workspace", []string{"cloudpods"}) + if err != nil { + return errors.Wrap(err, "new cadvisor") + } + if err := ca.Start(); err != nil { + return errors.Wrap(err, "start cadvisor") + } + h.containerStatsProvier = stats.NewCRIContainerStatsProvider(ca, cri.GetRuntimeClient(), cri.GetImageClient()) + return nil +} + +func (h *SHostInfo) GetCRI() pod.CRI { + return h.cri +} + +func (h *SHostInfo) GetContainerCPUMap() *pod.HostContainerCPUMap { + return h.containerCPUMap +} + +func (h *SHostInfo) GetContainerStatsProvider() stats.ContainerStatsProvider { + return h.containerStatsProvier +} diff --git a/pkg/hostman/hostinfo/finalize_network.go b/pkg/hostman/hostinfo/finalize_network.go new file mode 100644 index 0000000000..46b886a2b4 --- /dev/null +++ b/pkg/hostman/hostinfo/finalize_network.go @@ -0,0 +1,137 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hostinfo + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/netutils" + + computeapis "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/hostman/hostutils" + computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + computeoptions "yunion.io/x/onecloud/pkg/mcclient/options/compute" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/procutils" +) + +func (h *SHostInfo) finalizeNetworkSetup(ctx context.Context) error { + for i := 0; i < len(h.Nics); i++ { + if err := h.Nics[i].SetupDhcpRelay(); err != nil { + return errors.Wrapf(err, "SetupDhcpRelay %s", h.Nics[i]) + } + } + for i := 0; i < len(h.Nics); i++ { + if err := h.Nics[i].setupHostLocalNetworks(ctx); err != nil { + return errors.Wrapf(err, "SetupHostLocalNetworks %s", h.Nics[i]) + } + } + return nil +} + +func (n *SNIC) String() string { + return fmt.Sprintf("%s/%s/%s", n.Inter, n.Bridge, n.Ip) +} + +func (n *SNIC) setupHostLocalNetworks(ctx context.Context) error { + nets, err := n.fetchHostLocalNetworks(ctx) + if err != nil { + return errors.Wrap(err, "fetchHostLocalNetworks") + } + for i := range nets { + if len(nets[i].GuestGateway) == 0 { + continue + } + err := n.setupHostLocalNet(ctx, nets[i]) + if err != nil { + return errors.Wrap(err, "setupHostLocalNet") + } + } + return nil +} + +func (n *SNIC) setupHostLocalNet(ctx context.Context, netInfo computeapis.NetworkDetails) error { + // setup gateway ip + if err := n.setupSlaveIp(ctx, netInfo.GuestGateway, netInfo.GuestIpMask); err != nil { + return errors.Wrapf(err, "setupSlaveIp %s %s", n, netInfo.GuestGateway) + } + return nil +} + +func (n *SNIC) setupSlaveIp(ctx context.Context, gatewayIp string, maskLen byte) error { + bridgeIf := netutils2.NewNetInterface(n.Bridge) + slaveAddrs := bridgeIf.GetSlaveAddresses() + for i := range slaveAddrs { + if slaveAddrs[i][0] == gatewayIp { + // already configured, skip + return nil + } + } + if err := n.BridgeDev.SetupSlaveAddresses([][]string{[]string{gatewayIp, fmt.Sprintf("%d", maskLen)}}); err != nil { + return errors.Wrap(err, "SetupSlaveAddresses") + } + if err := n.setupMasquerateRule(ctx, gatewayIp, maskLen); err != nil { + return errors.Wrap(err, "setupMasquerateRule") + } + return nil +} + +func (n *SNIC) setupMasquerateRule(ctx context.Context, ipStr string, maskLen byte) error { + gwip, err := netutils.NewIPV4Addr(ipStr) + if err != nil { + return errors.Wrapf(err, "NewIPV4Addr %s", ipStr) + } + netip := gwip.NetAddr(int8(maskLen)) + maskip := netutils.Masklen2Mask(int8(maskLen)) + cmd := procutils.NewCommand("iptables", "-t", "nat", "-A", "POSTROUTING", "-s", + fmt.Sprintf("%s/%s", netip.String(), maskip.String()), "-o", n.Bridge, "-j", "MASQUERADE") + if err := cmd.Run(); err != nil { + return errors.Wrap(err, "add masquerade rule") + } + return nil +} + +func (n *SNIC) fetchHostLocalNetworks(ctx context.Context) ([]computeapis.NetworkDetails, error) { + s := hostutils.GetComputeSession(ctx) + params := computeoptions.NetworkListOptions{} + params.ServerType = "hostlocal" + limit := 50 + params.Limit = &limit + params.Wire = n.WireId + params.Scope = "system" + total := -1 + nets := make([]computeapis.NetworkDetails, 0) + for total < 0 || len(nets) < total { + offset := len(nets) + params.Offset = &offset + results, err := computemodules.Networks.List(s, jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "Networks.List") + } + total = results.Total + for i := range results.Data { + netDetails := computeapis.NetworkDetails{} + err := results.Data[i].Unmarshal(&netDetails) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + nets = append(nets, netDetails) + } + } + return nets, nil +} diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index d15780ea60..f01cb4c351 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -71,7 +71,6 @@ import ( "yunion.io/x/onecloud/pkg/util/netutils2" "yunion.io/x/onecloud/pkg/util/ovnutils" "yunion.io/x/onecloud/pkg/util/pod" - "yunion.io/x/onecloud/pkg/util/pod/cadvisor" "yunion.io/x/onecloud/pkg/util/pod/stats" "yunion.io/x/onecloud/pkg/util/procutils" "yunion.io/x/onecloud/pkg/util/qemutils" @@ -201,7 +200,7 @@ func (h *SHostInfo) HugepageSizeKb() int { * 4. parse host config, config ip address * 5. check is ovn support, setup ovn chassis */ -func (h *SHostInfo) Init() error { +func (h *SHostInfo) Init(ctx context.Context) error { if err := h.prepareEnv(); err != nil { return errors.Wrap(err, "Prepare environment") } @@ -240,54 +239,6 @@ func (h *SHostInfo) Init() error { return nil } -func (h *SHostInfo) initCRI() error { - cri, err := pod.NewCRI(h.GetContainerRuntimeEndpoint(), 3*time.Second) - if err != nil { - return errors.Wrapf(err, "New CRI by endpoint %q", h.GetContainerRuntimeEndpoint()) - } - ver, err := cri.Version(context.Background()) - if err != nil { - return errors.Wrap(err, "get runtime version") - } - log.Infof("Init container runtime: %s", ver) - h.cri = cri - return nil -} - -func (h *SHostInfo) initContainerCPUMap(topo *hostapi.HostTopology) error { - statefile := path.Join(options.HostOptions.ServersPath, "container_cpu_map") - cm, err := pod.NewHostContainerCPUMap(topo, statefile) - if err != nil { - return errors.Wrap(err, "NewHostContainerCPUMap") - } - h.containerCPUMap = cm - return nil -} - -func (h *SHostInfo) startContainerStatsProvider(cri pod.CRI) error { - ca, err := cadvisor.New(nil, "/opt/cloud/workspace", []string{"cloudpods"}) - if err != nil { - return errors.Wrap(err, "new cadvisor") - } - if err := ca.Start(); err != nil { - return errors.Wrap(err, "start cadvisor") - } - h.containerStatsProvier = stats.NewCRIContainerStatsProvider(ca, cri.GetRuntimeClient(), cri.GetImageClient()) - return nil -} - -func (h *SHostInfo) GetCRI() pod.CRI { - return h.cri -} - -func (h *SHostInfo) GetContainerCPUMap() *pod.HostContainerCPUMap { - return h.containerCPUMap -} - -func (h *SHostInfo) GetContainerStatsProvider() stats.ContainerStatsProvider { - return h.containerStatsProvier -} - func (h *SHostInfo) setupOvnChassis() error { opts := &options.HostOptions if opts.BridgeDriver != hostbridge.DRV_OPEN_VSWITCH { @@ -395,11 +346,7 @@ func (h *SHostInfo) parseConfig() error { } h.Nics = append(h.Nics, nic) } - for i := 0; i < len(h.Nics); i++ { - if err := h.Nics[i].SetupDhcpRelay(); err != nil { - return errors.Wrapf(err, "SetupDhcpRelay %s/%s/%s", h.Nics[i].Inter, h.Nics[i].Bridge, h.Nics[i].Ip) - } - } + if len(options.HostOptions.ListenInterface) > 0 { h.MasterNic = netutils2.NewNetInterface(options.HostOptions.ListenInterface) if len(h.MasterNic.Addr) == 0 { @@ -1143,24 +1090,34 @@ func (h *SHostInfo) register() { hostInfo, err := h.initHostRecord() if err != nil { h.onFail(errors.Wrap(err, "initHostRecords")) + return } defer h.reportHostErrors() err = h.initCgroup() if err != nil { h.onFail(errors.Wrap(err, "initCgroup")) + return } err = h.initHostNetworks(hostInfo) if err != nil { h.onFail(errors.Wrap(err, "initHostNetworks")) + return } err = h.initIsolatedDevices() if err != nil { h.onFail(errors.Wrap(err, "initIsolatedDevices")) + return } err = h.initStorages() if err != nil { h.onFail(errors.Wrap(err, "initStorages")) + return + } + err = h.finalizeNetworkSetup(context.Background()) + if err != nil { + h.onFail(errors.Wrap(err, "finalizeNetworkSetup")) + return } h.deployAdminAuthorizedKeys() h.onSucc() @@ -1257,7 +1214,7 @@ func (h *SHostInfo) tryCreateNetworkOnWire() (string, error) { params.Set("ip", jsonutils.NewString(masterIp)) params.Set("mask", jsonutils.NewInt(int64(mask))) params.Set("is_classic", jsonutils.JSONTrue) - params.Set("server_type", jsonutils.NewString(api.NETWORK_TYPE_BAREMETAL)) + params.Set("server_type", jsonutils.NewString(string(api.NETWORK_TYPE_BAREMETAL))) params.Set("is_on_premise", jsonutils.JSONTrue) ret, err := modules.Networks.PerformClassAction(h.GetSession(), "try-create-network", params) if err != nil { diff --git a/pkg/scheduler/algorithm/predicates/network_predicate.go b/pkg/scheduler/algorithm/predicates/network_predicate.go index e97f1dc22d..c6f390d678 100644 --- a/pkg/scheduler/algorithm/predicates/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/network_predicate.go @@ -100,7 +100,7 @@ func (p *NetworkPredicate) PreExecute(ctx context.Context, u *core.Unit, cs []co return true, nil } -func IsNetworksAvailable(ctx context.Context, c core.Candidater, data *api.SchedInfo, req *computeapi.NetworkConfig, networks []*api.CandidateNetwork, netTypes []string, getFreePort func(string) int) (int, []core.PredicateFailureReason) { +func IsNetworksAvailable(ctx context.Context, c core.Candidater, data *api.SchedInfo, req *computeapi.NetworkConfig, networks []*api.CandidateNetwork, netTypes []computeapi.TNetworkType, getFreePort func(string) int) (int, []core.PredicateFailureReason) { var fullErrMsgs []core.PredicateFailureReason var freeCnt int @@ -173,7 +173,7 @@ func IsNetworkAvailable( ctx context.Context, c core.Candidater, data *api.SchedInfo, req *computeapi.NetworkConfig, n *api.CandidateNetwork, - netTypes []string, getFreePort func(string) int, + netTypes []computeapi.TNetworkType, getFreePort func(string) int, ) core.PredicateFailureReason { address := req.Address private := req.Private @@ -181,7 +181,7 @@ func IsNetworkAvailable( wire := req.Wire isMatchServerType := func(network *models.SNetwork) bool { - return utils.IsInStringArray(network.ServerType, netTypes) + return computeapi.IsInNetworkTypes(network.ServerType, netTypes) } isMigrate := func() bool { @@ -293,14 +293,14 @@ func IsNetworkAvailable( return nil } -func (p *NetworkPredicate) GetNetworkTypes(u *core.Unit, specifyType string) []string { - netTypes := []string{} +func (p *NetworkPredicate) GetNetworkTypes(u *core.Unit, specifyType computeapi.TNetworkType) []computeapi.TNetworkType { + netTypes := []computeapi.TNetworkType{} driver := p.GetHypervisorDriver(u) if driver != nil { netTypes = driver.GetRandomNetworkTypes() } if len(specifyType) > 0 { - netTypes = []string{specifyType} + netTypes = []computeapi.TNetworkType{specifyType} } return netTypes } diff --git a/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go b/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go index b3bf786fd9..90ab651bde 100644 --- a/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go +++ b/pkg/scheduler/algorithm/predicates/network_schedtag_predicate.go @@ -102,14 +102,14 @@ func (p *NetworkSchedtagPredicate) IsResourceFitInput(ctx context.Context, u *co return IsNetworkAvailable(ctx, c, u.SchedData(), net.NetworkConfig, network, p.GetNetworkTypes(net.NetType), nil) } -func (p *NetworkSchedtagPredicate) GetNetworkTypes(specifyType string) []string { - netTypes := []string{} +func (p *NetworkSchedtagPredicate) GetNetworkTypes(specifyType computeapi.TNetworkType) []computeapi.TNetworkType { + netTypes := []computeapi.TNetworkType{} driver := p.GetHypervisorDriver() if driver != nil { netTypes = driver.GetRandomNetworkTypes() } if len(specifyType) > 0 { - netTypes = []string{specifyType} + netTypes = []computeapi.TNetworkType{specifyType} } return netTypes }