scheduler: refactor network_predicate filter

This commit is contained in:
Zexi Li
2019-09-28 18:23:27 +08:00
parent 11fe8bcf2b
commit 8286d318ac
7 changed files with 283 additions and 391 deletions
@@ -1,156 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package baremetal
import (
"fmt"
"strings"
"sync"
"yunion.io/x/pkg/utils"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
"yunion.io/x/onecloud/pkg/scheduler/core"
)
type NetworkPredicate struct {
BasePredicate
plugin.BasePlugin
SelectedNetworks sync.Map
}
func (p *NetworkPredicate) Name() string {
return "baremetal_network"
}
func (p *NetworkPredicate) Clone() core.FitPredicate {
return &NetworkPredicate{}
}
func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
notIgnore, _ := p.BasePredicate.PreExecute(u, cs)
if !notIgnore {
return false, nil
}
u.AppendSelectPlugin(p)
d := u.SchedData()
if len(d.HostId) > 0 && len(d.Networks) == 0 {
return false, nil
}
return true, nil
}
func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
h := predicates.NewPredicateHelper(p, u, c)
schedData := u.SchedData()
networks := c.Getter().Networks()
counters := core.NewCounters()
isMigrate := func() bool {
return len(schedData.HostId) > 0
}
isRandomNetworkAvailable := func(private bool, exit bool, wire string) string {
var errMsgs []string
for _, network := range networks {
appendError := func(errMsg string) {
errMsgs = append(errMsgs, fmt.Sprintf("%s: %s", network.Id, errMsg))
}
if !((network.GetPorts() > 0 || isMigrate()) && network.IsExitNetwork() == exit) {
appendError(predicates.ErrNoPorts)
}
if wire != "" && !utils.HasPrefix(wire, network.WireId) && !utils.HasPrefix(wire, network.GetWire().GetName()) { // re
appendError(predicates.ErrWireIsNotMatch)
}
if (!private && network.IsPublic) || (private && !network.IsPublic && network.ProjectId == schedData.Project) {
// TODO: support reservedNetworks
reservedNetworks := 0
restPort := int64(network.GetPorts() - reservedNetworks)
if restPort == 0 {
appendError("not enough network port")
continue
}
counter := u.CounterManager.GetOrCreate("net:"+network.Id, func() core.Counter {
return core.NewNormalCounter(restPort)
})
u.SharedResourceManager.Add(network.Id, counter)
counters.Add(counter)
p.SelectedNetworks.Store(network.Id, counter.GetCount())
return ""
} else {
appendError(predicates.ErrNotOwner)
}
}
return strings.Join(errMsgs, ";")
}
filterByRandomNetwork := func() {
if err_msg := isRandomNetworkAvailable(false, false, ""); err_msg != "" {
h.Exclude(err_msg)
}
h.SetCapacityCounter(counters)
}
isNetworkAvaliable := func(network *computeapi.NetworkConfig) string {
if network.Network == "" {
return isRandomNetworkAvailable(network.Private, network.Exit, network.Wire)
}
for _, net := range networks {
if (network.Network == net.Id || network.Network == net.Name) && (net.IsPublic || net.ProjectId == schedData.Project) && (net.GetPorts() > 0 || isMigrate()) {
h.SetCapacity(1)
return ""
}
}
return predicates.ErrUnknown
}
filterBySpecifiedNetworks := func() {
var errMsgs []string
for _, network := range schedData.Networks {
if err_msg := isNetworkAvaliable(network); err_msg != "" {
errMsgs = append(errMsgs, err_msg)
}
}
if len(errMsgs) > 0 {
h.Exclude(strings.Join(errMsgs, ", "))
}
}
loadUnknownNetworks := func() bool {
return true // TODO: ???
}
loadUnknownNetworks()
// Randomly assign networks if no network is specified.
if len(schedData.Networks) == 0 {
filterByRandomNetwork()
} else {
filterBySpecifiedNetworks()
}
return h.GetResult()
}
@@ -118,6 +118,7 @@ const (
NetworkDomain = "network_domain"
NetworkRange = "network_range"
NetworkFreeCount = "network_free_count"
NetworkPort = "network_port"
StorageEnable = "storage_status"
StorageMatch = "storage_match"
@@ -1,217 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package guest
import (
"fmt"
"strings"
"sync"
"yunion.io/x/pkg/util/sets"
"yunion.io/x/pkg/utils"
"yunion.io/x/onecloud/pkg/compute/models"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
"yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates"
"yunion.io/x/onecloud/pkg/scheduler/api"
"yunion.io/x/onecloud/pkg/scheduler/core"
)
// NetworkPredicate will filter the current network information with
// the specified scheduling information to match, if not specified will
// randomly match the available network resources.
type NetworkPredicate struct {
predicates.BasePredicate
plugin.BasePlugin
SelectedNetworks sync.Map
}
func (p *NetworkPredicate) Name() string {
return "host_network"
}
func (p *NetworkPredicate) Clone() core.FitPredicate {
return &NetworkPredicate{}
}
func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
data := u.SchedData()
if len(data.HostId) > 0 && len(data.Networks) == 0 {
return false, nil
}
return true, nil
}
func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
h := predicates.NewPredicateHelper(p, u, c)
getter := c.Getter()
networks := getter.Networks()
d := u.SchedData()
isMigrate := func() bool {
return len(d.HostId) > 0
}
// ServerType's value is 'guest', 'container' or ''(support all type) will return true.
isMatchServerType := func(network *models.SNetwork) bool {
return sets.NewString("guest", "", "container").Has(network.ServerType)
}
counterOfNetwork := func(u *core.Unit, n *models.SNetwork, r int) core.Counter {
counter := u.CounterManager.GetOrCreate("net:"+n.Id, func() core.Counter {
return core.NewNormalCounter(int64(getter.GetFreePort(n.Id) - r))
})
u.SharedResourceManager.Add(n.GetId(), counter)
return counter
}
isRandomNetworkAvailable := func(private bool, exit bool, wire string,
counters core.MultiCounter) string {
var fullErrMsgs []string
found := false
for _, n := range networks {
errMsgs := []string{}
appendError := func(errMsg string) {
errMsgs = append(errMsgs, errMsg)
}
if !isMatchServerType(n.SNetwork) {
appendError(predicates.ErrServerTypeIsNotMatch)
}
if n.IsExitNetwork() != exit {
appendError(predicates.ErrExitIsNotMatch)
}
if !(n.GetPorts() > 0 || isMigrate()) {
appendError(predicates.ErrNoPorts)
}
if wire != "" && !utils.HasPrefix(wire, n.WireId) && !utils.HasPrefix(wire, n.GetWire().GetName()) { // re
appendError(predicates.ErrWireIsNotMatch)
}
if !((!private && n.IsPublic) || (private && !n.IsPublic && n.ProjectId == d.Project)) {
appendError(predicates.ErrNotOwner)
}
if len(errMsgs) == 0 {
// add resource
reservedNetworks := 0
counter := counterOfNetwork(u, n.SNetwork, reservedNetworks)
p.SelectedNetworks.Store(n.GetId(), counter.GetCount())
counters.Add(counter)
found = true
} else {
fullErrMsgs = append(fullErrMsgs,
fmt.Sprintf("%s: %s", n.Id, strings.Join(errMsgs, ",")),
)
}
}
if !found {
return strings.Join(fullErrMsgs, "; ")
}
return ""
}
filterByRandomNetwork := func() {
counters := core.NewCounters()
if err_msg := isRandomNetworkAvailable(false, false, "", counters); err_msg != "" {
h.Exclude(err_msg)
}
h.SetCapacityCounter(counters)
}
isNetworkAvaliable := func(n *computeapi.NetworkConfig, counters *core.MinCounters,
networks []*api.CandidateNetwork) string {
if n.Network == "" {
counters0 := core.NewCounters()
ret_msg := isRandomNetworkAvailable(n.Private, n.Exit, n.Wire, counters0)
counters.Add(counters0)
return ret_msg
}
if len(networks) == 0 {
return predicates.ErrNoAvailableNetwork
}
errMsgs := make([]string, 0)
for _, net := range networks {
/*if !isMatchServerType(net) {
errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): server type not matched", net.Name, net.ID))
continue
}*/
if !(n.Network == net.GetId() || n.Network == net.GetName()) {
errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id))
} else if !(net.IsPublic || net.ProjectId == d.Project) {
errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): not owner (%v != %v)", net.Name, net.Id, net.ProjectId, d.Project))
} else if !(net.GetPorts() > 0 || isMigrate()) {
errMsgs = append(errMsgs, fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id))
} else {
// add resource
reservedNetworks := 0
counter := counterOfNetwork(u, net.SNetwork, reservedNetworks)
if counter.GetCount() < int64(d.Count) {
errMsgs = append(errMsgs, fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count))
continue
}
p.SelectedNetworks.Store(net.Id, counter.GetCount())
counters.Add(counter)
return ""
}
}
if len(errMsgs) == 0 {
return predicates.ErrUnknown
}
return strings.Join(errMsgs, "; ")
}
filterBySpecifiedNetworks := func() {
counters := core.NewMinCounters()
var errMsgs []string
for _, n := range d.Networks {
if err_msg := isNetworkAvaliable(n, counters, networks); err_msg != "" {
errMsgs = append(errMsgs, err_msg)
}
}
if len(errMsgs) > 0 {
h.Exclude(strings.Join(errMsgs, ", "))
} else {
h.SetCapacityCounter(counters)
}
}
if len(d.Networks) == 0 {
filterByRandomNetwork()
} else {
filterBySpecifiedNetworks()
}
return h.GetResult()
}
@@ -0,0 +1,280 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package predicates
import (
"fmt"
"sync"
"yunion.io/x/pkg/util/netutils"
"yunion.io/x/pkg/util/sets"
"yunion.io/x/pkg/utils"
computeapi "yunion.io/x/onecloud/pkg/apis/compute"
"yunion.io/x/onecloud/pkg/compute/models"
"yunion.io/x/onecloud/pkg/scheduler/algorithm/plugin"
"yunion.io/x/onecloud/pkg/scheduler/api"
"yunion.io/x/onecloud/pkg/scheduler/core"
"yunion.io/x/onecloud/pkg/util/rbacutils"
)
// NetworkPredicate will filter the current network information with
// the specified scheduling information to match, if not specified will
// randomly match the available network resources.
type NetworkPredicate struct {
BasePredicate
plugin.BasePlugin
SelectedNetworks sync.Map
}
func (p *NetworkPredicate) Name() string {
return "host_network"
}
func (p *NetworkPredicate) Clone() core.FitPredicate {
return &NetworkPredicate{}
}
func (p *NetworkPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
data := u.SchedData()
if len(data.HostId) > 0 && len(data.Networks) == 0 {
return false, nil
}
return true, nil
}
func (p *NetworkPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
h := NewPredicateHelper(p, u, c)
getter := c.Getter()
networks := getter.Networks()
d := u.SchedData()
isMigrate := func() bool {
return len(d.HostId) > 0
}
counterOfNetwork := func(u *core.Unit, n *models.SNetwork, r int) core.Counter {
counter := u.CounterManager.GetOrCreate("net:"+n.Id, func() core.Counter {
return core.NewNormalCounter(int64(getter.GetFreePort(n.Id) - r))
})
u.SharedResourceManager.Add(n.GetId(), counter)
return counter
}
isMatchServerType := func(network *models.SNetwork) bool {
if d.Hypervisor == computeapi.HYPERVISOR_BAREMETAL {
return network.ServerType == computeapi.NETWORK_TYPE_BAREMETAL
}
return sets.NewString(
"", computeapi.NETWORK_TYPE_GUEST,
computeapi.NETWORK_TYPE_CONTAINER).Has(network.ServerType)
}
checkAddress := func(addr string, net *models.SNetwork) error {
if len(addr) == 0 {
return nil
}
ipAddr, err := netutils.NewIPV4Addr(addr)
if err != nil {
return fmt.Errorf("Invalid ip address %s: %v", addr, err)
}
if !net.GetIPRange().Contains(ipAddr) {
return fmt.Errorf("Address %s not in network %s range", addr, net.Name)
}
return nil
}
checkNetCount := func(net *models.SNetwork, reqCount int) (core.Counter, core.PredicateFailureReason) {
counter := counterOfNetwork(u, net, 0)
if counter.GetCount() < int64(reqCount) {
return nil, FailReason{
Reason: fmt.Sprintf("%s: ports not enough, free: %d, required: %d", net.Name, counter.GetCount(), d.Count),
Type: NetworkFreeCount,
}
}
return counter, nil
}
isRandomNetworkAvailable := func(address, domain string, private bool, exit bool, wire string, counters core.MultiCounter) []core.PredicateFailureReason {
var fullErrMsgs []core.PredicateFailureReason
found := false
for _, n := range networks {
errMsgs := []core.PredicateFailureReason{}
appendError := func(msg core.PredicateFailureReason) {
errMsgs = append(errMsgs, msg)
}
if !isMatchServerType(n.SNetwork) {
appendError(FailReason{
Reason: fmt.Sprintf("Network %s type %s match", n.Name, n.ServerType),
Type: NetworkTypeMatch,
})
}
if n.IsExitNetwork() != exit {
appendError(FailReason{Reason: ErrExitIsNotMatch})
}
if !(n.GetPorts() > 0 || isMigrate()) {
appendError(FailReason{
Reason: fmt.Sprintf("%v(%v): ports use up", n.Name, n.Id),
Type: NetworkPort,
})
}
if wire != "" && !utils.HasPrefix(wire, n.WireId) && !utils.HasPrefix(wire, n.GetWire().GetName()) {
appendError(FailReason{
Reason: fmt.Sprintf("Wire %s != %s", wire, n.WireId),
Type: NetworkWire,
})
}
schedData := u.SchedData()
if private {
if n.IsPublic {
appendError(FailReason{
Reason: fmt.Sprintf("Network %s is public", n.Name),
Type: NetworkPublic,
})
} else if n.ProjectId != schedData.Project && utils.IsInStringArray(schedData.Project, n.GetSharedProjects()) {
appendError(FailReason{
Reason: fmt.Sprintf("Network project %s + %v not owner by %s", n.ProjectId, n.GetSharedProjects(), schedData.Project),
Type: NetworkOwner,
})
}
} else {
if !n.IsPublic {
appendError(FailReason{Reason: fmt.Sprintf("Network %s is private", n.Name), Type: NetworkPrivate})
} else if rbacutils.TRbacScope(n.PublicScope) == rbacutils.ScopeDomain {
netDomain := n.DomainId
reqDomain := domain
if netDomain != reqDomain {
appendError(FailReason{Reason: fmt.Sprintf("Network domain scope %s not owner by %s", netDomain, reqDomain), Type: NetworkDomain})
}
}
}
if err := checkAddress(address, n.SNetwork); err != nil {
appendError(FailReason{Reason: err.Error(), Type: NetworkRange})
}
if len(errMsgs) == 0 {
// add resource
counter := counterOfNetwork(u, n.SNetwork, 0)
p.SelectedNetworks.Store(n.GetId(), counter.GetCount())
counters.Add(counter)
found = true
} else {
fullErrMsgs = append(fullErrMsgs, errMsgs...)
}
}
if counters.GetCount() < int64(d.Count) {
found = false
fullErrMsgs = append(fullErrMsgs, FailReason{
Reason: fmt.Sprintf("total random ports not enough, free: %d, required: %d", counters.GetCount(), d.Count),
Type: NetworkFreeCount,
})
}
if !found {
return fullErrMsgs
}
return nil
}
filterByRandomNetwork := func() {
counters := core.NewCounters()
if errMsg := isRandomNetworkAvailable("", "", false, false, "", counters); len(errMsg) != 0 {
h.ExcludeByErrors(errMsg)
}
h.SetCapacityCounter(counters)
}
isNetworkAvaliable := func(n *computeapi.NetworkConfig, counters *core.MinCounters, networks []*api.CandidateNetwork) []core.PredicateFailureReason {
if len(networks) == 0 {
return []core.PredicateFailureReason{
FailReason{Reason: ErrNoAvailableNetwork},
}
}
if n.Network == "" {
counters0 := core.NewCounters()
retMsg := isRandomNetworkAvailable(n.Address, n.Domain, n.Private, n.Exit, n.Wire, counters0)
counters.Add(counters0)
return retMsg
}
errMsgs := make([]core.PredicateFailureReason, 0)
for _, net := range networks {
if !(n.Network == net.GetId() || n.Network == net.GetName()) {
errMsgs = append(errMsgs, &FailReason{
Reason: fmt.Sprintf("%v(%v): id/name not matched", net.Name, net.Id),
Type: NetworkMatch,
})
continue
}
if !(net.GetPorts() > 0 || isMigrate()) {
errMsgs = append(errMsgs, &FailReason{
Reason: fmt.Sprintf("%v(%v): ports use up", net.Name, net.Id),
Type: NetworkPort,
})
continue
}
counter, err := checkNetCount(net.SNetwork, d.Count)
if err != nil {
errMsgs = append(errMsgs, err)
continue
}
p.SelectedNetworks.Store(net.Id, counter.GetCount())
counters.Add(counter)
return nil
}
return nil
}
filterBySpecifiedNetworks := func() {
counters := core.NewMinCounters()
var errMsgs []core.PredicateFailureReason
for _, n := range d.Networks {
if errMsg := isNetworkAvaliable(n, counters, networks); len(errMsg) != 0 {
errMsgs = append(errMsgs, errMsg...)
}
}
if len(errMsgs) > 0 {
h.ExcludeByErrors(errMsgs)
} else {
h.SetCapacityCounter(counters)
}
}
if len(d.Networks) == 0 {
filterByRandomNetwork()
} else {
filterBySpecifiedNetworks()
}
return h.GetResult()
}
@@ -171,22 +171,6 @@ func (p *NetworkSchedtagPredicate) IsResourceFitInput(u *core.Unit, c core.Candi
}
}
}
free, err := network.GetFreeAddressCount()
if err != nil {
return &FailReason{
Reason: fmt.Sprintf("get free address count: %v", err),
Type: NetworkFreeCount,
}
}
req := u.SchedData().Count
if free < req {
return &FailReason{
Reason: fmt.Sprintf("Network %s no free IPs, free %d, require %d", network.Name, free, req),
Type: NetworkFreeCount,
}
}
h := NewPredicateHelper(p, u, c)
h.SetCapacity(int64(free))
return nil
}
+1 -1
View File
@@ -34,7 +34,7 @@ func baremetalPredicates() sets.String {
factory.RegisterFitPredicate("c-BaremetalCPUFilter", &predicatebm.CPUPredicate{}),
factory.RegisterFitPredicate("d-BaremetalMemoryFilter", &predicatebm.MemoryPredicate{}),
factory.RegisterFitPredicate("e-BaremetalStorageFilter", &predicatebm.StoragePredicate{}),
//factory.RegisterFitPredicate("f-BaremetalNetFilter", &predicatebm.NetworkPredicate{}),
factory.RegisterFitPredicate("f-BaremetalNetFilter", &predicates.NetworkPredicate{}),
factory.RegisterFitPredicate("g-BaremetalResourceTypeFilter", &predicates.ResourceTypePredicate{}),
factory.RegisterFitPredicate("h-DiskschedtagFilter", &predicates.DiskSchedtagPredicate{}),
factory.RegisterFitPredicate("i-NetschedtagFilter", &predicates.NetworkSchedtagPredicate{}),
+1 -1
View File
@@ -39,7 +39,7 @@ func defaultPredicates() sets.String {
factory.RegisterFitPredicate("g-GuestCPUFilter", &predicateguest.CPUPredicate{}),
factory.RegisterFitPredicate("h-GuestMemoryFilter", &predicateguest.MemoryPredicate{}),
factory.RegisterFitPredicate("i-GuestStorageFilter", &predicateguest.StoragePredicate{}),
//factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicateguest.NetworkPredicate{}),
factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicates.NetworkPredicate{}),
factory.RegisterFitPredicate("k-GuestIsolatedDeviceFilter", &predicateguest.IsolatedDevicePredicate{}),
factory.RegisterFitPredicate("l-GuestResourceTypeFilter", &predicates.ResourceTypePredicate{}),
factory.RegisterFitPredicate("m-GuestDiskschedtagFilter", &predicates.DiskSchedtagPredicate{}),