Merge pull request #3132 from Mjoycarry/feature/instance_group

Scheduler Feature

cmd/climc/shell/instance_group.go

@@ -0,0 +1,97 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type InstanceGroupListOptions struct {
+		options.BaseListOptions
+
+		ServiceType string `help:"Service Type"`
+		ParentId    string `help:"Parent ID"`
+		ZoneId      string `help:"Zone ID"`
+	}
+
+	R(&InstanceGroupListOptions{}, "instance-group-list", "List instance group", func(s *mcclient.ClientSession,
+		args *InstanceGroupListOptions) error {
+
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.InstanceGroup.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.InstanceGroup.GetColumns(s))
+		return nil
+	})
+
+	type InstanceGroupShowOptions struct {
+		ID string `help:"ID or Name of instance group"`
+	}
+	R(&InstanceGroupShowOptions{}, "instance-group-show", "Show details of a instance group",
+		func(s *mcclient.ClientSession, args *InstanceGroupShowOptions) error {
+
+			result, err := modules.InstanceGroup.GetById(s, args.ID, nil)
+			if err != nil {
+				return err
+			}
+			printObject(result)
+			return nil
+		})
+
+	type InstanceGroupCreateOptions struct {
+		NAME   string `help:"name of instance group"`
+		ZONEID string `help:"zone id" json:"zone_id"`
+
+		ServiceType   string `help:"service type"`
+		ParentId      string `help:"parent id"`
+		SchedStrategy string `help:"scheduler strategy"`
+		Granularity   string `help:"the upper limit number of guests with this group in a host"`
+	}
+
+	R(&InstanceGroupCreateOptions{}, "instance-group-create", "Create a instance group",
+		func(s *mcclient.ClientSession, args *InstanceGroupCreateOptions) error {
+			params, err := options.StructToParams(args)
+			if err != nil {
+				return err
+			}
+			result, err := modules.InstanceGroup.Create(s, params)
+			if err != nil {
+				return err
+			}
+			printObject(result)
+			return nil
+		},
+	)
+
+	R(&InstanceGroupShowOptions{}, "instance-group-delete", "delete a instance group",
+		func(s *mcclient.ClientSession, args *InstanceGroupShowOptions) error {
+			result, err := modules.InstanceGroup.Delete(s, args.ID, nil)
+			if err != nil {
+				return err
+			}
+			printObject(result)
+			return nil
+		},
+	)
+
+}

cmd/climc/shell/servers.go

@@ -1032,4 +1032,29 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type ServerGroupOptions struct {
+		ID      string `help:"ID or name of VM"`
+		GROUPID string `help:"ID or name of instance group"`
+	}
+	R(&ServerGroupOptions{}, "server-join-group", "Join a group", func(s *mcclient.ClientSession,
+		opts *ServerGroupOptions) error {
+
+		ret, err := modules.GroupGuest.Attach(s, opts.ID, opts.GROUPID, jsonutils.JSONNull)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
+	R(&ServerGroupOptions{}, "server-leave-group", "Leave a group", func(s *mcclient.ClientSession,
+		opts *ServerGroupOptions) error {
+
+		ret, err := modules.GroupGuest.Detach(s, opts.ID, opts.GROUPID, jsonutils.JSONNull)
+		if err != nil {
+			return err
+		}
+		printObject(ret)
+		return nil
+	})
 }

pkg/apis/compute/api.go

@@ -132,10 +132,23 @@ type ServerConfigs struct {
 	IsolatedDevices      []*IsolatedDeviceConfig `json:"isolated_devices"`
 	BaremetalDiskConfigs []*BaremetalDiskConfig  `json:"baremetal_disk_configs"`
 
+	InstanceGroupIds []string `json:"groups"`
+
 	// DEPRECATE
 	Suggestion bool `json:"suggestion"`
 }
 
+func NewServerConfigs() *ServerConfigs {
+	return &ServerConfigs{
+		Disks:                make([]*DiskConfig, 0),
+		Networks:             make([]*NetworkConfig, 0),
+		Schedtags:            make([]*SchedtagConfig, 0),
+		IsolatedDevices:      make([]*IsolatedDeviceConfig, 0),
+		BaremetalDiskConfigs: make([]*BaremetalDiskConfig, 0),
+		InstanceGroupIds:     make([]string, 0),
+	}
+}
+
 type DeployConfig struct {
 	Action  string `json:"action"`
 	Path    string `json:"path"`

pkg/compute/models/groupguests.go

@@ -76,14 +76,6 @@ func (self *SGroupguest) GetExtraDetails(ctx context.Context, userCred mcclient.
 	return db.JointModelExtra(self, extra), nil
 }
 
-func (self *SGroupguest) GetGuest() *SGuest {
-	guest, _ := GuestManager.FetchById(self.GuestId)
-	if guest != nil {
-		return guest.(*SGuest)
-	}
-	return nil
-}
-
 func (self *SGroupguest) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
 	return db.DeleteModel(ctx, userCred, self)
 }
@@ -91,3 +83,27 @@ func (self *SGroupguest) Delete(ctx context.Context, userCred mcclient.TokenCred
 func (self *SGroupguest) Detach(ctx context.Context, userCred mcclient.TokenCredential) error {
 	return db.DetachJoint(ctx, userCred, self)
 }
+
+func (self *SGroupguestManager) FetchByGuestId(guestId string) ([]SGroupguest, error) {
+	q := self.Query().Equals("guest_id", guestId)
+	joints := make([]SGroupguest, 0, 1)
+	err := db.FetchModelObjects(self, q, &joints)
+	if err != nil {
+		return nil, err
+	}
+	return joints, err
+}
+
+func (self *SGroupguestManager) Attach(ctx context.Context, groupId, guestId string) (*SGroupguest, error) {
+
+	joint := &SGroupguest{}
+	joint.GuestId = guestId
+	joint.GroupId = groupId
+
+	err := self.TableSpec().Insert(joint)
+	if err != nil {
+		return nil, err
+	}
+	joint.SetModelManager(self, joint)
+	return joint, nil
+}

pkg/compute/models/groups.go

@@ -28,12 +28,14 @@ type SGroupManager struct {
 var GroupManager *SGroupManager
 
 func init() {
+	// GroupManager's Keyword and KeywordPlural is instancegroup and instancegroups because group has been used by
+	// keystone.
 	GroupManager = &SGroupManager{
 		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
 			SGroup{},
 			"groups_tbl",
-			"group",
-			"groups",
+			"instancegroup",
+			"instancegroups",
 		),
 	}
 	GroupManager.SetVirtualObject(GroupManager)
@@ -49,6 +51,9 @@ type SGroup struct {
 	ZoneId string `width:"36" charset:"ascii" nullable:"true" list:"user" update:"user" create:"required"` // Column(VARCHAR(36, charset='ascii'), nullable=True)
 
 	SchedStrategy string `width:"16" charset:"ascii" nullable:"true" default:"" list:"user" update:"user" create:"optional"` // Column(VARCHAR(16, charset='ascii'), nullable=True, default='')
+
+	// the upper limit number of guests with this group in a host
+	Granularity int `nullable:"false" list:"user" get:"user" create:"optional" default:"1"`
 }
 
 func (group *SGroup) GetNetworks() ([]SGroupnetwork, error) {

pkg/compute/models/guests.go

@@ -876,6 +876,20 @@ func (manager *SGuestManager) validateCreateData(
 		input.ResetPassword = &resetPassword
 	}
 
+	// check group
+	if input.InstanceGroupIds != nil && len(input.InstanceGroupIds) != 0 {
+		newGroupIds := make([]string, len(input.InstanceGroupIds))
+		for index, id := range input.InstanceGroupIds {
+			model, err := GroupManager.FetchByIdOrName(userCred, id)
+			if err != nil {
+				return nil, httperrors.NewResourceNotFoundError("no such group %s", id)
+			}
+			newGroupIds[index] = model.GetId()
+		}
+		// list of id or name ==> ids
+		input.InstanceGroupIds = newGroupIds
+	}
+
 	var hypervisor string
 	// var rootStorageType string
 	var osProf osprofile.SOSProfile
@@ -3043,8 +3057,14 @@ func (self *SGuest) attachIsolatedDevice(ctx context.Context, userCred mcclient.
 	return nil
 }
 
-func (self *SGuest) JoinGroups(userCred mcclient.TokenCredential, params *jsonutils.JSONDict) {
-	// TODO
+func (self *SGuest) JoinGroups(ctx context.Context, userCred mcclient.TokenCredential, groupIds []string) error {
+	for _, id := range groupIds {
+		_, err := GroupguestManager.Attach(ctx, id, self.Id)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 type SGuestDiskCategory struct {
@@ -3108,6 +3128,7 @@ func (self *SGuest) LeaveAllGroups(ctx context.Context, userCred mcclient.TokenC
 		return
 	}
 	for _, gg := range groupGuests {
+		gg.SetModelManager(GroupguestManager, &gg)
 		gg.Delete(context.Background(), userCred)
 		var group SGroup
 		gq := GroupManager.Query()
@@ -3116,6 +3137,7 @@ func (self *SGuest) LeaveAllGroups(ctx context.Context, userCred mcclient.TokenC
 			log.Errorln(err.Error())
 			return
 		}
+		group.SetModelManager(GroupManager, &group)
 		db.OpsLog.LogDetachEvent(ctx, self, &group, userCred, nil)
 	}
 }
@@ -3150,6 +3172,17 @@ func (self *SGuest) Delete(ctx context.Context, userCred mcclient.TokenCredentia
 }
 
 func (self *SGuest) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	// delete group
+	joints, err := GroupguestManager.FetchByGuestId(self.Id)
+	if err != nil {
+		return err
+	}
+	for i := range joints {
+		err = joints[i].Detach(ctx, userCred)
+		if err != nil {
+			return err
+		}
+	}
 	return self.SVirtualResourceBase.Delete(ctx, userCred)
 }
 

pkg/compute/models/hosts.go

@@ -4129,3 +4129,41 @@ func (host *SHost) StartMaintainTask(ctx context.Context, userCred mcclient.Toke
 func (host *SHost) IsMaintaining() bool {
 	return utils.IsInStringArray(host.Status, []string{api.HOST_START_MAINTAIN, api.HOST_MAINTAINING, api.HOST_MAINTAIN_FAILE})
 }
+
+// InstanceGroups returns the group of guest in host and their frequency of occurrence
+func (host *SHost) InstanceGroups() ([]SGroup, map[string]int, error) {
+	guests := host.GetGuests()
+	if len(guests) == 0 {
+		return []SGroup{}, make(map[string]int), nil
+	}
+	guestIds := make([]string, len(guests))
+	for i := range guests {
+		guestIds[i] = guests[i].GetId()
+	}
+	q := GroupguestManager.Query().In("guest_id", guestIds)
+	groupguests := make([]SGroupguest, 0, 1)
+	err := db.FetchModelObjects(GroupguestManager, q, &groupguests)
+	if err != nil {
+		return nil, nil, err
+	}
+	groupIds, groupSet := make([]string, 0, len(groupguests)), make(map[string]int)
+	for i := range groupguests {
+		id := groupguests[i].GroupId
+		if _, ok := groupSet[id]; !ok {
+			groupIds = append(groupIds, id)
+			groupSet[id] = 1
+			continue
+		}
+		groupSet[id] += 1
+	}
+	if len(groupIds) == 0 {
+		return []SGroup{}, make(map[string]int), nil
+	}
+	groups := make([]SGroup, 0, len(groupIds))
+	q = GroupManager.Query().In("id", groupIds)
+	err = db.FetchModelObjects(GroupManager, q, &groups)
+	if err != nil {
+		return nil, nil, err
+	}
+	return groups, groupSet, nil
+}

pkg/compute/tasks/guest_batch_create_task.go

@@ -155,7 +155,15 @@ func (self *GuestBatchCreateTask) allocateGuestOnHost(ctx context.Context, guest
 		return err
 	}
 
-	guest.JoinGroups(self.UserCred, self.Params)
+	// join groups
+	if input.InstanceGroupIds != nil && len(input.InstanceGroupIds) != 0 {
+		err := guest.JoinGroups(ctx, self.UserCred, input.InstanceGroupIds)
+		if err != nil {
+			log.Errorf("Join Groups failed: %v", err)
+			guest.SetStatus(self.UserCred, api.VM_CREATE_FAILED, err.Error())
+			return err
+		}
+	}
 
 	if guest.IsPrepaidRecycle() {
 		err := host.RebuildRecycledGuest(ctx, self.UserCred, guest)

pkg/mcclient/modules/mod_groupguest.go

@@ -0,0 +1,31 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import "yunion.io/x/onecloud/pkg/mcclient/modulebase"
+
+var (
+	GroupGuest modulebase.JointResourceManager
+)
+
+func init() {
+	GroupGuest = NewJointComputeManager(
+		"groupguest",
+		"groupguests",
+		[]string{"Guest_ID", "Group_ID", "Tag"},
+		[]string{},
+		&Servers,
+		&InstanceGroup)
+}

pkg/mcclient/modules/mod_instance_group.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import "yunion.io/x/onecloud/pkg/mcclient/modulebase"
+
+var (
+	InstanceGroup modulebase.ResourceManager
+)
+
+func init() {
+	InstanceGroup = NewComputeManager("instancegroup", "instancegroups",
+		[]string{"ID", "Name", "Service_Type", "Parent_Id", "Zone_Id", "Sched_Strategy", "Domain_Id", "Project_Id",
+			"Granularity"},
+		[]string{})
+
+	registerCompute(&InstanceGroup)
+}

pkg/mcclient/options/servers.go

@@ -254,7 +254,7 @@ type ServerCreateOptions struct {
 	ShutdownBehavior string   `help:"Behavior after VM server shutdown" metavar:"<SHUTDOWN_BEHAVIOR>" choices:"stop|terminate"`
 	AutoStart        bool     `help:"Auto start server after it is created"`
 	Deploy           []string `help:"Specify deploy files in virtual server file system" json:"-"`
-	Group            []string `help:"Group of virtual server"`
+	Group            []string `help:"Group ID or Name of virtual server"`
 	System           bool     `help:"Create a system VM, sysadmin ONLY option" json:"is_system"`
 	TaskNotify       *bool    `help:"Setup task notify" json:"-"`
 	DryRun           *bool    `help:"Dry run to test scheduler" json:"-"`
@@ -274,7 +274,9 @@ type ServerCreateOptions struct {
 }
 
 func (o *ServerCreateOptions) ToScheduleInput() (*schedapi.ScheduleInput, error) {
+	// so serious error
 	data := new(schedapi.ServerConfig)
+	data.ServerConfigs = computeapi.NewServerConfigs()
 
 	// only support digit number as for now
 	memSize, err := strconv.Atoi(o.MemSpec)
@@ -311,8 +313,11 @@ func (o *ServerCreateOptions) ToScheduleInput() (*schedapi.ScheduleInput, error)
 		count = o.Count
 	}
 	input := new(schedapi.ScheduleInput)
-	input.Count = count
+
+	data.Count = count
+	data.InstanceGroupIds = o.Group
 	input.ServerConfig = *data
+
 	if o.DryRun != nil && *o.DryRun {
 		input.Details = true
 	}
@@ -400,6 +405,9 @@ func (opts *ServerCreateOptions) Params() (*computeapi.ServerCreateInput, error)
 		params.Suggestion = true
 	}
 
+	// group
+	params.InstanceGroupIds = opts.Group
+
 	return params, nil
 }
 

pkg/scheduler/algorithm/predicates/instance_group_predicate.go

@@ -0,0 +1,70 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package predicates
+
+import (
+	"fmt"
+	"math"
+
+	"yunion.io/x/onecloud/pkg/scheduler/core"
+)
+
+type InstanceGroupPredicate struct {
+	BasePredicate
+}
+
+func (p *InstanceGroupPredicate) Name() string {
+	return "instance_group"
+}
+
+func (p *InstanceGroupPredicate) Clone() core.FitPredicate {
+	return &InstanceGroupPredicate{}
+}
+
+func (p *InstanceGroupPredicate) PreExecute(u *core.Unit, cs []core.Candidater) (bool, error) {
+	schedDate := u.SchedData()
+	if schedDate.InstanceGroupIds == nil || len(schedDate.InstanceGroupIds) == 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
+func (p *InstanceGroupPredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) {
+	h := NewPredicateHelper(p, u, c)
+	schedDate := u.SchedData()
+
+	instanceGroups := c.Getter().InstanceGroups()
+	minFree := math.MaxInt16
+	for _, id := range schedDate.InstanceGroupIds {
+		var free int
+		if _, ok := instanceGroups[id]; ok {
+			free, _ = c.Getter().GetFreeGroupCount(id)
+			if free < 1 {
+				h.AppendPredicateFailMsg(fmt.Sprintf(
+					"the number of guests with same group %s in this host has reached the upper limit", id))
+				break
+			}
+		} else {
+			detail := schedDate.InstanceGroupsDetail[id]
+			free = detail.Granularity
+		}
+		if free < minFree {
+			minFree = free
+		}
+	}
+	// chose the min capacity of groups
+	h.SetCapacity(int64(minFree))
+	return h.GetResult()
+}

pkg/scheduler/algorithmprovider/defaults.go

@@ -45,6 +45,7 @@ func defaultPredicates() sets.String {
 		factory.RegisterFitPredicate("m-GuestDiskschedtagFilter", &predicates.DiskSchedtagPredicate{}),
 		factory.RegisterFitPredicate("n-ServerSkuFilter", &predicates.InstanceTypePredicate{}),
 		factory.RegisterFitPredicate("o-GuestNetschedtagFilter", &predicates.NetworkSchedtagPredicate{}),
+		factory.RegisterFitPredicate("p-GuestDispersionFilter", &predicates.InstanceGroupPredicate{}),
 	)
 }
 

pkg/scheduler/api/sched.go

@@ -24,6 +24,7 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/scheduler"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cmdline"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	o "yunion.io/x/onecloud/pkg/scheduler/options"
 )
@@ -40,6 +41,8 @@ type SchedInfo struct {
 	IsSuggestion          bool            `json:"suggestion"`
 	ShowSuggestionDetails bool            `json:"suggestion_details"`
 	Raw                   string
+
+	InstanceGroupsDetail map[string]*models.SGroup
 }
 
 func FetchSchedInfo(req *http.Request) (*SchedInfo, error) {
@@ -64,6 +67,22 @@ func FetchSchedInfo(req *http.Request) (*SchedInfo, error) {
 		}
 	}
 
+	if data.InstanceGroupIds == nil || len(data.InstanceGroupIds) == 0 {
+		return data, nil
+	}
+	// fill instance group detail
+	groups := make([]models.SGroup, 0, 1)
+	q := models.GroupManager.Query().In("id", data.InstanceGroupIds)
+	err = db.FetchModelObjects(models.GroupManager, q, &groups)
+	if err != nil {
+		return nil, err
+	}
+	details := make(map[string]*models.SGroup)
+	for i := range groups {
+		details[groups[i].Id] = &groups[i]
+	}
+	data.InstanceGroupsDetail = details
+
 	return data, nil
 }
 

pkg/scheduler/api/types.go

@@ -102,3 +102,8 @@ type CandidateNetwork struct {
 	*models.SNetwork
 	Schedtags []models.SSchedtag `json:"schedtags"`
 }
+
+type CandidateGroup struct {
+	*models.SGroup
+	ReferCount int
+}

pkg/scheduler/cache/candidate/base.go

@@ -41,6 +41,8 @@ type BaseHostDesc struct {
 
 	Tenants       map[string]int64          `json:"tenants"`
 	HostSchedtags []computemodels.SSchedtag `json:"schedtags"`
+
+	InstanceGroups map[string]*api.CandidateGroup `json:"instance_groups"`
 }
 
 type baseHostGetter struct {
@@ -107,6 +109,27 @@ func (b baseHostGetter) Storages() []*api.CandidateStorage {
 	return b.h.Storages
 }
 
+func (b baseHostGetter) InstanceGroups() map[string]*api.CandidateGroup {
+	return b.h.InstanceGroups
+}
+
+func (b baseHostGetter) GetFreeGroupCount(groupId string) (int, error) {
+	// Must Be
+	scg, ok := b.h.InstanceGroups[groupId]
+	if !ok {
+		return 0, fmt.Errorf("No such Group id")
+	}
+	free := scg.Granularity - scg.ReferCount
+	if free < 1 {
+		return 0, nil
+	}
+	pendingScg, ok := b.h.GetPendingUsage().InstanceGroupUsage[groupId]
+	if ok {
+		free -= pendingScg.ReferCount
+	}
+	return free, nil
+}
+
 func (b baseHostGetter) Networks() []*api.CandidateNetwork {
 	return b.h.Networks
 }
@@ -209,6 +232,9 @@ func newBaseHostDesc(host *computemodels.SHost) (*BaseHostDesc, error) {
 	if err := desc.fillSchedtags(); err != nil {
 		return nil, fmt.Errorf("Fill schedtag error: %v", err)
 	}
+	if err := desc.fillInstanceGroups(host); err != nil {
+		return nil, fmt.Errorf("Fill instance group error: %v", err)
+	}
 
 	return desc, nil
 }
@@ -340,6 +366,24 @@ func (b *BaseHostDesc) fillStorages(host *computemodels.SHost) error {
 	return nil
 }
 
+func (b *BaseHostDesc) fillInstanceGroups(host *computemodels.SHost) error {
+	candidateSet := make(map[string]*api.CandidateGroup)
+	groups, groupSet, err := host.InstanceGroups()
+	if err != nil {
+		b.InstanceGroups = candidateSet
+		return err
+	}
+	for i := range groups {
+		id := groups[i].GetId()
+		candidateSet[id] = &api.CandidateGroup{
+			SGroup:     &groups[i],
+			ReferCount: groupSet[id],
+		}
+	}
+	b.InstanceGroups = candidateSet
+	return nil
+}
+
 func (h *BaseHostDesc) GetEnableStatus() string {
 	if h.Enabled {
 		return "enable"

pkg/scheduler/core/types.go

@@ -86,6 +86,9 @@ type CandidatePropertyGetter interface {
 	GetFreeStorageSizeOfType(storageType string, useRsvd bool) int64
 
 	GetFreePort(netId string) int
+
+	InstanceGroups() map[string]*api.CandidateGroup
+	GetFreeGroupCount(groupId string) (int, error)
 }
 
 // Candidater replace host Candidate resource info

pkg/scheduler/handler/handler.go

@@ -301,7 +301,7 @@ func doSyncSchedule(c *gin.Context) {
 }
 
 func IsDriverSkipScheduleDirtyMark(driver computemodels.IGuestDriver) bool {
-	return driver.DoScheduleCPUFilter() || driver.DoScheduleMemoryFilter() || driver.DoScheduleStorageFilter()
+	return !(driver.DoScheduleCPUFilter() && driver.DoScheduleMemoryFilter() && driver.DoScheduleStorageFilter())
 }
 
 func setSchedPendingUsage(driver computemodels.IGuestDriver, req *api.SchedInfo, resp *schedapi.ScheduleOutput) error {

pkg/scheduler/models/pending_usage.go

@@ -29,6 +29,7 @@ import (
 	schedapi "yunion.io/x/onecloud/pkg/apis/scheduler"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
+	computemodels "yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/scheduler/api"
 )
 
@@ -281,6 +282,8 @@ type SPendingUsage struct {
 	IsolatedDevice int
 	DiskUsage      *SResourcePendingUsage
 	NetUsage       *SResourcePendingUsage
+	// Lock is not need here
+	InstanceGroupUsage map[string]*api.CandidateGroup
 }
 
 func NewPendingUsageBySchedInfo(hostId string, req *api.SchedInfo) *SPendingUsage {
@@ -289,6 +292,10 @@ func NewPendingUsageBySchedInfo(hostId string, req *api.SchedInfo) *SPendingUsag
 		DiskUsage: NewResourcePendingUsage(nil),
 		NetUsage:  NewResourcePendingUsage(nil),
 	}
+
+	// group init
+	u.InstanceGroupUsage = make(map[string]*api.CandidateGroup)
+
 	if req == nil {
 		return u
 	}
@@ -312,6 +319,15 @@ func NewPendingUsageBySchedInfo(hostId string, req *api.SchedInfo) *SPendingUsag
 		u.NetUsage.Set(id, ocount+1)
 	}
 
+	// group add
+	for _, groupId := range req.InstanceGroupIds {
+		// For now, info about instancegroup in api.SchedInfo is only "ID",
+		// but in the future, info may increase
+		group := &computemodels.SGroup{}
+		group.Id = groupId
+		u.InstanceGroupUsage[groupId] = &api.CandidateGroup{group, 1}
+	}
+
 	return u
 }
 
@@ -322,6 +338,7 @@ func (self *SPendingUsage) ToMap() map[string]interface{} {
 		"isolated_device": self.IsolatedDevice,
 		"disk":            self.DiskUsage.ToMap(),
 		"net":             self.NetUsage.ToMap(),
+		"instance_groups": self.InstanceGroupUsage,
 	}
 }
 
@@ -331,6 +348,13 @@ func (self *SPendingUsage) Add(sUsage *SPendingUsage) {
 	self.IsolatedDevice = self.IsolatedDevice + sUsage.IsolatedDevice
 	self.DiskUsage.Add(sUsage.DiskUsage)
 	self.NetUsage.Add(sUsage.NetUsage)
+	for id, cg := range sUsage.InstanceGroupUsage {
+		if scg, ok := self.InstanceGroupUsage[id]; ok {
+			scg.ReferCount += cg.ReferCount
+			continue
+		}
+		self.InstanceGroupUsage[id] = cg
+	}
 }
 
 func (self *SPendingUsage) Sub(sUsage *SPendingUsage) {
@@ -339,6 +363,16 @@ func (self *SPendingUsage) Sub(sUsage *SPendingUsage) {
 	self.IsolatedDevice = quotas.NonNegative(self.IsolatedDevice - sUsage.IsolatedDevice)
 	self.DiskUsage.Sub(sUsage.DiskUsage)
 	self.NetUsage.Sub(sUsage.NetUsage)
+	for id, cg := range sUsage.InstanceGroupUsage {
+		if scg, ok := self.InstanceGroupUsage[id]; ok {
+			count := scg.ReferCount - cg.ReferCount
+			if count <= 0 {
+				delete(self.InstanceGroupUsage, id)
+				continue
+			}
+			scg.ReferCount = count
+		}
+	}
 }
 
 func (self *SPendingUsage) IsEmpty() bool {
@@ -357,6 +391,9 @@ func (self *SPendingUsage) IsEmpty() bool {
 	if !self.NetUsage.IsEmpty() {
 		return false
 	}
+	if len(self.InstanceGroupUsage) != 0 {
+		return false
+	}
 	return true
 }