fix(region): Fix problems of AutoScaling

1. Add List dispaly Params 'brand' and List filter Params 'brand' for
GuestTemplate and ScalingGroup.
2. ScalingActivity's list elem should be sorted in order by StartTime and EndTime
3. HealthCheckCycle and HealthCheckGov in ScalingGroup work now
4. Detach all ScalingGroup when pending delete Guest
5. If the scaling behavior of the scaling group fails n(n is 3 for now) times consecutively,
the controller will automatically disablet this group
6. Timer will trigger ScalingPolicy throught request but func call

pkg/apis/compute/guesttemplate.go

@@ -67,6 +67,7 @@ type GuestTemplateListInput struct {
 	RegionalFilterListInput
 	VpcFilterListInput
 	BillingType string `json:"billing_type"`
+	Brand       string `json:"brand"`
 }
 
 type GuestTemplateConfigInfo struct {

pkg/apis/compute/scaling_group.go

@@ -19,6 +19,7 @@ import "yunion.io/x/onecloud/pkg/apis"
 type ScalingGroupCreateInput struct {
 	apis.VirtualResourceCreateInput
 	apis.EnabledBaseResourceCreateInput
+	VpcResourceInput
 
 	// description: cloud region id or name
 	// required: true
@@ -32,13 +33,6 @@ type ScalingGroupCreateInput struct {
 	// example: kvm
 	Hypervisor string `json:"hypervisor"`
 
-	// description: VPC(ID or Name)
-	// example: vpc-1234
-	Vpc string `json:"vpc"`
-
-	// swagger: ignore
-	VpcId string `json:"vpc_id"`
-
 	// description: 多个网络(ID或者Name),
 	// example: n-test-one
 	Networks []string `json:"networks"`
@@ -113,6 +107,10 @@ type ScalingGroupListInput struct {
 	// description: hypervisor
 	// example: kvm
 	Hypervisor string `json:"hypervisor"`
+
+	// desription: 平台
+	// example: OneCloud
+	Brand string `json:"brand"`
 }
 
 type ScalingGroupDetails struct {
@@ -132,6 +130,10 @@ type ScalingGroupDetails struct {
 	// example: 3
 	ScalingPolicyNumber int `json:"scaling_policy_number"`
 
+	// description: 平台
+	// example: OneCloud
+	Brand string `json:"brand"`
+
 	// description: 网络ID
 	// example: net-12345
 	Networks []string `json:"networks"`

pkg/compute/models/guest_template.go

@@ -148,6 +148,21 @@ var HypervisorBrandMap = map[string]string{
 	computeapis.HYPERVISOR_CTYUN:     computeapis.CLOUD_PROVIDER_CTYUN,
 }
 
+var BrandHypervisorMap = map[string]string{
+	computeapis.CLOUD_PROVIDER_ONECLOUD:  computeapis.HYPERVISOR_KVM,
+	computeapis.CLOUD_PROVIDER_VMWARE:    computeapis.HYPERVISOR_ESXI,
+	computeapis.CLOUD_PROVIDER_ALIYUN:    computeapis.HYPERVISOR_ALIYUN,
+	computeapis.CLOUD_PROVIDER_QCLOUD:    computeapis.HYPERVISOR_QCLOUD,
+	computeapis.CLOUD_PROVIDER_AZURE:     computeapis.HYPERVISOR_AZURE,
+	computeapis.CLOUD_PROVIDER_AWS:       computeapis.HYPERVISOR_AWS,
+	computeapis.CLOUD_PROVIDER_HUAWEI:    computeapis.HYPERVISOR_HUAWEI,
+	computeapis.CLOUD_PROVIDER_OPENSTACK: computeapis.HYPERVISOR_OPENSTACK,
+	computeapis.CLOUD_PROVIDER_UCLOUD:    computeapis.HYPERVISOR_UCLOUD,
+	computeapis.CLOUD_PROVIDER_ZSTACK:    computeapis.HYPERVISOR_ZSTACK,
+	computeapis.CLOUD_PROVIDER_GOOGLE:    computeapis.HYPERVISOR_GOOGLE,
+	computeapis.CLOUD_PROVIDER_CTYUN:     computeapis.HYPERVISOR_CTYUN,
+}
+
 func Hypervisor2Brand(hypervisor string) string {
 	brand, ok := HypervisorBrandMap[hypervisor]
 	if !ok {
@@ -156,6 +171,14 @@ func Hypervisor2Brand(hypervisor string) string {
 	return brand
 }
 
+func Brand2Hypervisor(brand string) string {
+	hypervisor, ok := BrandHypervisorMap[brand]
+	if !ok {
+		return "unkown"
+	}
+	return hypervisor
+}
+
 func (gtm *SGuestTemplateManager) validateData(
 	ctx context.Context,
 	userCred mcclient.TokenCredential,
@@ -596,6 +619,9 @@ func (manager *SGuestTemplateManager) ListItemFilter(
 	if len(input.BillingType) > 0 {
 		q = q.Equals("billing_type", input.BillingType)
 	}
+	if len(input.Brand) > 0 {
+		q = q.Equals("hypervisor", Brand2Hypervisor(input.Brand))
+	}
 	return q, nil
 }
 

pkg/compute/models/scaling_activity.go

@@ -178,5 +178,10 @@ func (sam *SScalingActivityManager) ListItemFilter(ctx context.Context, q *sqlch
 	if err != nil {
 		return nil, err
 	}
-	return sam.SScalingGroupResourceBaseManager.ListItemFilter(ctx, q, userCred, input.ScalingGroupFilterListInput)
+	q, err = sam.SScalingGroupResourceBaseManager.ListItemFilter(ctx, q, userCred, input.ScalingGroupFilterListInput)
+	if err != nil {
+		return nil, err
+	}
+	q = q.Desc("start_time").Desc("end_time")
+	return q, nil
 }

pkg/compute/models/scaling_group.go

@@ -84,6 +84,8 @@ type SScalingGroup struct {
 
 	// Time to allow scale
 	AllowScaleTime time.Time
+	// NextCheckTime descripe the next time to check instance's health
+	NextCheckTime time.Time
 }
 
 var ScalingGroupManager *SScalingGroupManager
@@ -134,18 +136,10 @@ func (sgm *SScalingGroupManager) ValidateCreateData(ctx context.Context, userCre
 	input.CloudregionId = cloudregion.GetId()
 
 	// check vpc
-	idOrName = input.Vpc
-	if len(input.Vpc) != 0 {
-		idOrName = input.Vpc
-	}
-	vpc, err := VpcManager.FetchByIdOrName(userCred, idOrName)
-	if errors.Cause(err) == sql.ErrNoRows {
-		return input, httperrors.NewInputParameterError("no such vpc %s", idOrName)
-	}
+	_, input.VpcResourceInput, err = ValidateVpcResourceInput(userCred, input.VpcResourceInput)
 	if err != nil {
-		return input, errors.Wrap(err, "VpcManager.FetchByIdOrName")
+		return input, err
 	}
-	input.VpcId = vpc.GetId()
 
 	// check networks
 	if len(input.Networks) == 0 {
@@ -172,8 +166,8 @@ func (sgm *SScalingGroupManager) ValidateCreateData(ctx context.Context, userCre
 		if vpc == nil {
 			return input, fmt.Errorf("Get vpc of network '%s' failed", networks[i].Id)
 		}
-		if vpc.Id != input.VpcId {
-			return input, httperrors.NewInputParameterError("network '%s' not in vpc '%s'", networks[i].Id, input.VpcId)
+		if vpc.Id != input.Vpc {
+			return input, httperrors.NewInputParameterError("network '%s' not in vpc '%s'", networks[i].Id, input.Vpc)
 		}
 		input.Networks[i] = networks[i].Id
 	}
@@ -191,7 +185,7 @@ func (sgm *SScalingGroupManager) ValidateCreateData(ctx context.Context, userCre
 		return input, errors.Wrap(err, "GuestTempalteManager.FetchByIdOrName")
 	}
 	if ok, reason := guestTemplate.(*SGuestTemplate).Validate(ctx, userCred, ownerId,
-		SGuestTemplateValidate{input.Hypervisor, input.CloudregionId, input.VpcId, input.Networks}); !ok {
+		SGuestTemplateValidate{input.Hypervisor, input.CloudregionId, input.Vpc, input.Networks}); !ok {
 		return input, httperrors.NewInputParameterError("the guest template %s is not valid in cloudregion %s, "+
 			"reason: %s", idOrName, input.CloudregionId, reason)
 	}
@@ -328,9 +322,12 @@ func (sgm *SScalingGroupManager) ListItemFilter(ctx context.Context, q *sqlchemy
 	if err != nil {
 		return q, err
 	}
-	if len(input.Hypervisor) != 0 {
+	if len(input.Hypervisor) > 0 {
 		q = q.Equals("hypervisor", input.Hypervisor)
 	}
+	if len(input.Brand) > 0 {
+		q = q.Equals("hypervisor", Brand2Hypervisor(input.Brand))
+	}
 	return q, nil
 }
 
@@ -369,6 +366,7 @@ func (sgm *SScalingGroupManager) FetchCustomizeColumns(
 		rows[i].InstanceNumber = n
 		n, _ = sg.ScalingPolicyNumber()
 		rows[i].ScalingPolicyNumber = n
+		rows[i].Brand = Hypervisor2Brand(sg.Hypervisor)
 	}
 	return rows
 }
@@ -594,9 +592,11 @@ func (sg *SScalingGroup) PostCreate(ctx context.Context, userCred mcclient.Token
 			return
 		}
 	}
+	now := time.Now()
 	db.Update(sg, func() error {
 		sg.Status = api.SG_STATUS_READY
-		sg.AllowScaleTime = time.Now()
+		sg.AllowScaleTime = now
+		sg.NextCheckTime = now.Add(time.Duration(sg.HealthCheckCycle) * time.Second)
 		sg.SetEnabled(true)
 		return nil
 	})

pkg/compute/models/scaling_policy.go

@@ -374,18 +374,19 @@ func (sp *SScalingPolicy) PerformTrigger(ctx context.Context, userCred mcclient.
 		return nil, nil
 	}
 
-	if !data.Contains("alarm_id") {
-		// considered manual trigger
+	unmanual, _ := data.Bool("unmanual")
+	if !unmanual {
 		triggerDesc = SScalingManual{SScalingPolicyBase{sp.Id}}
-
 	} else {
-		alarmId, _ := data.GetString("alarm_id")
 		trigger, err := sp.Trigger(nil)
 		if err != nil {
 			return nil, errors.Wrap(err, "fetch trigger failed")
 		}
-		if alarmId != triggerDesc.(*SScalingAlarm).AlarmId {
-			return nil, httperrors.NewInputParameterError("mismatched alarm id")
+		if data.Contains("alarm_id") {
+			alarmId, _ := data.GetString("alarm_id")
+			if alarmId != trigger.(*SScalingAlarm).AlarmId {
+				return nil, httperrors.NewInputParameterError("mismatched alarm id")
+			}
 		}
 		if !trigger.IsTrigger() {
 			return nil, nil

pkg/compute/models/scalinggroup_guest.go

@@ -16,6 +16,7 @@ package models
 
 import (
 	"context"
+	"time"
 
 	"yunion.io/x/pkg/tristate"
 
@@ -93,6 +94,8 @@ func (sgg *SScalingGroupGuest) SetGuestStatus(status string) error {
 	}
 	_, err := db.Update(sgg, func() error {
 		sgg.GuestStatus = status
+		sgg.UpdatedAt = time.Now()
+		sgg.UpdateVersion += 1
 		return nil
 	})
 	return err

pkg/compute/options/options.go

@@ -147,7 +147,7 @@ type SASControllerOptions struct {
 	TimerInterval       int `help:"The interval between the tow checks about timer, unit: s" default:"60"`
 	ConcurrentUpper     int `help:"This represents the upper limit of concurrent sacling sctivities" default:"500"`
 	CheckScaleInterval  int `help:"The interval between the two checks about scaling, unit: s" default:"60"`
-	CheckHealthInterval int `help:"The interval bewteen the two check about instance's health unit: m" default:"5"`
+	CheckHealthInterval int `help:"The interval bewteen the two check about instance's health unit: m" default:"1"`
 }
 
 var (

pkg/compute/tasks/guest_delete_task.go

@@ -182,6 +182,8 @@ func (self *GuestDeleteTask) OnSyncConfigComplete(ctx context.Context, obj db.IS
 
 	// try to leave all groups
 	guest.LeaveAllGroups(ctx, self.UserCred)
+	// detach
+	guest.DetachScalingGroup(ctx, self.UserCred)
 	isPurge := jsonutils.QueryBoolean(self.Params, "purge", false)
 	overridePendingDelete := jsonutils.QueryBoolean(self.Params, "override_pending_delete", false)
 
@@ -295,8 +297,6 @@ func (self *GuestDeleteTask) OnGuestDeleteComplete(ctx context.Context, obj db.I
 	guest.EjectIso(self.UserCred)
 	guest.DeleteEip(ctx, self.UserCred)
 	guest.GetDriver().OnDeleteGuestFinalCleanup(ctx, guest, self.UserCred)
-	// detach
-	guest.DetachScalingGroup(ctx, self.UserCred)
 	self.DeleteGuest(ctx, guest)
 }
 

pkg/compute/tasks/guest_detach_scalinggroup.go

@@ -87,7 +87,6 @@ func (self *GuestDetachScalingGroupTask) OnInit(ctx context.Context, obj db.ISta
 func (self *GuestDetachScalingGroupTask) OnDetachLoadbalancerComplete(ctx context.Context, sg *models.SScalingGroup, data jsonutils.JSONObject) {
 	guestId, _ := self.Params.GetString("guest")
 	delete, _ := self.Params.Bool("delete_server")
-	self.SetStage("OnDeleteGuestComplete", nil)
 	if !delete {
 		self.OnDeleteGuestComplete(ctx, sg, data)
 		return
@@ -98,6 +97,7 @@ func (self *GuestDetachScalingGroupTask) OnDetachLoadbalancerComplete(ctx contex
 		return
 	}
 	self.Params.Set("guest_name", jsonutils.NewString(guest.GetName()))
+	self.SetStage("OnDeleteGuestComplete", nil)
 	if err := guest.StartDeleteGuestTask(ctx, self.UserCred, self.Id, true, true, true); err != nil {
 		self.taskFailed(ctx, sg, nil, err.Error())
 	}

pkg/controller/autoscaling/controller.go

@@ -38,6 +38,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type SASController struct {
@@ -45,7 +46,9 @@ type SASController struct {
 	scalingQueue    chan struct{}
 	timerQueue      chan struct{}
 	scalingGroupSet *SLockedSet
-	scalingQuery    *sqlchemy.SQuery
+	scalingSql      *sqlchemy.SQuery
+	// record the consecutive failures of scaling group's scale
+	failRecord map[string]int
 }
 
 type SScalingInfo struct {
@@ -82,20 +85,23 @@ var ASController = new(SASController)
 
 func (asc *SASController) Init(options options.SASControllerOptions, cronm *cronman.SCronJobManager) {
 	asc.options = options
-	cronm.AddJobAtIntervals("CheckTimer", time.Duration(options.TimerInterval)*time.Second, asc.Timer)
-	cronm.AddJobAtIntervals("CheckScale", time.Duration(options.CheckScaleInterval)*time.Second, asc.CheckScale)
-	cronm.AddJobAtIntervals("CheckInstanceHealth", time.Duration(options.CheckHealthInterval)*time.Minute, asc.CheckInstanceHealth)
+	cronm.AddJobAtIntervalsWithStartRun("CheckTimer", time.Duration(options.TimerInterval)*time.Second, asc.Timer, true)
+	cronm.AddJobAtIntervalsWithStartRun("CheckScale", time.Duration(options.CheckScaleInterval)*time.Second, asc.CheckScale, true)
+	cronm.AddJobAtIntervalsWithStartRun("CheckInstanceHealth", time.Duration(options.CheckHealthInterval)*time.Minute, asc.CheckInstanceHealth, true)
 	asc.timerQueue = make(chan struct{}, 20)
 	asc.scalingQueue = make(chan struct{}, options.ConcurrentUpper)
 	asc.scalingGroupSet = &SLockedSet{set: sets.NewString()}
+	asc.failRecord = make(map[string]int)
 
+	// init scalingSql
 	sggQ := models.ScalingGroupGuestManager.Query("scaling_group_id").GroupBy("scaling_group_id")
 	sggQ = sggQ.AppendField(sqlchemy.COUNT("total", sggQ.Field("guest_id")))
 	sggSubQ := sggQ.SubQuery()
 	sgQ := models.ScalingGroupManager.Query("id", "desire_instance_number").IsTrue("enabled")
-	asc.scalingQuery = sgQ.LeftJoin(sggSubQ, sqlchemy.AND(sqlchemy.Equals(sggSubQ.Field("scaling_group_id"),
+	sgQ = sgQ.LeftJoin(sggSubQ, sqlchemy.AND(sqlchemy.Equals(sggSubQ.Field("scaling_group_id"),
 		sgQ.Field("id")), sqlchemy.NotEquals(sggSubQ.Field("total"), sgQ.Field("desire_instance_number"))))
 	sgQ.AppendField(sggSubQ.Field("total"))
+	asc.scalingSql = sgQ
 
 	// check all scaling activity
 	log.Infof("check and update scaling activities...")
@@ -112,6 +118,33 @@ func (asc *SASController) Init(options options.SASControllerOptions, cronm *cron
 	log.Infof("check and update scalngactivities complete")
 }
 
+func (asc *SASController) PreScale(group *models.SScalingGroup, userCred mcclient.TokenCredential) bool {
+	maxFailures := 3
+	disableReason := fmt.Sprintf("The number of consecutive failures of creating a machine exceeds %d times", maxFailures)
+	times := asc.failRecord[group.GetId()]
+	if times >= maxFailures {
+		_, err := db.Update(group, func() error {
+			group.SetEnabled(false)
+			return nil
+		})
+		if err != nil {
+			return false
+		}
+		logclient.AddSimpleActionLog(group, logclient.ACT_DISABLE, disableReason, userCred, true)
+		return false
+	}
+	return true
+}
+
+func (asc *SASController) Finish(groupId string, success bool) {
+	asc.scalingGroupSet.Delete(groupId)
+	if success {
+		asc.failRecord[groupId] = 0
+		return
+	}
+	asc.failRecord[groupId]++
+}
+
 // SScalingGroupShort wrap the ScalingGroup's ID and DesireInstanceNumber with field 'total' which means the total
 // guests number in this ScalingGroup
 type SScalingGroupShort struct {
@@ -143,12 +176,19 @@ func (asc *SASController) CheckScale(ctx context.Context, userCred mcclient.Toke
 
 func (asc *SASController) Scale(ctx context.Context, userCred mcclient.TokenCredential, short SScalingGroupShort) {
 	log.Debugf("scale for ScalingGroup '%s', desire: %d, total: %d", short.ID, short.DesireInstanceNumber, short.Total)
-	var err error
+	var (
+		err     error
+		success = true
+	)
+	setFail := func(sa *models.SScalingActivity, reason string) {
+		success = false
+		err = sa.SetFailed("", reason)
+	}
 	defer func() {
 		if err != nil {
 			log.Errorf("Scaling for ScalingGroup '%s': %s", short.ID, err.Error())
 		}
-		asc.scalingGroupSet.Delete(short.ID)
+		asc.Finish(short.ID, success)
 		<-asc.scalingQueue
 		log.Debugf("Scale for ScalingGroup '%s' finished", short.ID)
 	}()
@@ -161,8 +201,11 @@ func (asc *SASController) Scale(ctx context.Context, userCred mcclient.TokenCred
 		}
 		return
 	}
-	log.Debugf("fetch the latest total")
 	sg := model.(*models.SScalingGroup)
+	if !asc.PreScale(sg, userCred) {
+		success = true
+		return
+	}
 	total, err := sg.GuestNumber()
 	if err != nil {
 		return
@@ -195,12 +238,12 @@ func (asc *SASController) Scale(ctx context.Context, userCred mcclient.TokenCred
 		// check guest template
 		gt := sg.GetGuestTemplate()
 		if gt == nil {
-			err = scalingActivity.SetFailed("", fmt.Sprintf("fetch GuestTemplate of ScalingGroup '%s' error", sg.Id))
+			setFail(scalingActivity, fmt.Sprintf("fetch GuestTemplate of ScalingGroup '%s' error", sg.Id))
 			return
 		}
 		nets, err := sg.NetworkIds()
 		if err != nil {
-			err = scalingActivity.SetFailed("", fmt.Sprintf("fetch Networks of ScalingGroup '%s' error", sg.Id))
+			setFail(scalingActivity, fmt.Sprintf("fetch Networks of ScalingGroup '%s' error", sg.Id))
 			return
 		}
 		valid, msg := gt.Validate(context.TODO(), auth.AdminCredential(), gt.GetOwnerId(),
@@ -218,7 +261,7 @@ func (asc *SASController) Scale(ctx context.Context, userCred mcclient.TokenCred
 		succeedInstances, err := asc.CreateInstances(ctx, userCred, ownerId, sg, gt, nets[0], num)
 		switch len(succeedInstances) {
 		case 0:
-			err = scalingActivity.SetFailed("", fmt.Sprintf("All instances create failed: %s", err.Error()))
+			setFail(scalingActivity, fmt.Sprintf("All instances create failed: %s", err.Error()))
 		case num:
 			var action bytes.Buffer
 			action.WriteString("Instances ")
@@ -244,7 +287,7 @@ func (asc *SASController) Scale(ctx context.Context, userCred mcclient.TokenCred
 		succeedInstances, err := asc.DetachInstances(ctx, userCred, ownerId, sg, num)
 		switch len(succeedInstances) {
 		case 0:
-			err = scalingActivity.SetFailed("", fmt.Sprintf("All instance remove failed: %s", err.Error()))
+			setFail(scalingActivity, fmt.Sprintf("All instance remove failed: %s", err.Error()))
 		case num:
 			var action bytes.Buffer
 			action.WriteString("Instances ")
@@ -679,7 +722,7 @@ func (asc *SASController) countPRAndRequests(num int) (int, int) {
 
 // ScalingGroupNeedScale will fetch all ScalingGroup need to scale
 func (asc *SASController) ScalingGroupsNeedScale() ([]SScalingGroupShort, error) {
-	rows, err := asc.scalingQuery.Rows()
+	rows, err := asc.scalingSql.Rows()
 	if err != nil {
 		return nil, errors.Wrap(err, "execute scaling sql error")
 	}
@@ -687,7 +730,7 @@ func (asc *SASController) ScalingGroupsNeedScale() ([]SScalingGroupShort, error)
 	sgShorts := make([]SScalingGroupShort, 0, 10)
 	for rows.Next() {
 		sgPro := SScalingGroupShort{}
-		err := asc.scalingQuery.Row2Struct(rows, &sgPro)
+		err := asc.scalingSql.Row2Struct(rows, &sgPro)
 		if err != nil {
 			return nil, errors.Wrap(err, "sqlchemy.SQuery.Row2Struct error")
 		}

pkg/controller/autoscaling/health_check.go

@@ -20,9 +20,11 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/sqlchemy"
 
 	apis "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
@@ -35,31 +37,65 @@ var UnhealthStatus = []string{
 }
 
 type sUnnormalGuest struct {
-	Id            string `json:"id"`
-	Status        string `json:"status"`
-	ScalngGroupId string `json:"scaling_group_id"`
+	Id                 string    `json:"id"`
+	Status             string    `json:"status"`
+	ScalngGroupId      string    `json:"scaling_group_id"`
+	CreateCompleteTime time.Time `json:"create_complete_time"`
+}
+
+func (asc *SASController) HealthCheckSql() *sqlchemy.SQuery {
+	now := time.Now()
+	sgSubQ := models.ScalingGroupManager.Query("id").IsTrue("enabled").LT("next_check_time", now).SubQuery()
+	sggQ := models.ScalingGroupGuestManager.Query("guest_id", "scaling_group_id", "updated_at").Equals("guest_status", apis.SG_GUEST_STATUS_READY)
+	sggSubQ := sggQ.Join(sgSubQ, sqlchemy.Equals(sgSubQ.Field("id"), sggQ.Field("scaling_group_id"))).SubQuery()
+	q := models.GuestManager.Query("id", "status").In("status", UnhealthStatus)
+	q = q.Join(sggSubQ, sqlchemy.Equals(q.Field("id"), sggSubQ.Field("guest_id")))
+	q = q.AppendField(sggSubQ.Field("scaling_group_id"), sggSubQ.Field("updated_at", "create_complete_time"))
+	return q
 }
 
 func (asc *SASController) CheckInstanceHealth(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
-	checkEarliestTime := time.Now().Add(-10 * time.Minute)
 	// Fetch all unhealth status instace
 	unnormalGuests := make([]sUnnormalGuest, 0, 5)
-	sgQ := models.ScalingGroupManager.Query("id").IsTrue("enabled").SubQuery()
-	sggQ := models.ScalingGroupGuestManager.Query("guest_id", "scaling_group_id").In("scaling_group_id", sgQ).SubQuery()
-	q := models.GuestManager.Query("id", "status").In("status", UnhealthStatus).LT("created_at", checkEarliestTime)
-	q = q.Join(sggQ, sqlchemy.Equals(q.Field("id"), sggQ.Field("guest_id")))
-	q = q.AppendField(sggQ.Field("scaling_group_id"))
-	rows, err := q.Rows()
+	scalingGroupIdSet := sets.NewString()
+	rows, err := asc.HealthCheckSql().Rows()
 	if err != nil {
 		log.Errorf("GuestManager's SQuery.Rows: %s", err.Error())
 	}
 	for rows.Next() {
 		var ug sUnnormalGuest
-		rows.Scan(&ug.Id, &ug.Status, &ug.ScalngGroupId)
+		rows.Scan(&ug.Id, &ug.Status, &ug.ScalngGroupId, &ug.CreateCompleteTime)
+		scalingGroupIdSet.Insert(ug.ScalngGroupId)
 		unnormalGuests = append(unnormalGuests, ug)
 	}
 	rows.Close()
 
+	// fetch all ScalingGroup
+	scalingGroups := make([]models.SScalingGroup, 0, scalingGroupIdSet.Len())
+	q := models.ScalingGroupManager.Query().In("id", scalingGroupIdSet.UnsortedList())
+	err = db.FetchModelObjects(models.ScalingGroupManager, q, &scalingGroups)
+	if err != nil {
+		log.Errorf("unable to fetch ScalingGroup")
+		return
+	}
+	scalingGroupMap := make(map[string]*models.SScalingGroup, len(scalingGroups))
+	for i := range scalingGroups {
+		scalingGroupMap[scalingGroups[i].GetId()] = &scalingGroups[i]
+	}
+
+	// update NextCheckTime for ScalingGroup
+	now := time.Now()
+	for i := range scalingGroups {
+		sg := &scalingGroups[i]
+		_, err := db.Update(sg, func() error {
+			sg.NextCheckTime = now.Add(time.Duration(sg.HealthCheckCycle) * time.Second)
+			return nil
+		})
+		if err != nil {
+			log.Errorf("unable to update NextCheckTime for ScalingGroup '%s'", sg.GetId())
+		}
+	}
+
 	// request to detach
 	readyGuestList := make([]string, 0, 5)
 	readyGuestMap := make(map[string]string, 5)
@@ -68,7 +104,11 @@ func (asc *SASController) CheckInstanceHealth(ctx context.Context, userCred mccl
 	removeParams.Set("delete_server", jsonutils.JSONTrue)
 	removeParams.Set("auto", jsonutils.JSONTrue)
 	session := auth.GetSession(ctx, userCred, "", "")
-	for _, ug := range unnormalGuests {
+	for i := range unnormalGuests {
+		ug := unnormalGuests[i]
+		if ug.CreateCompleteTime.Add(time.Duration(scalingGroupMap[ug.Id].HealthCheckGov) * time.Second).After(now) {
+			continue
+		}
 		if ug.Status == apis.VM_READY {
 			readyGuestList = append(readyGuestList, ug.Id)
 			readyGuestMap[ug.Id] = ug.ScalngGroupId
@@ -81,6 +121,8 @@ func (asc *SASController) CheckInstanceHealth(ctx context.Context, userCred mccl
 		}
 	}
 
+	// check NextCheckTime for ScalngGroup
+
 	if len(readyGuestList) > 0 {
 		go func() {
 			time.Sleep(2 * time.Minute)

pkg/controller/autoscaling/timer.go

@@ -18,12 +18,15 @@ import (
 	"context"
 	"time"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
 	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
 type STimeScope struct {
@@ -57,6 +60,9 @@ func (asc *SASController) Timer(ctx context.Context, userCred mcclient.TokenCred
 	}
 	log.Debugf("total %d need to exec, %v", len(scalingTimers), scalingTimers)
 	log.Debugf("timeScope: start: %s, end: %s", timeScope.Start, timeScope.End)
+	session := auth.GetSession(ctx, userCred, "", "")
+	triggerParams := jsonutils.NewDict()
+	triggerParams.Set("unmanual", jsonutils.JSONTrue)
 	for i := range scalingTimers {
 		scalingTimer := scalingTimers[i]
 		asc.timerQueue <- struct{}{}
@@ -77,19 +83,10 @@ func (asc *SASController) Timer(ctx context.Context, userCred mcclient.TokenCred
 					return
 				}
 			}
-			sp, err := scalingTimer.ScalingPolicy()
+			_, err = modules.ScalingPolicy.PerformAction(session, scalingTimer.ScalingPolicyId, "trigger",
+				triggerParams)
 			if err != nil {
-				log.Errorf("fail to get ScalingPolicy of ScalingTimer '%s': %s", scalingTimer.Id, err)
-				return
-			}
-			sg, err := sp.ScalingGroup()
-			if err != nil {
-				log.Errorf("fail to get ScalingGroup of ScalingPolicy '%s': %s", sp.Id, err)
-				return
-			}
-			err = sg.Scale(ctx, &scalingTimer, sp)
-			if err != nil {
-				log.Errorf("ScalingGroup '%s' scale error", sg.Id)
+				log.Errorf("unable to request to trigger ScalingPolicy '%s'", scalingTimer.ScalingPolicyId)
 			}
 			scalingTimer.Update(timeScope.End)
 			err = models.ScalingTimerManager.TableSpec().InsertOrUpdate(&scalingTimer)