Merge pull request #5644 from zexi/feature/alert-pause

monitor: add paused api

cmd/climc/shell/monitor/alert.go

@@ -97,4 +97,20 @@ func init() {
 			fmt.Println(ret.JSON(ret).YAMLString())
 			return nil
 		})
+
+	R(&options.AlertPauseOptions{}, aN("pause"), "Pause or unpause alert",
+		func(s *mcclient.ClientSession, args *options.AlertPauseOptions) error {
+			data := new(monitorapi.AlertPauseInput)
+			if args.UnPause {
+				data.Paused = false
+			} else {
+				data.Paused = true
+			}
+			ret, err := monitor.Alerts.PerformAction(s, args.ID, "pause", data.JSON(data))
+			if err != nil {
+				return err
+			}
+			printObject(ret)
+			return nil
+		})
 }

pkg/apis/monitor/alert.go

@@ -105,12 +105,18 @@ type AlertCreateInput struct {
 	Name string `json:"name"`
 	// 报警执行频率
 	Frequency int64 `json:"frequency"`
+	// 报警持续时间
+	For int64 `json:"for"`
 	// 报警设置
 	Settings AlertSetting `json:"settings"`
 	// 启用报警
 	Enabled *bool `json:"enabled"`
 	// 报警级别
 	Level string `json:"level"`
+	// 没有收到监控指标时将当前报警状态设置为对应的状态
+	NoDataState string `json:"no_data_state"`
+	// 报警执行错误将当前报警状态设置为对应的状态
+	ExecutionErrorState string `json:"execution_error_state"`
 }
 
 type AlertUpdateInput struct {
@@ -126,6 +132,10 @@ type AlertUpdateInput struct {
 	Enabled *bool `json:"enabled"`
 	// 报警级别
 	Level *string `json:"level"`
+	// 没有收到监控指标时将当前报警状态设置为对应的状态
+	NoDataState string `json:"no_data_state"`
+	// 报警执行错误将当前报警状态设置为对应的状态
+	ExecutionErrorState string `json:"execution_error_state"`
 }
 
 type AlertListInput struct {
@@ -172,16 +182,8 @@ type AlertTestRunOutput struct {
 	EndTime        time.Time         `json:"end_time"`
 }
 
-type AlertAttachNotificationInput struct {
+type AlertPauseInput struct {
 	apis.Meta
 
-	NotificationId string `json:"notification_id"`
-	UsedBy         string `json:"used_by"`
-}
-
-type AlertAttachNotificationOutput struct {
-	apis.Meta
-
-	NotificationId string `json:"notification_id"`
-	UsedBy         string `json:"used_by"`
+	Paused bool `json:"paused"`
 }

pkg/mcclient/modules/monitor/alert.go

@@ -35,7 +35,7 @@ type SAlertManager struct {
 
 func NewAlertManager() *SAlertManager {
 	man := modules.NewMonitorV2Manager("alert", "alerts",
-		[]string{"id", "name", "state", "frequency", "enabled", "settings"},
+		[]string{"id", "name", "frequency", "enabled", "settings", "state"},
 		[]string{})
 	return &SAlertManager{
 		ResourceManager: &man,

pkg/mcclient/modules/monitor/helper.go

@@ -10,11 +10,14 @@ import (
 
 // AlertConfig is a helper to generate monitor service alert related api input
 type AlertConfig struct {
-	name       string
-	frequency  int64
-	level      string
-	enabled    bool
-	conditions []*AlertCondition
+	name           string
+	frequency      int64
+	forTime        int64
+	level          string
+	enabled        bool
+	conditions     []*AlertCondition
+	execErrorState string
+	noDataState    string
 }
 
 func NewAlertConfig(name string, frequency string, enabled bool) (*AlertConfig, error) {
@@ -32,6 +35,16 @@ func NewAlertConfig(name string, frequency string, enabled bool) (*AlertConfig,
 	return input, nil
 }
 
+func (c *AlertConfig) ExecutionErrorState(s string) *AlertConfig {
+	c.execErrorState = s
+	return c
+}
+
+func (c *AlertConfig) NoDataState(s string) *AlertConfig {
+	c.noDataState = s
+	return c
+}
+
 func (c *AlertConfig) Level(l string) *AlertConfig {
 	c.level = l
 	return c
@@ -44,11 +57,14 @@ func (c *AlertConfig) Enable(e bool) *AlertConfig {
 
 func (c *AlertConfig) ToAlertCreateInput() monitor.AlertCreateInput {
 	return monitor.AlertCreateInput{
-		Name:      c.name,
-		Frequency: c.frequency,
-		Settings:  c.ToAlertSetting(),
-		Enabled:   &c.enabled,
-		Level:     c.level,
+		Name:                c.name,
+		Frequency:           c.frequency,
+		Settings:            c.ToAlertSetting(),
+		Enabled:             &c.enabled,
+		Level:               c.level,
+		For:                 c.forTime,
+		ExecutionErrorState: c.execErrorState,
+		NoDataState:         c.noDataState,
 	}
 }
 

pkg/mcclient/options/monitor/alert.go

@@ -43,6 +43,11 @@ type AlertTestRunOptions struct {
 	Debug bool   `help:"Show more debug info"`
 }
 
+type AlertPauseOptions struct {
+	ID      string `help:"ID of alert to delete"`
+	UnPause bool   `help:"Unpause alert"`
+}
+
 type AlertConditionOptions struct {
 	REDUCER    string   `help:"Metric query reducer, e.g. 'avg'" choices:"avg|sum|min|max|count|last|median"`
 	DATABASE   string   `help:"Metric database, e.g. 'telegraf'"`
@@ -51,6 +56,7 @@ type AlertConditionOptions struct {
 	THRESHOLD  float64  `help:"Alert threshold"`
 	Period     string   `help:"Query metric period e.g. '5m', '1h'" default:"5m"`
 	Tag        []string `help:"Query tag, e.g. 'zone=zon0,name=vmname'"`
+	For        string   `help:"For time duration"`
 }
 
 func (opt AlertConditionOptions) Params(conf *monitor2.AlertConfig) (*monitor2.AlertCondition, error) {
@@ -96,8 +102,14 @@ func (opt AlertConditionOptions) Params(conf *monitor2.AlertConfig) (*monitor2.A
 	return cond, nil
 }
 
+type AlertStatesOptions struct {
+	NoDataState         string `help:"Set state when no data"`
+	ExecutionErrorState string `help:"Set state when execution error"`
+}
+
 type AlertCreateOptions struct {
 	AlertConditionOptions
+	AlertStatesOptions
 	NAME      string `help:"Name of the alert"`
 	Frequency string `help:"Alert execute frequency, e.g. '5m', '1h'"`
 	Enabled   bool   `help:"Enable alert"`
@@ -113,7 +125,8 @@ func (opt AlertCreateOptions) Params() (*monitor2.AlertConfig, error) {
 	if err != nil {
 		return nil, err
 	}
-
+	input.NoDataState(opt.NoDataState)
+	input.ExecutionErrorState(opt.ExecutionErrorState)
 	return input, nil
 }
 
@@ -121,6 +134,7 @@ type AlertUpdateOptions struct {
 	ID        string `help:"ID or name of the alert"`
 	Name      string `help:"Update alert name"`
 	Frequency string `help:"Alert execute frequency, e.g. '5m', '1h'"`
+	AlertStatesOptions
 }
 
 func (opt AlertUpdateOptions) Params() (*monitor.AlertUpdateInput, error) {
@@ -136,6 +150,8 @@ func (opt AlertUpdateOptions) Params() (*monitor.AlertUpdateInput, error) {
 		f := int64(freq / time.Second)
 		input.Frequency = &f
 	}
+	input.NoDataState = opt.NoDataState
+	input.ExecutionErrorState = opt.ExecutionErrorState
 	return input, nil
 }
 

pkg/monitor/models/alert.go

@@ -91,19 +91,26 @@ type SAlert struct {
 	db.SVirtualResourceBase
 	db.SEnabledResourceBase
 
+	// Frequency is evaluate period
 	Frequency int64                `nullable:"false" list:"user" create:"required" update:"user"`
 	Settings  jsonutils.JSONObject `nullable:"false" list:"user" create:"required" update:"user"`
 	Level     string               `charset:"ascii" width:"36"nullable:"false" default:"normal" list:"user" update:"user"`
-	Message   string               `charset:"utf8" list:"user" update:"user"`
+	Message   string               `charset:"utf8" list:"user" create:"optional" update:"user"`
 	UsedBy    string               `charset:"ascii" list:"user"`
 
 	// Silenced       bool
-	ExecutionError      string               `charset:"utf8" list:"user"`
-	For                 int64                `nullable:"false" list:"user"`
+	ExecutionError string `charset:"utf8" list:"user"`
+
+	// If an alert rule has a configured `For` and the query violates the configured threshold
+	// it will first go from `OK` to `Pending`. Going from `OK` to `Pending` will not send any
+	// notifications. Once the alert rule has been firing for more than `For` duration, it will
+	// change to `Alerting` and send alert notifications.
+	For int64 `nullable:"false" list:"user" update:"user"`
+
 	EvalData            jsonutils.JSONObject `list:"user" list:"user"`
-	State               string               `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user"`
-	NoDataState         string               `width:"36" charset:"ascii" nullable:"false" default:"pending" list:"user"`
-	ExecutionErrorState string               `width:"36" charset:"ascii" nullable:"false" default:"alerting" list:"user"`
+	State               string               `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"`
+	NoDataState         string               `width:"36" charset:"ascii" nullable:"false" default:"no_data" create:"optional"  list:"user" update:"user"`
+	ExecutionErrorState string               `width:"36" charset:"ascii" nullable:"false" default:"alerting" create:"optional" list:"user" update:"user"`
 	LastStateChange     time.Time            `list:"user"`
 	StateChanges        int                  `default:"0" nullable:"false" list:"user"`
 }
@@ -211,9 +218,42 @@ func (man *SAlertManager) ValidateCreateData(ctx context.Context, userCred mccli
 	if err := validators.ValidateAlertCreateInput(data); err != nil {
 		return data, err
 	}
+
+	if err := man.validateStates(data.NoDataState, data.ExecutionErrorState); err != nil {
+		return data, err
+	}
 	return data, nil
 }
 
+func (man *SAlertManager) validateStates(noData string, execErr string) error {
+	if noData != "" {
+		if err := man.validateNoDataState(monitor.NoDataOption(noData)); err != nil {
+			return err
+		}
+	}
+
+	if execErr != "" {
+		if err := man.validateExecutionErrorState(monitor.ExecutionErrorOption(execErr)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (man *SAlertManager) validateNoDataState(state monitor.NoDataOption) error {
+	if !state.IsValid() {
+		return httperrors.NewInputParameterError("unsupported no_data_state %s", state)
+	}
+	return nil
+}
+
+func (man *SAlertManager) validateExecutionErrorState(state monitor.ExecutionErrorOption) error {
+	if !state.IsValid() {
+		return httperrors.NewInputParameterError("unsupported execution_error_state %s", state)
+	}
+	return nil
+}
+
 func (man *SAlertManager) ListItemFilter(
 	ctx context.Context,
 	q *sqlchemy.SQuery,
@@ -359,6 +399,18 @@ const (
 	ErrAlertChannotChangeStateOnPaused = errors.Error("Cannot change state on pause alert")
 )
 
+func (alert *SAlert) GetExecutionErrorState() monitor.ExecutionErrorOption {
+	return monitor.ExecutionErrorOption(alert.ExecutionErrorState)
+}
+
+func (alert *SAlert) GetNoDataState() monitor.NoDataOption {
+	return monitor.NoDataOption(alert.NoDataState)
+}
+
+func (alert *SAlert) GetState() monitor.AlertStateType {
+	return monitor.AlertStateType(alert.State)
+}
+
 type AlertSetStateInput struct {
 	State          monitor.AlertStateType
 	EvalData       jsonutils.JSONObject
@@ -399,6 +451,9 @@ func (alert *SAlert) ValidateUpdateData(ctx context.Context, userCred mcclient.T
 		return input, errors.Wrap(err, "SVirtualResourceBase.ValidateUpdateData")
 	}
 
+	if err := AlertManager.validateStates(input.NoDataState, input.ExecutionErrorState); err != nil {
+		return input, err
+	}
 	return input, nil
 }
 
@@ -560,3 +615,39 @@ func (alert *SAlert) CustomizeDelete(
 	}
 	return nil
 }
+
+func (alert *SAlert) AllowPerformPause(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) bool {
+	return db.IsProjectAllowPerform(userCred, alert, "pause")
+}
+
+func (alert *SAlert) PerformPause(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input monitor.AlertPauseInput,
+) (jsonutils.JSONObject, error) {
+	curState := alert.GetState()
+	if curState != monitor.AlertStatePaused && !input.Paused {
+		return nil, httperrors.NewNotAcceptableError("Alert is already un-paused")
+	}
+
+	if curState == monitor.AlertStatePaused && input.Paused {
+		return nil, httperrors.NewNotAcceptableError("Alert is already paused")
+	}
+
+	var newState monitor.AlertStateType
+	if input.Paused {
+		newState = monitor.AlertStatePaused
+	} else {
+		newState = monitor.AlertStateUnknown
+	}
+	err := alert.SetState(AlertSetStateInput{
+		State: newState,
+	})
+	return nil, err
+}