From 395f2af41d68ec1cd48e39ce647aa2e8f57d676c Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Wed, 1 Apr 2020 13:18:37 +0800 Subject: [PATCH] monitor: add paused api --- cmd/climc/shell/monitor/alert.go | 16 ++++ pkg/apis/monitor/alert.go | 22 +++--- pkg/mcclient/modules/monitor/alert.go | 2 +- pkg/mcclient/modules/monitor/helper.go | 36 ++++++--- pkg/mcclient/options/monitor/alert.go | 18 ++++- pkg/monitor/models/alert.go | 103 +++++++++++++++++++++++-- 6 files changed, 169 insertions(+), 28 deletions(-) diff --git a/cmd/climc/shell/monitor/alert.go b/cmd/climc/shell/monitor/alert.go index cde0861205..a9b30a3e3d 100644 --- a/cmd/climc/shell/monitor/alert.go +++ b/cmd/climc/shell/monitor/alert.go @@ -97,4 +97,20 @@ func init() { fmt.Println(ret.JSON(ret).YAMLString()) return nil }) + + R(&options.AlertPauseOptions{}, aN("pause"), "Pause or unpause alert", + func(s *mcclient.ClientSession, args *options.AlertPauseOptions) error { + data := new(monitorapi.AlertPauseInput) + if args.UnPause { + data.Paused = false + } else { + data.Paused = true + } + ret, err := monitor.Alerts.PerformAction(s, args.ID, "pause", data.JSON(data)) + if err != nil { + return err + } + printObject(ret) + return nil + }) } diff --git a/pkg/apis/monitor/alert.go b/pkg/apis/monitor/alert.go index e7a6e9a7ae..840bc4b209 100644 --- a/pkg/apis/monitor/alert.go +++ b/pkg/apis/monitor/alert.go @@ -105,12 +105,18 @@ type AlertCreateInput struct { Name string `json:"name"` // 报警执行频率 Frequency int64 `json:"frequency"` + // 报警持续时间 + For int64 `json:"for"` // 报警设置 Settings AlertSetting `json:"settings"` // 启用报警 Enabled *bool `json:"enabled"` // 报警级别 Level string `json:"level"` + // 没有收到监控指标时将当前报警状态设置为对应的状态 + NoDataState string `json:"no_data_state"` + // 报警执行错误将当前报警状态设置为对应的状态 + ExecutionErrorState string `json:"execution_error_state"` } type AlertUpdateInput struct { @@ -126,6 +132,10 @@ type AlertUpdateInput struct { Enabled *bool `json:"enabled"` // 报警级别 Level *string `json:"level"` + // 没有收到监控指标时将当前报警状态设置为对应的状态 + NoDataState string `json:"no_data_state"` + // 报警执行错误将当前报警状态设置为对应的状态 + ExecutionErrorState string `json:"execution_error_state"` } type AlertListInput struct { @@ -172,16 +182,8 @@ type AlertTestRunOutput struct { EndTime time.Time `json:"end_time"` } -type AlertAttachNotificationInput struct { +type AlertPauseInput struct { apis.Meta - NotificationId string `json:"notification_id"` - UsedBy string `json:"used_by"` -} - -type AlertAttachNotificationOutput struct { - apis.Meta - - NotificationId string `json:"notification_id"` - UsedBy string `json:"used_by"` + Paused bool `json:"paused"` } diff --git a/pkg/mcclient/modules/monitor/alert.go b/pkg/mcclient/modules/monitor/alert.go index 9880369d26..5329225536 100644 --- a/pkg/mcclient/modules/monitor/alert.go +++ b/pkg/mcclient/modules/monitor/alert.go @@ -35,7 +35,7 @@ type SAlertManager struct { func NewAlertManager() *SAlertManager { man := modules.NewMonitorV2Manager("alert", "alerts", - []string{"id", "name", "state", "frequency", "enabled", "settings"}, + []string{"id", "name", "frequency", "enabled", "settings", "state"}, []string{}) return &SAlertManager{ ResourceManager: &man, diff --git a/pkg/mcclient/modules/monitor/helper.go b/pkg/mcclient/modules/monitor/helper.go index edc26fc578..ec96ffb8bb 100644 --- a/pkg/mcclient/modules/monitor/helper.go +++ b/pkg/mcclient/modules/monitor/helper.go @@ -10,11 +10,14 @@ import ( // AlertConfig is a helper to generate monitor service alert related api input type AlertConfig struct { - name string - frequency int64 - level string - enabled bool - conditions []*AlertCondition + name string + frequency int64 + forTime int64 + level string + enabled bool + conditions []*AlertCondition + execErrorState string + noDataState string } func NewAlertConfig(name string, frequency string, enabled bool) (*AlertConfig, error) { @@ -32,6 +35,16 @@ func NewAlertConfig(name string, frequency string, enabled bool) (*AlertConfig, return input, nil } +func (c *AlertConfig) ExecutionErrorState(s string) *AlertConfig { + c.execErrorState = s + return c +} + +func (c *AlertConfig) NoDataState(s string) *AlertConfig { + c.noDataState = s + return c +} + func (c *AlertConfig) Level(l string) *AlertConfig { c.level = l return c @@ -44,11 +57,14 @@ func (c *AlertConfig) Enable(e bool) *AlertConfig { func (c *AlertConfig) ToAlertCreateInput() monitor.AlertCreateInput { return monitor.AlertCreateInput{ - Name: c.name, - Frequency: c.frequency, - Settings: c.ToAlertSetting(), - Enabled: &c.enabled, - Level: c.level, + Name: c.name, + Frequency: c.frequency, + Settings: c.ToAlertSetting(), + Enabled: &c.enabled, + Level: c.level, + For: c.forTime, + ExecutionErrorState: c.execErrorState, + NoDataState: c.noDataState, } } diff --git a/pkg/mcclient/options/monitor/alert.go b/pkg/mcclient/options/monitor/alert.go index 9fc366e30c..64ee2cdd7c 100644 --- a/pkg/mcclient/options/monitor/alert.go +++ b/pkg/mcclient/options/monitor/alert.go @@ -43,6 +43,11 @@ type AlertTestRunOptions struct { Debug bool `help:"Show more debug info"` } +type AlertPauseOptions struct { + ID string `help:"ID of alert to delete"` + UnPause bool `help:"Unpause alert"` +} + type AlertConditionOptions struct { REDUCER string `help:"Metric query reducer, e.g. 'avg'" choices:"avg|sum|min|max|count|last|median"` DATABASE string `help:"Metric database, e.g. 'telegraf'"` @@ -51,6 +56,7 @@ type AlertConditionOptions struct { THRESHOLD float64 `help:"Alert threshold"` Period string `help:"Query metric period e.g. '5m', '1h'" default:"5m"` Tag []string `help:"Query tag, e.g. 'zone=zon0,name=vmname'"` + For string `help:"For time duration"` } func (opt AlertConditionOptions) Params(conf *monitor2.AlertConfig) (*monitor2.AlertCondition, error) { @@ -96,8 +102,14 @@ func (opt AlertConditionOptions) Params(conf *monitor2.AlertConfig) (*monitor2.A return cond, nil } +type AlertStatesOptions struct { + NoDataState string `help:"Set state when no data"` + ExecutionErrorState string `help:"Set state when execution error"` +} + type AlertCreateOptions struct { AlertConditionOptions + AlertStatesOptions NAME string `help:"Name of the alert"` Frequency string `help:"Alert execute frequency, e.g. '5m', '1h'"` Enabled bool `help:"Enable alert"` @@ -113,7 +125,8 @@ func (opt AlertCreateOptions) Params() (*monitor2.AlertConfig, error) { if err != nil { return nil, err } - + input.NoDataState(opt.NoDataState) + input.ExecutionErrorState(opt.ExecutionErrorState) return input, nil } @@ -121,6 +134,7 @@ type AlertUpdateOptions struct { ID string `help:"ID or name of the alert"` Name string `help:"Update alert name"` Frequency string `help:"Alert execute frequency, e.g. '5m', '1h'"` + AlertStatesOptions } func (opt AlertUpdateOptions) Params() (*monitor.AlertUpdateInput, error) { @@ -136,6 +150,8 @@ func (opt AlertUpdateOptions) Params() (*monitor.AlertUpdateInput, error) { f := int64(freq / time.Second) input.Frequency = &f } + input.NoDataState = opt.NoDataState + input.ExecutionErrorState = opt.ExecutionErrorState return input, nil } diff --git a/pkg/monitor/models/alert.go b/pkg/monitor/models/alert.go index 2e446ba28b..781ffd4bbc 100644 --- a/pkg/monitor/models/alert.go +++ b/pkg/monitor/models/alert.go @@ -91,19 +91,26 @@ type SAlert struct { db.SVirtualResourceBase db.SEnabledResourceBase + // Frequency is evaluate period Frequency int64 `nullable:"false" list:"user" create:"required" update:"user"` Settings jsonutils.JSONObject `nullable:"false" list:"user" create:"required" update:"user"` Level string `charset:"ascii" width:"36"nullable:"false" default:"normal" list:"user" update:"user"` - Message string `charset:"utf8" list:"user" update:"user"` + Message string `charset:"utf8" list:"user" create:"optional" update:"user"` UsedBy string `charset:"ascii" list:"user"` // Silenced bool - ExecutionError string `charset:"utf8" list:"user"` - For int64 `nullable:"false" list:"user"` + ExecutionError string `charset:"utf8" list:"user"` + + // If an alert rule has a configured `For` and the query violates the configured threshold + // it will first go from `OK` to `Pending`. Going from `OK` to `Pending` will not send any + // notifications. Once the alert rule has been firing for more than `For` duration, it will + // change to `Alerting` and send alert notifications. + For int64 `nullable:"false" list:"user" update:"user"` + EvalData jsonutils.JSONObject `list:"user" list:"user"` - State string `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user"` - NoDataState string `width:"36" charset:"ascii" nullable:"false" default:"pending" list:"user"` - ExecutionErrorState string `width:"36" charset:"ascii" nullable:"false" default:"alerting" list:"user"` + State string `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"` + NoDataState string `width:"36" charset:"ascii" nullable:"false" default:"no_data" create:"optional" list:"user" update:"user"` + ExecutionErrorState string `width:"36" charset:"ascii" nullable:"false" default:"alerting" create:"optional" list:"user" update:"user"` LastStateChange time.Time `list:"user"` StateChanges int `default:"0" nullable:"false" list:"user"` } @@ -211,9 +218,42 @@ func (man *SAlertManager) ValidateCreateData(ctx context.Context, userCred mccli if err := validators.ValidateAlertCreateInput(data); err != nil { return data, err } + + if err := man.validateStates(data.NoDataState, data.ExecutionErrorState); err != nil { + return data, err + } return data, nil } +func (man *SAlertManager) validateStates(noData string, execErr string) error { + if noData != "" { + if err := man.validateNoDataState(monitor.NoDataOption(noData)); err != nil { + return err + } + } + + if execErr != "" { + if err := man.validateExecutionErrorState(monitor.ExecutionErrorOption(execErr)); err != nil { + return err + } + } + return nil +} + +func (man *SAlertManager) validateNoDataState(state monitor.NoDataOption) error { + if !state.IsValid() { + return httperrors.NewInputParameterError("unsupported no_data_state %s", state) + } + return nil +} + +func (man *SAlertManager) validateExecutionErrorState(state monitor.ExecutionErrorOption) error { + if !state.IsValid() { + return httperrors.NewInputParameterError("unsupported execution_error_state %s", state) + } + return nil +} + func (man *SAlertManager) ListItemFilter( ctx context.Context, q *sqlchemy.SQuery, @@ -359,6 +399,18 @@ const ( ErrAlertChannotChangeStateOnPaused = errors.Error("Cannot change state on pause alert") ) +func (alert *SAlert) GetExecutionErrorState() monitor.ExecutionErrorOption { + return monitor.ExecutionErrorOption(alert.ExecutionErrorState) +} + +func (alert *SAlert) GetNoDataState() monitor.NoDataOption { + return monitor.NoDataOption(alert.NoDataState) +} + +func (alert *SAlert) GetState() monitor.AlertStateType { + return monitor.AlertStateType(alert.State) +} + type AlertSetStateInput struct { State monitor.AlertStateType EvalData jsonutils.JSONObject @@ -399,6 +451,9 @@ func (alert *SAlert) ValidateUpdateData(ctx context.Context, userCred mcclient.T return input, errors.Wrap(err, "SVirtualResourceBase.ValidateUpdateData") } + if err := AlertManager.validateStates(input.NoDataState, input.ExecutionErrorState); err != nil { + return input, err + } return input, nil } @@ -560,3 +615,39 @@ func (alert *SAlert) CustomizeDelete( } return nil } + +func (alert *SAlert) AllowPerformPause( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) bool { + return db.IsProjectAllowPerform(userCred, alert, "pause") +} + +func (alert *SAlert) PerformPause( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + input monitor.AlertPauseInput, +) (jsonutils.JSONObject, error) { + curState := alert.GetState() + if curState != monitor.AlertStatePaused && !input.Paused { + return nil, httperrors.NewNotAcceptableError("Alert is already un-paused") + } + + if curState == monitor.AlertStatePaused && input.Paused { + return nil, httperrors.NewNotAcceptableError("Alert is already paused") + } + + var newState monitor.AlertStateType + if input.Paused { + newState = monitor.AlertStatePaused + } else { + newState = monitor.AlertStateUnknown + } + err := alert.SetState(AlertSetStateInput{ + State: newState, + }) + return nil, err +}