From c39e6778e62177f43241238a317d4e9a1ae7b814 Mon Sep 17 00:00:00 2001 From: --global <1422928955@qq.com> Date: Tue, 22 Dec 2020 15:15:48 +0800 Subject: [PATCH] feat(monitor): add global alertrecord MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.支持时间、资源类型、报警级别、报警状态的过滤 2.报警记录删除时，删除关联的报警记录 --- pkg/apis/monitor/alertrecord.go | 11 +- pkg/apis/monitor/commalert.go | 3 +- pkg/monitor/alerting/notifier.go | 1 + pkg/monitor/alerting/rule.go | 1 + pkg/monitor/models/alertrecord.go | 101 ++++++++++++++++++- pkg/monitor/models/commonalert.go | 35 ++++++- pkg/monitor/service/service.go | 2 + pkg/monitor/tasks/delete_alertrecord_task.go | 49 +++++++++ 8 files changed, 196 insertions(+), 7 deletions(-) create mode 100644 pkg/monitor/tasks/delete_alertrecord_task.go diff --git a/pkg/apis/monitor/alertrecord.go b/pkg/apis/monitor/alertrecord.go index d452bb0dee..69b0973f9a 100644 --- a/pkg/apis/monitor/alertrecord.go +++ b/pkg/apis/monitor/alertrecord.go @@ -9,16 +9,18 @@ type AlertRecordListInput struct { apis.EnabledResourceBaseListInput apis.StatusStandaloneResourceListInput - AlertId string `json:"alert_id"` - Level string `json:"level"` - State string `json:"state"` + AlertId string `json:"alert_id"` + Level string `json:"level"` + State string `json:"state"` + ResType []string `json:"res_type"` } type AlertRecordDetails struct { apis.StatusStandaloneResourceDetails apis.ScopedResourceBaseInfo - ResNum int64 `json:"res_num"` + ResNum int64 `json:"res_num"` + AlertName string `json:"alert_name"` } type AlertRecordCreateInput struct { @@ -28,6 +30,7 @@ type AlertRecordCreateInput struct { // 报警级别 Level string `json:"level"` State string `json:"state"` + ResType string `json:"res_type"` EvalData []*EvalMatch `json:"eval_data"` AlertRule AlertRecordRule } diff --git a/pkg/apis/monitor/commalert.go b/pkg/apis/monitor/commalert.go index 80a97becb0..f1afa77f5c 100644 --- a/pkg/apis/monitor/commalert.go +++ b/pkg/apis/monitor/commalert.go @@ -82,7 +82,8 @@ type CommonAlertListInput struct { // 监控指标名称 Metric string `json:"metric"` - Level string `json:"level"` + Level string `json:"level"` + ResType []string `json:"res_type"` } type CommonAlertUpdateInput struct { diff --git a/pkg/monitor/alerting/notifier.go b/pkg/monitor/alerting/notifier.go index 930f02d055..ad21f4b431 100644 --- a/pkg/monitor/alerting/notifier.go +++ b/pkg/monitor/alerting/notifier.go @@ -168,6 +168,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext) EvalData: matches, AlertRule: newAlertRecordRule(evalCtx), } + recordCreateInput.ResType = recordCreateInput.AlertRule.ResType createData := recordCreateInput.JSON(recordCreateInput) record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), createData, evalCtx.UserCred) if err != nil { diff --git a/pkg/monitor/alerting/rule.go b/pkg/monitor/alerting/rule.go index 3e7ab02a80..986bf672cf 100644 --- a/pkg/monitor/alerting/rule.go +++ b/pkg/monitor/alerting/rule.go @@ -170,6 +170,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) { func newRuleDescription(rule *Rule, alertDetails *monitor.CommonAlertMetricDetails) { ruleDes := RuleDescription{ AlertRecordRule: monitor.AlertRecordRule{ + ResType: alertDetails.ResType, Metric: fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field), Measurement: alertDetails.Measurement, MeasurementDesc: alertDetails.MeasurementDisplayName, diff --git a/pkg/monitor/models/alertrecord.go b/pkg/monitor/models/alertrecord.go index 24292e3d5c..c84a3d63da 100644 --- a/pkg/monitor/models/alertrecord.go +++ b/pkg/monitor/models/alertrecord.go @@ -3,14 +3,18 @@ package models import ( "context" "database/sql" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/onecloud/pkg/apis/monitor" "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/stringutils2" @@ -37,6 +41,7 @@ type SAlertRecord struct { State string `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"` EvalData jsonutils.JSONObject `list:"user" update:"user"` AlertRule jsonutils.JSONObject `list:"user" update:"user"` + ResType string `width:"36" list:"user" update:"user"` } func init() { @@ -97,10 +102,83 @@ func (manager *SAlertRecordManager) ListItemFilter( } if len(query.AlertId) != 0 { q = q.Equals("alert_id", query.AlertId) + } else { + q = q.IsNotEmpty("res_type").IsNotNull("res_type") } return q, nil } +func (man *SAlertRecordManager) CustomizeFilterList( + ctx context.Context, q *sqlchemy.SQuery, + userCred mcclient.TokenCredential, query jsonutils.JSONObject) ( + *db.CustomizeListFilters, error) { + filters := db.NewCustomizeListFilters() + input := new(monitor.AlertRecordListInput) + if err := query.Unmarshal(input); err != nil { + return nil, err + } + wrapF := func(f func(obj *SAlertRecord) (bool, error)) func(object jsonutils.JSONObject) (bool, error) { + return func(data jsonutils.JSONObject) (bool, error) { + id, err := data.GetString("id") + if err != nil { + return false, err + } + obj, err := man.GetAlertRecord(id) + if err != nil { + return false, err + } + return f(obj) + } + } + + if len(input.ResType) != 0 { + mF := func(obj *SAlertRecord) (bool, error) { + rule := new(monitor.AlertRecordRule) + if err := obj.AlertRule.Unmarshal(rule); err != nil { + return false, errors.Wrapf(err, "alert %s unmarshal", obj.GetId()) + } + if ok, _ := utils.InStringArray(rule.ResType, input.ResType); ok { + return true, nil + } + return false, nil + } + filters.Append(wrapF(mF)) + } + return filters, nil +} + +func (man *SAlertRecordManager) GetAlertRecord(id string) (*SAlertRecord, error) { + obj, err := man.FetchById(id) + if err != nil { + return nil, err + } + return obj.(*SAlertRecord), nil +} + +func (man *SAlertRecordManager) GetAlertRecordsByAlertId(id string) ([]SAlertRecord, error) { + records := make([]SAlertRecord, 0) + query := man.Query() + query = query.Equals("alert_id", id) + err := db.FetchModelObjects(man, query, &records) + if err != nil { + return nil, err + } + return records, nil +} + +func (manager *SAlertRecordManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { + var err error + if field == "res_type" { + resTypeQuery := MetricMeasurementManager.Query("res_type").Distinct() + return resTypeQuery, nil + } + q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field) + if err == nil { + return q, nil + } + return q, httperrors.ErrNotFound +} + func (man *SAlertRecordManager) OrderByExtraFields( ctx context.Context, q *sqlchemy.SQuery, @@ -150,7 +228,8 @@ func (record *SAlertRecord) GetMoreDetails(out monitor.AlertRecordDetails) (moni } out.ResNum = int64(len(evalMatchs)) } - + commonAlert, _ := CommonAlertManager.GetAlert(record.AlertId) + out.AlertName = commonAlert.GetName() return out, nil } @@ -233,3 +312,23 @@ func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.To func (record *SAlertRecord) GetState() monitor.AlertStateType { return monitor.AlertStateType(record.State) } + +func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Context, userCred mcclient.TokenCredential, + isStart bool) { + records := make([]SAlertRecord, 0) + query := manager.Query() + query = query.LE("created_at", timeutils.MysqlTime(time.Now().Add(-time.Hour*24*30))) + log.Errorf("query:%s", query.String()) + err := db.FetchModelObjects(manager, query, &records) + if err != nil { + log.Errorf("fetch records ofthirty days ago err:%v", err) + return + } + for i, _ := range records { + err := db.DeleteModel(ctx, userCred, &records[i]) + if err != nil { + log.Errorf("delete expire record:%s err:%v", records[i].GetId(), err) + } + } + +} diff --git a/pkg/monitor/models/commonalert.go b/pkg/monitor/models/commonalert.go index 7aa879c1be..d0a562aa3c 100644 --- a/pkg/monitor/models/commonalert.go +++ b/pkg/monitor/models/commonalert.go @@ -859,9 +859,39 @@ func (alert *SCommonAlert) CustomizeDelete( alert.SetStatus(userCred, monitor.ALERT_STATUS_DELETE_FAIL, "") return errors.Wrap(err, "customizeDeleteNotis") } + alert.StartDeleteTask(ctx, userCred) return alert.SAlert.CustomizeDelete(ctx, userCred, query, data) } +func (alert *SCommonAlert) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error { + return alert.SStandaloneResourceBase.Delete(ctx, userCred) +} + +func (self *SCommonAlert) StartDeleteTask( + ctx context.Context, userCred mcclient.TokenCredential) error { + task, err := taskman.TaskManager.NewTask(ctx, "DeleteAlertRecordTask", self, userCred, jsonutils.NewDict(), "", "", nil) + if err != nil { + return err + } + task.ScheduleRun(nil) + return nil +} + +func (self *SCommonAlert) DeleteAttachAlertRecords(ctx context.Context, userCred mcclient.TokenCredential) (errs []error) { + records, err := AlertRecordManager.GetAlertRecordsByAlertId(self.GetId()) + if err != nil { + errs = append(errs, errors.Wrap(err, "GetAlertRecordsByAlertId error")) + return + } + for i, _ := range records { + err := records[i].Delete(ctx, userCred) + if err != nil { + errs = append(errs, errors.Wrapf(err, "delete attach record:%s error", records[i].GetId())) + } + } + return +} + func (alert *SCommonAlert) customizeDeleteNotis( ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error { @@ -890,7 +920,7 @@ func (alert *SCommonAlert) customizeDeleteNotis( func (alert *SCommonAlert) Delete(ctx context.Context, userCred mcclient.TokenCredential) error { CommonAlertManager.DeleteSubscriptionAlert(alert) - return alert.SAlert.Delete(ctx, userCred) + return nil } func (self *SCommonAlertManager) GetSystemAlerts() ([]SCommonAlert, error) { @@ -948,6 +978,9 @@ func (manager *SCommonAlertManager) QueryDistinctExtraField(q *sqlchemy.SQuery, case "status": q.AppendField(sqlchemy.DISTINCT(field, q.Field("status"))).Distinct() return q, nil + case "res_type": + resTypeQuery := MetricMeasurementManager.Query("res_type").Distinct() + return resTypeQuery, nil } return q, httperrors.ErrNotFound } diff --git a/pkg/monitor/service/service.go b/pkg/monitor/service/service.go index 50a79ddb37..4cd0112e1b 100644 --- a/pkg/monitor/service/service.go +++ b/pkg/monitor/service/service.go @@ -64,6 +64,8 @@ func StartService() { cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount) cron.AddJobAtIntervalsWithStartRun("InitAlertResourceAdminRoleUsers", time.Duration(opts.InitAlertResourceAdminRoleUsersIntervalSeconds)*time.Second, models.GetAlertResourceManager().GetAdminRoleUsers, true) + cron.AddJobEveryFewDays("DeleteRecordsOfThirtyDaysAgoRecords", 1, 0, 0, 0, + models.AlertRecordManager.DeleteRecordsOfThirtyDaysAgo, false) cron.Start() defer cron.Stop() diff --git a/pkg/monitor/tasks/delete_alertrecord_task.go b/pkg/monitor/tasks/delete_alertrecord_task.go new file mode 100644 index 0000000000..75a7fb5469 --- /dev/null +++ b/pkg/monitor/tasks/delete_alertrecord_task.go @@ -0,0 +1,49 @@ +package tasks + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/monitor/models" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type DeleteAlertRecordTask struct { + taskman.STask +} + +func init() { + taskman.RegisterTask(&DeleteAlertRecordTask{}) +} + +func (self *DeleteAlertRecordTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { + alert := obj.(*models.SCommonAlert) + errs := alert.DeleteAttachAlertRecords(ctx, self.GetUserCred()) + if len(errs) != 0 { + msg := jsonutils.NewString(fmt.Sprintf("fail to DeleteAttachAlertRecords:%s.err:%v", alert.Name, errors.NewAggregate(errs))) + self.taskFail(ctx, alert, msg) + return + } + + err := alert.RealDelete(ctx, self.UserCred) + if err != nil { + msg := fmt.Sprintf("delete SCommonAlert err:%v", err) + self.taskFail(ctx, alert, jsonutils.NewString(msg)) + return + } + db.OpsLog.LogEvent(alert, db.ACT_DELETE, nil, self.GetUserCred()) + logclient.AddActionLogWithStartable(self, alert, logclient.ACT_DELETE, nil, self.UserCred, true) + self.SetStageComplete(ctx, nil) +} + +func (self *DeleteAlertRecordTask) taskFail(ctx context.Context, alert *models.SCommonAlert, msg jsonutils.JSONObject) { + db.OpsLog.LogEvent(alert, db.ACT_DELETE_FAIL, msg, self.GetUserCred()) + logclient.AddActionLogWithStartable(self, alert, logclient.ACT_DELETE, msg, self.UserCred, false) + self.SetStageFailed(ctx, msg) + return +}