opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-01 15:07:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #9577 from zhaoxiangchun/feature/zxc-alerthistory-master

Browse Source 
feat(monitor): add global alertrecord
This commit is contained in:
Zexi Li
2020-12-23 09:45:53 +08:00
committed by GitHub
parent b27dc38605 c39e6778e6
commit 3be6296f1c
8 changed files with 196 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/apis/monitor/alertrecord.go
+7 -4
View File
@@ -9,16 +9,18 @@ type AlertRecordListInput struct {
apis.EnabledResourceBaseListInput
apis.StatusStandaloneResourceListInput
AlertId string `json:"alert_id"`
Level string `json:"level"`
State string `json:"state"`
AlertId string `json:"alert_id"`
Level string `json:"level"`
State string `json:"state"`
ResType []string `json:"res_type"`
}
type AlertRecordDetails struct {
apis.StatusStandaloneResourceDetails
apis.ScopedResourceBaseInfo
ResNum int64 `json:"res_num"`
ResNum int64 `json:"res_num"`
AlertName string `json:"alert_name"`
}
type AlertRecordCreateInput struct {
@@ -28,6 +30,7 @@ type AlertRecordCreateInput struct {
// 报警级别
Level string `json:"level"`
State string `json:"state"`
ResType string `json:"res_type"`
EvalData []*EvalMatch `json:"eval_data"`
AlertRule AlertRecordRule
}
pkg/apis/monitor/commalert.go
+2 -1
View File
@@ -82,7 +82,8 @@ type CommonAlertListInput struct {
// 监控指标名称
Metric string `json:"metric"`
Level string `json:"level"`
Level string `json:"level"`
ResType []string `json:"res_type"`
}
type CommonAlertUpdateInput struct {
pkg/monitor/alerting/notifier.go
+1
View File
@@ -168,6 +168,7 @@ func (n *notificationService) createAlertRecordWhenNotify(evalCtx *EvalContext)
EvalData: matches,
AlertRule: newAlertRecordRule(evalCtx),
}
recordCreateInput.ResType = recordCreateInput.AlertRule.ResType
createData := recordCreateInput.JSON(recordCreateInput)
record, err := db.DoCreate(models.AlertRecordManager, evalCtx.Ctx, evalCtx.UserCred, jsonutils.NewDict(), createData, evalCtx.UserCred)
if err != nil {
pkg/monitor/alerting/rule.go
+1
View File
@@ -170,6 +170,7 @@ func NewRuleFromDBAlert(ruleDef *models.SAlert) (*Rule, error) {
func newRuleDescription(rule *Rule, alertDetails *monitor.CommonAlertMetricDetails) {
ruleDes := RuleDescription{
AlertRecordRule: monitor.AlertRecordRule{
ResType: alertDetails.ResType,
Metric: fmt.Sprintf("%s.%s", alertDetails.Measurement, alertDetails.Field),
Measurement: alertDetails.Measurement,
MeasurementDesc: alertDetails.MeasurementDisplayName,
pkg/monitor/models/alertrecord.go
+100 -1
View File
@@ -3,14 +3,18 @@ package models
import (
"context"
"database/sql"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/timeutils"
"yunion.io/x/pkg/utils"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/apis/monitor"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/rbacutils"
"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -37,6 +41,7 @@ type SAlertRecord struct {
State string `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"`
EvalData jsonutils.JSONObject `list:"user" update:"user"`
AlertRule jsonutils.JSONObject `list:"user" update:"user"`
ResType string `width:"36" list:"user" update:"user"`
}
func init() {
@@ -97,10 +102,83 @@ func (manager *SAlertRecordManager) ListItemFilter(
}
if len(query.AlertId) != 0 {
q = q.Equals("alert_id", query.AlertId)
} else {
q = q.IsNotEmpty("res_type").IsNotNull("res_type")
}
return q, nil
}
func (man *SAlertRecordManager) CustomizeFilterList(
ctx context.Context, q *sqlchemy.SQuery,
userCred mcclient.TokenCredential, query jsonutils.JSONObject) (
*db.CustomizeListFilters, error) {
filters := db.NewCustomizeListFilters()
input := new(monitor.AlertRecordListInput)
if err := query.Unmarshal(input); err != nil {
return nil, err
}
wrapF := func(f func(obj *SAlertRecord) (bool, error)) func(object jsonutils.JSONObject) (bool, error) {
return func(data jsonutils.JSONObject) (bool, error) {
id, err := data.GetString("id")
if err != nil {
return false, err
}
obj, err := man.GetAlertRecord(id)
if err != nil {
return false, err
}
return f(obj)
}
}
if len(input.ResType) != 0 {
mF := func(obj *SAlertRecord) (bool, error) {
rule := new(monitor.AlertRecordRule)
if err := obj.AlertRule.Unmarshal(rule); err != nil {
return false, errors.Wrapf(err, "alert %s unmarshal", obj.GetId())
}
if ok, _ := utils.InStringArray(rule.ResType, input.ResType); ok {
return true, nil
}
return false, nil
}
filters.Append(wrapF(mF))
}
return filters, nil
}
func (man *SAlertRecordManager) GetAlertRecord(id string) (*SAlertRecord, error) {
obj, err := man.FetchById(id)
if err != nil {
return nil, err
}
return obj.(*SAlertRecord), nil
}
func (man *SAlertRecordManager) GetAlertRecordsByAlertId(id string) ([]SAlertRecord, error) {
records := make([]SAlertRecord, 0)
query := man.Query()
query = query.Equals("alert_id", id)
err := db.FetchModelObjects(man, query, &records)
if err != nil {
return nil, err
}
return records, nil
}
func (manager *SAlertRecordManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
var err error
if field == "res_type" {
resTypeQuery := MetricMeasurementManager.Query("res_type").Distinct()
return resTypeQuery, nil
}
q, err = manager.SStandaloneResourceBaseManager.QueryDistinctExtraField(q, field)
if err == nil {
return q, nil
}
return q, httperrors.ErrNotFound
}
func (man *SAlertRecordManager) OrderByExtraFields(
ctx context.Context,
q *sqlchemy.SQuery,
@@ -150,7 +228,8 @@ func (record *SAlertRecord) GetMoreDetails(out monitor.AlertRecordDetails) (moni
}
out.ResNum = int64(len(evalMatchs))
}
commonAlert, _ := CommonAlertManager.GetAlert(record.AlertId)
out.AlertName = commonAlert.GetName()
return out, nil
}
@@ -233,3 +312,23 @@ func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.To
func (record *SAlertRecord) GetState() monitor.AlertStateType {
return monitor.AlertStateType(record.State)
}
func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Context, userCred mcclient.TokenCredential,
isStart bool) {
records := make([]SAlertRecord, 0)
query := manager.Query()
query = query.LE("created_at", timeutils.MysqlTime(time.Now().Add(-time.Hour*24*30)))
log.Errorf("query:%s", query.String())
err := db.FetchModelObjects(manager, query, &records)
if err != nil {
log.Errorf("fetch records ofthirty days ago err:%v", err)
return
}
for i, _ := range records {
err := db.DeleteModel(ctx, userCred, &records[i])
if err != nil {
log.Errorf("delete expire record:%s err:%v", records[i].GetId(), err)
}
}
}
pkg/monitor/models/commonalert.go
+34 -1
View File
@@ -859,9 +859,39 @@ func (alert *SCommonAlert) CustomizeDelete(
alert.SetStatus(userCred, monitor.ALERT_STATUS_DELETE_FAIL, "")
return errors.Wrap(err, "customizeDeleteNotis")
}
alert.StartDeleteTask(ctx, userCred)
return alert.SAlert.CustomizeDelete(ctx, userCred, query, data)
}
func (alert *SCommonAlert) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
return alert.SStandaloneResourceBase.Delete(ctx, userCred)
}
func (self *SCommonAlert) StartDeleteTask(
ctx context.Context, userCred mcclient.TokenCredential) error {
task, err := taskman.TaskManager.NewTask(ctx, "DeleteAlertRecordTask", self, userCred, jsonutils.NewDict(), "", "", nil)
if err != nil {
return err
}
task.ScheduleRun(nil)
return nil
}
func (self *SCommonAlert) DeleteAttachAlertRecords(ctx context.Context, userCred mcclient.TokenCredential) (errs []error) {
records, err := AlertRecordManager.GetAlertRecordsByAlertId(self.GetId())
if err != nil {
errs = append(errs, errors.Wrap(err, "GetAlertRecordsByAlertId error"))
return
}
for i, _ := range records {
err := records[i].Delete(ctx, userCred)
if err != nil {
errs = append(errs, errors.Wrapf(err, "delete attach record:%s error", records[i].GetId()))
}
}
return
}
func (alert *SCommonAlert) customizeDeleteNotis(
ctx context.Context, userCred mcclient.TokenCredential,
query jsonutils.JSONObject, data jsonutils.JSONObject) error {
@@ -890,7 +920,7 @@ func (alert *SCommonAlert) customizeDeleteNotis(
func (alert *SCommonAlert) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
CommonAlertManager.DeleteSubscriptionAlert(alert)
return alert.SAlert.Delete(ctx, userCred)
return nil
}
func (self *SCommonAlertManager) GetSystemAlerts() ([]SCommonAlert, error) {
@@ -948,6 +978,9 @@ func (manager *SCommonAlertManager) QueryDistinctExtraField(q *sqlchemy.SQuery,
case "status":
q.AppendField(sqlchemy.DISTINCT(field, q.Field("status"))).Distinct()
return q, nil
case "res_type":
resTypeQuery := MetricMeasurementManager.Query("res_type").Distinct()
return resTypeQuery, nil
}
return q, httperrors.ErrNotFound
}
pkg/monitor/service/service.go
+2
View File
@@ -64,6 +64,8 @@ func StartService() {
cron := cronman.InitCronJobManager(true, opts.CronJobWorkerCount)
cron.AddJobAtIntervalsWithStartRun("InitAlertResourceAdminRoleUsers", time.Duration(opts.InitAlertResourceAdminRoleUsersIntervalSeconds)*time.Second, models.GetAlertResourceManager().GetAdminRoleUsers, true)
cron.AddJobEveryFewDays("DeleteRecordsOfThirtyDaysAgoRecords", 1, 0, 0, 0,
models.AlertRecordManager.DeleteRecordsOfThirtyDaysAgo, false)
cron.Start()
defer cron.Stop()
pkg/monitor/tasks/delete_alertrecord_task.go
+49
View File
@@ -0,0 +1,49 @@
package tasks
import (
"context"
"fmt"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/monitor/models"
"yunion.io/x/onecloud/pkg/util/logclient"
)
type DeleteAlertRecordTask struct {
taskman.STask
}
func init() {
taskman.RegisterTask(&DeleteAlertRecordTask{})
}
func (self *DeleteAlertRecordTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
alert := obj.(*models.SCommonAlert)
errs := alert.DeleteAttachAlertRecords(ctx, self.GetUserCred())
if len(errs) != 0 {
msg := jsonutils.NewString(fmt.Sprintf("fail to DeleteAttachAlertRecords:%s.err:%v", alert.Name, errors.NewAggregate(errs)))
self.taskFail(ctx, alert, msg)
return
}
err := alert.RealDelete(ctx, self.UserCred)
if err != nil {
msg := fmt.Sprintf("delete SCommonAlert err:%v", err)
self.taskFail(ctx, alert, jsonutils.NewString(msg))
return
}
db.OpsLog.LogEvent(alert, db.ACT_DELETE, nil, self.GetUserCred())
logclient.AddActionLogWithStartable(self, alert, logclient.ACT_DELETE, nil, self.UserCred, true)
self.SetStageComplete(ctx, nil)
}
func (self *DeleteAlertRecordTask) taskFail(ctx context.Context, alert *models.SCommonAlert, msg jsonutils.JSONObject) {
db.OpsLog.LogEvent(alert, db.ACT_DELETE_FAIL, msg, self.GetUserCred())
logclient.AddActionLogWithStartable(self, alert, logclient.ACT_DELETE, msg, self.UserCred, false)
self.SetStageFailed(ctx, msg)
return
}