opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13566 from ioito/hotfix/qx-notify-panic

Browse Source 
fix(notify): add panic notify
This commit is contained in:
Zexi Li
2022-03-01 19:18:42 +08:00
committed by GitHub
parent f6bbd05a30 6f32a0281a
commit f0d14fce4f
23 changed files with 113 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/notify/root/opt/yunion/share/local-templates/content@cn/EXCEPTION.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.name }} 发生异常: {{ $d.message }}
build/notify/root/opt/yunion/share/local-templates/content@cn/OFFLINE.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.name }} 离线 {{- if $d.reason -}} 原因: {{ $d.reason }} {{- end -}}
build/notify/root/opt/yunion/share/local-templates/content@cn/PANIC.tmpl
+4
View File
@@ -0,0 +1,4 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.task_name }} PANIC {{- if $d.error -}} 错误: {{ $d.error }} {{- end -}}
堆栈信息:
{{ $d.stack }}
build/notify/root/opt/yunion/share/local-templates/content@en/EXCEPTION.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.name }} exception occurs: {{ $d.message }}
build/notify/root/opt/yunion/share/local-templates/content@en/OFFLINE.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
The {{ .resource_type }} {{ $d.name }} Offline {{- if $d.reason -}} Reason: {{ $d.reason }}的 {{- end -}}
build/notify/root/opt/yunion/share/local-templates/content@en/PANIC.tmpl
+4
View File
@@ -0,0 +1,4 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.task_name }} PANIC {{- if $d.error -}} Error: {{ $d.error }} {{- end -}}
Stack Info:
{{ $d.stack }}
build/notify/root/opt/yunion/share/local-templates/title@cn/EXCEPTION.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.name }} 发生异常: {{ $d.message }}
build/notify/root/opt/yunion/share/local-templates/title@cn/OFFLINE.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.name }} 离线
build/notify/root/opt/yunion/share/local-templates/title@cn/PANIC.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.task_name }} PANIC
build/notify/root/opt/yunion/share/local-templates/title@en/EXCEPTION.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.name }} exception occurs: {{ $d.message }}
build/notify/root/opt/yunion/share/local-templates/title@en/OFFLINE.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
The {{ .resource_type }} {{ $d.name }} Offline
build/notify/root/opt/yunion/share/local-templates/title@en/PANIC.tmpl
+2
View File
@@ -0,0 +1,2 @@
{{- $d := .resource_details -}}
{{ .resource_type }} {{ $d.task_name }} PANIC
pkg/apis/notify/const.go
+2
View File
@@ -119,6 +119,8 @@ const (
TOPIC_RESOURCE_DNSRECORDSET = "dns_recordset"
TOPIC_RESOURCE_LOADBALANCERLISTENER = "loadbalancerlistener"
TOPIC_RESOURCE_LOADBALANCERBACKEDNGROUP = "loadbalancerbackendgroup"
TOPIC_RESOURCE_HOST = "host"
TOPIC_RESOURCE_TASK = "task"
SUBSCRIBER_TYPE_ROLE = "role"
SUBSCRIBER_TYPE_ROBOT = "robot"
pkg/apis/notify/event.go
+4
View File
@@ -44,6 +44,10 @@ var (
ActionSyncUpdate SAction = "sync_update"
ActionSyncDelete SAction = "sync_delete"
ActionOffline SAction = "offline"
ActionSystemPanic SAction = "panic"
ActionSystemException SAction = "exception"
ResultFailed SResult = "failed"
ResultSucceed SResult = "succeed"
)
pkg/cloudcommon/db/taskman/worker.go
+9 -2
View File
@@ -17,11 +17,13 @@ package taskman
import (
"context"
"fmt"
"runtime/debug"
"yunion.io/x/jsonutils"
api "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/util/panicutils"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
)
var taskWorkMan *appsrv.SWorkerManager
@@ -61,7 +63,12 @@ func runTask(taskId string, data jsonutils.JSONObject) error {
}
isOk := worker.Run(task, nil, func(err error) {
panicutils.SendPanicMessage(context.TODO(), err)
data := jsonutils.NewDict()
data.Add(jsonutils.NewString(taskName), "task_name")
data.Add(jsonutils.NewString(taskId), "task_id")
data.Add(jsonutils.NewString(string(debug.Stack())), "stack")
data.Add(jsonutils.NewString(err.Error()), "error")
notifyclient.SystemExceptionNotify(context.TODO(), api.ActionSystemPanic, api.TOPIC_RESOURCE_TASK, data)
})
if !isOk {
return fmt.Errorf("worker %s(%s) not running may be droped", taskName, taskId)
pkg/cloudcommon/notifyclient/notify.go
+14
View File
@@ -299,6 +299,20 @@ func EventNotify(ctx context.Context, userCred mcclient.TokenCredential, ep SEve
notifyClientWorkerMan.Run(&t, nil, nil)
}
func SystemExceptionNotify(ctx context.Context, action api.SAction, resType string, obj jsonutils.JSONObject) {
event := api.Event.WithAction(action).WithResourceType(resType)
params := api.NotificationManagerEventNotifyInput{
ReceiverIds: []string{},
ResourceDetails: obj.(*jsonutils.JSONDict),
Event: event.String(),
Priority: string(npk.NotifyPriorityCritical),
}
t := eventTask{
params: params,
}
notifyClientWorkerMan.Run(&t, nil, nil)
}
func RawNotifyWithCtx(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
rawNotify(ctx, sNotifyParams{
recipientId: recipientId,
pkg/compute/models/cloudproviderregions.go
+5 -8
View File
@@ -33,7 +33,6 @@ import (
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/util/nopanic"
"yunion.io/x/onecloud/pkg/util/stringutils2"
)
@@ -450,13 +449,11 @@ func (self *SCloudproviderregion) getSyncTaskKey() string {
func (self *SCloudproviderregion) submitSyncTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange) {
self.markStartSync(userCred)
RunSyncCloudproviderRegionTask(ctx, self.getSyncTaskKey(), func() {
nopanic.Run(func() {
ctx = context.WithValue(ctx, "provider-region", fmt.Sprintf("%d", self.RowId))
err := self.DoSync(ctx, userCred, syncRange)
if err != nil {
log.Errorf("DoSync faild %v", err)
}
})
ctx = context.WithValue(ctx, "provider-region", fmt.Sprintf("%d", self.RowId))
err := self.DoSync(ctx, userCred, syncRange)
if err != nil {
log.Errorf("DoSync faild %v", err)
}
})
}
pkg/compute/models/hosts.go
+7
View File
@@ -40,11 +40,13 @@ import (
"yunion.io/x/onecloud/pkg/apis"
billing_api "yunion.io/x/onecloud/pkg/apis/billing"
api "yunion.io/x/onecloud/pkg/apis/compute"
napi "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
"yunion.io/x/onecloud/pkg/cloudcommon/types"
"yunion.io/x/onecloud/pkg/cloudprovider"
"yunion.io/x/onecloud/pkg/compute/baremetal"
@@ -3770,6 +3772,11 @@ func (self *SHost) PerformOffline(ctx context.Context, userCred mcclient.TokenCr
}
db.OpsLog.LogEvent(self, db.ACT_OFFLINE, input.Reason, userCred)
logclient.AddActionLogWithContext(ctx, self, logclient.ACT_OFFLINE, input, userCred, true)
ndata := jsonutils.Marshal(self).(*jsonutils.JSONDict)
if len(input.Reason) > 0 {
ndata.Add(jsonutils.NewString(input.Reason), "reason")
}
notifyclient.SystemExceptionNotify(ctx, napi.ActionOffline, HostManager.Keyword(), ndata)
self.SyncAttachedStorageStatus()
}
return nil, nil
pkg/compute/models/syncworkers.go
+16 -3
View File
@@ -17,14 +17,17 @@ package models
import (
"context"
"fmt"
"runtime/debug"
"strconv"
"github.com/serialx/hashring"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
api "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/util/panicutils"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
)
var (
@@ -76,7 +79,12 @@ func RunSyncCloudproviderRegionTask(ctx context.Context, key string, syncFunc fu
}
log.Debugf("run sync task at %d len %d", nodeIdx, len(syncWorkers))
syncWorkers[nodeIdx].Run(&task, nil, func(err error) {
panicutils.SendPanicMessage(ctx, err)
data := jsonutils.NewDict()
data.Add(jsonutils.NewString("SyncCloudproviderRegion"), "task_name")
data.Add(jsonutils.NewString(key), "task_id")
data.Add(jsonutils.NewString(string(debug.Stack())), "stack")
data.Add(jsonutils.NewString(err.Error()), "error")
notifyclient.SystemExceptionNotify(context.TODO(), api.ActionSystemPanic, api.TOPIC_RESOURCE_TASK, data)
})
}
@@ -86,6 +94,11 @@ func RunSyncCloudAccountTask(ctx context.Context, probeFunc func()) {
key: "AccountProb",
}
syncAccountWorker.Run(&task, nil, func(err error) {
panicutils.SendPanicMessage(ctx, err)
data := jsonutils.NewDict()
data.Add(jsonutils.NewString("SyncCloudAccountTask"), "task_name")
data.Add(jsonutils.NewString(task.key), "task_id")
data.Add(jsonutils.NewString(string(debug.Stack())), "stack")
data.Add(jsonutils.NewString(err.Error()), "error")
notifyclient.SystemExceptionNotify(context.TODO(), api.ActionSystemPanic, api.TOPIC_RESOURCE_TASK, data)
})
}
pkg/hostman/hostinfo/hostinfo.go
+10
View File
@@ -38,6 +38,8 @@ import (
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/compute"
identityapi "yunion.io/x/onecloud/pkg/apis/identity"
napi "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
"yunion.io/x/onecloud/pkg/hostman/host_health"
deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
@@ -863,6 +865,10 @@ func (h *SHostInfo) register() {
func (h *SHostInfo) onFail(reason interface{}) {
if len(h.HostId) > 0 && !h.isLoged {
logclient.AddSimpleActionLog(h, logclient.ACT_ONLINE, reason, hostutils.GetComputeSession(context.Background()).GetToken(), false)
data := jsonutils.NewDict()
data.Add(jsonutils.NewString(h.GetName()), "name")
data.Add(jsonutils.NewString(fmt.Sprintf("register failed: %v", reason)), "message")
notifyclient.SystemExceptionNotify(context.TODO(), napi.ActionSystemException, napi.TOPIC_RESOURCE_HOST, data)
h.isLoged = true
}
log.Errorf("register failed: %s", reason)
@@ -1013,6 +1019,10 @@ func (h *SHostInfo) getHostInfo(zoneId string) {
if hosts[0].HostStatus == api.HOST_ONLINE {
reason := fmt.Sprintf("The host status is online when it staring. Maybe the control center was down earlier")
logclient.AddSimpleActionLog(h, logclient.ACT_HEALTH_CHECK, map[string]string{"reason": reason}, hostutils.GetComputeSession(context.Background()).GetToken(), false)
data := jsonutils.NewDict()
data.Add(jsonutils.NewString(h.GetName()), "name")
data.Add(jsonutils.NewString(reason), "message")
notifyclient.SystemExceptionNotify(context.TODO(), napi.ActionSystemException, napi.TOPIC_RESOURCE_HOST, data)
}
h.updateHostRecord(hosts[0].Id)
}
pkg/notify/models/topic.go
+18
View File
@@ -91,6 +91,7 @@ const (
DefaultSnapshotPolicyExecute = "snapshot policy execute"
DefaultResourceOperationFailed = "resource operation failed"
DefaultResourceSync = "resource sync"
DefaultSystemExceptionEvent = "system exception event"
)
func (sm *STopicManager) InitializeData() error {
@@ -106,6 +107,7 @@ func (sm *STopicManager) InitializeData() error {
DefaultSnapshotPolicyExecute,
DefaultResourceOperationFailed,
DefaultResourceSync,
DefaultSystemExceptionEvent,
)
q := sm.Query()
topics := make([]STopic, 0, initSNames.Len())
@@ -281,6 +283,17 @@ func (sm *STopicManager) InitializeData() error {
)
t.Type = notify.TOPIC_TYPE_RESOURCE
t.WebconsoleDisable = tristate.True
case DefaultSystemExceptionEvent:
t.addResources(
notify.TOPIC_RESOURCE_HOST,
notify.TOPIC_RESOURCE_TASK,
)
t.addAction(
notify.ActionSystemPanic,
notify.ActionSystemException,
notify.ActionOffline,
)
t.Type = notify.TOPIC_TYPE_RESOURCE
}
if topic == nil {
err := sm.TableSpec().Insert(ctx, t)
@@ -490,6 +503,8 @@ func init() {
notify.TOPIC_RESOURCE_DNSRECORDSET: 29,
notify.TOPIC_RESOURCE_LOADBALANCERLISTENER: 30,
notify.TOPIC_RESOURCE_LOADBALANCERBACKEDNGROUP: 31,
notify.TOPIC_RESOURCE_HOST: 32,
notify.TOPIC_RESOURCE_TASK: 33,
},
)
converter.registerAction(
@@ -512,6 +527,9 @@ func init() {
notify.ActionSyncCreate: 15,
notify.ActionSyncUpdate: 16,
notify.ActionSyncDelete: 17,
notify.ActionOffline: 18,
notify.ActionSystemPanic: 19,
notify.ActionSystemException: 20,
},
)
}
pkg/util/panicutils/doc.go
-15
View File
@@ -1,15 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package panicutils // import "yunion.io/x/onecloud/pkg/util/panicutils"
pkg/util/panicutils/panicutil.go
-32
View File
@@ -1,32 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package panicutils
import (
"context"
"runtime/debug"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
)
const PANIC_ERROR = "PANIC_ERROR"
func SendPanicMessage(ctx context.Context, err error) {
iNotes := jsonutils.Marshal(map[string]string{"stack": string(debug.Stack()), "error": err.Error()})
notifyclient.SystemNotifyWithCtx(ctx, npk.NotifyPriorityCritical, PANIC_ERROR, iNotes)
}