diff --git a/build/notify/root/opt/yunion/share/local-templates/content@cn/EXCEPTION.tmpl b/build/notify/root/opt/yunion/share/local-templates/content@cn/EXCEPTION.tmpl new file mode 100644 index 0000000000..b978b8fd77 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/content@cn/EXCEPTION.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.name }} 发生异常: {{ $d.message }} diff --git a/build/notify/root/opt/yunion/share/local-templates/content@cn/OFFLINE.tmpl b/build/notify/root/opt/yunion/share/local-templates/content@cn/OFFLINE.tmpl new file mode 100644 index 0000000000..43101a93d9 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/content@cn/OFFLINE.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.name }} 离线 {{- if $d.reason -}} 原因: {{ $d.reason }} {{- end -}} diff --git a/build/notify/root/opt/yunion/share/local-templates/content@cn/PANIC.tmpl b/build/notify/root/opt/yunion/share/local-templates/content@cn/PANIC.tmpl new file mode 100644 index 0000000000..16768beaec --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/content@cn/PANIC.tmpl @@ -0,0 +1,4 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.task_name }} PANIC {{- if $d.error -}} 错误: {{ $d.error }} {{- end -}} +堆栈信息: +{{ $d.stack }} diff --git a/build/notify/root/opt/yunion/share/local-templates/content@en/EXCEPTION.tmpl b/build/notify/root/opt/yunion/share/local-templates/content@en/EXCEPTION.tmpl new file mode 100644 index 0000000000..94ffaaffe6 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/content@en/EXCEPTION.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.name }} exception occurs: {{ $d.message }} diff --git a/build/notify/root/opt/yunion/share/local-templates/content@en/OFFLINE.tmpl b/build/notify/root/opt/yunion/share/local-templates/content@en/OFFLINE.tmpl new file mode 100644 index 0000000000..9405166407 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/content@en/OFFLINE.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +The {{ .resource_type }} {{ $d.name }} Offline {{- if $d.reason -}} Reason: {{ $d.reason }}的 {{- end -}} diff --git a/build/notify/root/opt/yunion/share/local-templates/content@en/PANIC.tmpl b/build/notify/root/opt/yunion/share/local-templates/content@en/PANIC.tmpl new file mode 100644 index 0000000000..35518ccca1 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/content@en/PANIC.tmpl @@ -0,0 +1,4 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.task_name }} PANIC {{- if $d.error -}} Error: {{ $d.error }} {{- end -}} +Stack Info: +{{ $d.stack }} diff --git a/build/notify/root/opt/yunion/share/local-templates/title@cn/EXCEPTION.tmpl b/build/notify/root/opt/yunion/share/local-templates/title@cn/EXCEPTION.tmpl new file mode 100644 index 0000000000..b978b8fd77 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/title@cn/EXCEPTION.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.name }} 发生异常: {{ $d.message }} diff --git a/build/notify/root/opt/yunion/share/local-templates/title@cn/OFFLINE.tmpl b/build/notify/root/opt/yunion/share/local-templates/title@cn/OFFLINE.tmpl new file mode 100644 index 0000000000..c326cf546b --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/title@cn/OFFLINE.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.name }} 离线 diff --git a/build/notify/root/opt/yunion/share/local-templates/title@cn/PANIC.tmpl b/build/notify/root/opt/yunion/share/local-templates/title@cn/PANIC.tmpl new file mode 100644 index 0000000000..a319fb1541 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/title@cn/PANIC.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.task_name }} PANIC diff --git a/build/notify/root/opt/yunion/share/local-templates/title@en/EXCEPTION.tmpl b/build/notify/root/opt/yunion/share/local-templates/title@en/EXCEPTION.tmpl new file mode 100644 index 0000000000..94ffaaffe6 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/title@en/EXCEPTION.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.name }} exception occurs: {{ $d.message }} diff --git a/build/notify/root/opt/yunion/share/local-templates/title@en/OFFLINE.tmpl b/build/notify/root/opt/yunion/share/local-templates/title@en/OFFLINE.tmpl new file mode 100644 index 0000000000..929155c96c --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/title@en/OFFLINE.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +The {{ .resource_type }} {{ $d.name }} Offline diff --git a/build/notify/root/opt/yunion/share/local-templates/title@en/PANIC.tmpl b/build/notify/root/opt/yunion/share/local-templates/title@en/PANIC.tmpl new file mode 100644 index 0000000000..a319fb1541 --- /dev/null +++ b/build/notify/root/opt/yunion/share/local-templates/title@en/PANIC.tmpl @@ -0,0 +1,2 @@ +{{- $d := .resource_details -}} +{{ .resource_type }} {{ $d.task_name }} PANIC diff --git a/pkg/apis/notify/const.go b/pkg/apis/notify/const.go index 76870151b3..37a8d3be1c 100644 --- a/pkg/apis/notify/const.go +++ b/pkg/apis/notify/const.go @@ -119,6 +119,8 @@ const ( TOPIC_RESOURCE_DNSRECORDSET = "dns_recordset" TOPIC_RESOURCE_LOADBALANCERLISTENER = "loadbalancerlistener" TOPIC_RESOURCE_LOADBALANCERBACKEDNGROUP = "loadbalancerbackendgroup" + TOPIC_RESOURCE_HOST = "host" + TOPIC_RESOURCE_TASK = "task" SUBSCRIBER_TYPE_ROLE = "role" SUBSCRIBER_TYPE_ROBOT = "robot" diff --git a/pkg/apis/notify/event.go b/pkg/apis/notify/event.go index dcac06cf74..cfe4748897 100644 --- a/pkg/apis/notify/event.go +++ b/pkg/apis/notify/event.go @@ -44,6 +44,10 @@ var ( ActionSyncUpdate SAction = "sync_update" ActionSyncDelete SAction = "sync_delete" + ActionOffline SAction = "offline" + ActionSystemPanic SAction = "panic" + ActionSystemException SAction = "exception" + ResultFailed SResult = "failed" ResultSucceed SResult = "succeed" ) diff --git a/pkg/cloudcommon/db/taskman/worker.go b/pkg/cloudcommon/db/taskman/worker.go index 0b5feb6877..6fa7ab5017 100644 --- a/pkg/cloudcommon/db/taskman/worker.go +++ b/pkg/cloudcommon/db/taskman/worker.go @@ -17,11 +17,13 @@ package taskman import ( "context" "fmt" + "runtime/debug" "yunion.io/x/jsonutils" + api "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/util/panicutils" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" ) var taskWorkMan *appsrv.SWorkerManager @@ -61,7 +63,12 @@ func runTask(taskId string, data jsonutils.JSONObject) error { } isOk := worker.Run(task, nil, func(err error) { - panicutils.SendPanicMessage(context.TODO(), err) + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(taskName), "task_name") + data.Add(jsonutils.NewString(taskId), "task_id") + data.Add(jsonutils.NewString(string(debug.Stack())), "stack") + data.Add(jsonutils.NewString(err.Error()), "error") + notifyclient.SystemExceptionNotify(context.TODO(), api.ActionSystemPanic, api.TOPIC_RESOURCE_TASK, data) }) if !isOk { return fmt.Errorf("worker %s(%s) not running may be droped", taskName, taskId) diff --git a/pkg/cloudcommon/notifyclient/notify.go b/pkg/cloudcommon/notifyclient/notify.go index 30c5b7d201..f2da74c738 100644 --- a/pkg/cloudcommon/notifyclient/notify.go +++ b/pkg/cloudcommon/notifyclient/notify.go @@ -299,6 +299,20 @@ func EventNotify(ctx context.Context, userCred mcclient.TokenCredential, ep SEve notifyClientWorkerMan.Run(&t, nil, nil) } +func SystemExceptionNotify(ctx context.Context, action api.SAction, resType string, obj jsonutils.JSONObject) { + event := api.Event.WithAction(action).WithResourceType(resType) + params := api.NotificationManagerEventNotifyInput{ + ReceiverIds: []string{}, + ResourceDetails: obj.(*jsonutils.JSONDict), + Event: event.String(), + Priority: string(npk.NotifyPriorityCritical), + } + t := eventTask{ + params: params, + } + notifyClientWorkerMan.Run(&t, nil, nil) +} + func RawNotifyWithCtx(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) { rawNotify(ctx, sNotifyParams{ recipientId: recipientId, diff --git a/pkg/compute/models/cloudproviderregions.go b/pkg/compute/models/cloudproviderregions.go index 46bd5ae8bd..b9bf7478e8 100644 --- a/pkg/compute/models/cloudproviderregions.go +++ b/pkg/compute/models/cloudproviderregions.go @@ -33,7 +33,6 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/util/nopanic" "yunion.io/x/onecloud/pkg/util/stringutils2" ) @@ -450,13 +449,11 @@ func (self *SCloudproviderregion) getSyncTaskKey() string { func (self *SCloudproviderregion) submitSyncTask(ctx context.Context, userCred mcclient.TokenCredential, syncRange SSyncRange) { self.markStartSync(userCred) RunSyncCloudproviderRegionTask(ctx, self.getSyncTaskKey(), func() { - nopanic.Run(func() { - ctx = context.WithValue(ctx, "provider-region", fmt.Sprintf("%d", self.RowId)) - err := self.DoSync(ctx, userCred, syncRange) - if err != nil { - log.Errorf("DoSync faild %v", err) - } - }) + ctx = context.WithValue(ctx, "provider-region", fmt.Sprintf("%d", self.RowId)) + err := self.DoSync(ctx, userCred, syncRange) + if err != nil { + log.Errorf("DoSync faild %v", err) + } }) } diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 4ba6719e6c..584efc45f9 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -40,11 +40,13 @@ import ( "yunion.io/x/onecloud/pkg/apis" billing_api "yunion.io/x/onecloud/pkg/apis/billing" api "yunion.io/x/onecloud/pkg/apis/compute" + napi "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/appsrv" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/cloudcommon/db/quotas" "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/compute/baremetal" @@ -3770,6 +3772,11 @@ func (self *SHost) PerformOffline(ctx context.Context, userCred mcclient.TokenCr } db.OpsLog.LogEvent(self, db.ACT_OFFLINE, input.Reason, userCred) logclient.AddActionLogWithContext(ctx, self, logclient.ACT_OFFLINE, input, userCred, true) + ndata := jsonutils.Marshal(self).(*jsonutils.JSONDict) + if len(input.Reason) > 0 { + ndata.Add(jsonutils.NewString(input.Reason), "reason") + } + notifyclient.SystemExceptionNotify(ctx, napi.ActionOffline, HostManager.Keyword(), ndata) self.SyncAttachedStorageStatus() } return nil, nil diff --git a/pkg/compute/models/syncworkers.go b/pkg/compute/models/syncworkers.go index 68d06aa60e..695acee9bd 100644 --- a/pkg/compute/models/syncworkers.go +++ b/pkg/compute/models/syncworkers.go @@ -17,14 +17,17 @@ package models import ( "context" "fmt" + "runtime/debug" "strconv" "github.com/serialx/hashring" + "yunion.io/x/jsonutils" "yunion.io/x/log" + api "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/util/panicutils" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" ) var ( @@ -76,7 +79,12 @@ func RunSyncCloudproviderRegionTask(ctx context.Context, key string, syncFunc fu } log.Debugf("run sync task at %d len %d", nodeIdx, len(syncWorkers)) syncWorkers[nodeIdx].Run(&task, nil, func(err error) { - panicutils.SendPanicMessage(ctx, err) + data := jsonutils.NewDict() + data.Add(jsonutils.NewString("SyncCloudproviderRegion"), "task_name") + data.Add(jsonutils.NewString(key), "task_id") + data.Add(jsonutils.NewString(string(debug.Stack())), "stack") + data.Add(jsonutils.NewString(err.Error()), "error") + notifyclient.SystemExceptionNotify(context.TODO(), api.ActionSystemPanic, api.TOPIC_RESOURCE_TASK, data) }) } @@ -86,6 +94,11 @@ func RunSyncCloudAccountTask(ctx context.Context, probeFunc func()) { key: "AccountProb", } syncAccountWorker.Run(&task, nil, func(err error) { - panicutils.SendPanicMessage(ctx, err) + data := jsonutils.NewDict() + data.Add(jsonutils.NewString("SyncCloudAccountTask"), "task_name") + data.Add(jsonutils.NewString(task.key), "task_id") + data.Add(jsonutils.NewString(string(debug.Stack())), "stack") + data.Add(jsonutils.NewString(err.Error()), "error") + notifyclient.SystemExceptionNotify(context.TODO(), api.ActionSystemPanic, api.TOPIC_RESOURCE_TASK, data) }) } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index f9a21fa822..1bb599a6e6 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -38,6 +38,8 @@ import ( "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/compute" identityapi "yunion.io/x/onecloud/pkg/apis/identity" + napi "yunion.io/x/onecloud/pkg/apis/notify" + "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver" "yunion.io/x/onecloud/pkg/hostman/host_health" deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis" @@ -863,6 +865,10 @@ func (h *SHostInfo) register() { func (h *SHostInfo) onFail(reason interface{}) { if len(h.HostId) > 0 && !h.isLoged { logclient.AddSimpleActionLog(h, logclient.ACT_ONLINE, reason, hostutils.GetComputeSession(context.Background()).GetToken(), false) + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(h.GetName()), "name") + data.Add(jsonutils.NewString(fmt.Sprintf("register failed: %v", reason)), "message") + notifyclient.SystemExceptionNotify(context.TODO(), napi.ActionSystemException, napi.TOPIC_RESOURCE_HOST, data) h.isLoged = true } log.Errorf("register failed: %s", reason) @@ -1013,6 +1019,10 @@ func (h *SHostInfo) getHostInfo(zoneId string) { if hosts[0].HostStatus == api.HOST_ONLINE { reason := fmt.Sprintf("The host status is online when it staring. Maybe the control center was down earlier") logclient.AddSimpleActionLog(h, logclient.ACT_HEALTH_CHECK, map[string]string{"reason": reason}, hostutils.GetComputeSession(context.Background()).GetToken(), false) + data := jsonutils.NewDict() + data.Add(jsonutils.NewString(h.GetName()), "name") + data.Add(jsonutils.NewString(reason), "message") + notifyclient.SystemExceptionNotify(context.TODO(), napi.ActionSystemException, napi.TOPIC_RESOURCE_HOST, data) } h.updateHostRecord(hosts[0].Id) } diff --git a/pkg/notify/models/topic.go b/pkg/notify/models/topic.go index c46acc6a87..fd0cdbe3ef 100644 --- a/pkg/notify/models/topic.go +++ b/pkg/notify/models/topic.go @@ -91,6 +91,7 @@ const ( DefaultSnapshotPolicyExecute = "snapshot policy execute" DefaultResourceOperationFailed = "resource operation failed" DefaultResourceSync = "resource sync" + DefaultSystemExceptionEvent = "system exception event" ) func (sm *STopicManager) InitializeData() error { @@ -106,6 +107,7 @@ func (sm *STopicManager) InitializeData() error { DefaultSnapshotPolicyExecute, DefaultResourceOperationFailed, DefaultResourceSync, + DefaultSystemExceptionEvent, ) q := sm.Query() topics := make([]STopic, 0, initSNames.Len()) @@ -281,6 +283,17 @@ func (sm *STopicManager) InitializeData() error { ) t.Type = notify.TOPIC_TYPE_RESOURCE t.WebconsoleDisable = tristate.True + case DefaultSystemExceptionEvent: + t.addResources( + notify.TOPIC_RESOURCE_HOST, + notify.TOPIC_RESOURCE_TASK, + ) + t.addAction( + notify.ActionSystemPanic, + notify.ActionSystemException, + notify.ActionOffline, + ) + t.Type = notify.TOPIC_TYPE_RESOURCE } if topic == nil { err := sm.TableSpec().Insert(ctx, t) @@ -490,6 +503,8 @@ func init() { notify.TOPIC_RESOURCE_DNSRECORDSET: 29, notify.TOPIC_RESOURCE_LOADBALANCERLISTENER: 30, notify.TOPIC_RESOURCE_LOADBALANCERBACKEDNGROUP: 31, + notify.TOPIC_RESOURCE_HOST: 32, + notify.TOPIC_RESOURCE_TASK: 33, }, ) converter.registerAction( @@ -512,6 +527,9 @@ func init() { notify.ActionSyncCreate: 15, notify.ActionSyncUpdate: 16, notify.ActionSyncDelete: 17, + notify.ActionOffline: 18, + notify.ActionSystemPanic: 19, + notify.ActionSystemException: 20, }, ) } diff --git a/pkg/util/panicutils/doc.go b/pkg/util/panicutils/doc.go deleted file mode 100644 index c507e5eb8c..0000000000 --- a/pkg/util/panicutils/doc.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package panicutils // import "yunion.io/x/onecloud/pkg/util/panicutils" diff --git a/pkg/util/panicutils/panicutil.go b/pkg/util/panicutils/panicutil.go deleted file mode 100644 index e02f59160e..0000000000 --- a/pkg/util/panicutils/panicutil.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package panicutils - -import ( - "context" - "runtime/debug" - - "yunion.io/x/jsonutils" - - "yunion.io/x/onecloud/pkg/cloudcommon/notifyclient" - npk "yunion.io/x/onecloud/pkg/mcclient/modules/notify" -) - -const PANIC_ERROR = "PANIC_ERROR" - -func SendPanicMessage(ctx context.Context, err error) { - iNotes := jsonutils.Marshal(map[string]string{"stack": string(debug.Stack()), "error": err.Error()}) - notifyclient.SystemNotifyWithCtx(ctx, npk.NotifyPriorityCritical, PANIC_ERROR, iNotes) -}