Merge pull request #13865 from rainzm/notify/checksum

Security notify

build/notify/root/opt/yunion/share/local-templates/content@cn/CHECKSUM_TEST_FAILED.tmpl

@@ -0,0 +1,14 @@
+{{- $d := .resource_details -}}
+{{- if eq .resource_type "db_table_field" }}
+数据库{{ $d.db_name }}表{{ $d.table_name }}记录{{ $d.name }}被修改，
+{{- end -}}
+{{- if eq .resource_type "cloudpods_component" }}
+组件{{ $d.name }}
+{{- end -}}
+{{- if eq .resource_type "snapshot" }}
+快照{{ $d.name }}的内存快照
+{{- end -}}
+{{- if eq .resource_type "image" }}
+镜像{{ $d.name }}
+{{- end -}}
+完整性校验失败

build/notify/root/opt/yunion/share/local-templates/content@cn/USER_LOCK.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+账号{{ $d.name }}由于异常登录已被锁定，请核实情况，如果需要为用户解锁，请到用户列表启用该用户。

build/notify/root/opt/yunion/share/local-templates/content@en/CHECKSUM_TEST_FAILED.tmpl

@@ -0,0 +1,14 @@
+{{- $d := .resource_details -}}
+{{- if eq .resource_type "db_table_field" }}
+The record {{ $d.name }} in table {{ $d.table_name }} of the database {{ $d.db_name }} has been modified because the checksum test failed.
+{{- end -}}
+{{- if eq .resource_type "cloudpods_component" }}
+The checksum of cloudpods component {{ $d.name }} test failed.
+{{- end -}}
+{{- if eq .resource_type "snapshot" }}
+The checksum of the memory snapshot of the snapshot {{ $d.name }} test failed.
+{{- end -}}
+{{- if eq .resource_type "image" }}
+The checksum of the image {{ $d.name }} test failed.
+{{- end -}}
+

build/notify/root/opt/yunion/share/local-templates/content@en/USER_LOCK.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+The account {{ $d.name }} has been locked due to abnormal login. Please verify the situation. If you need to unlock the user, please go to the user list to enable the user.

build/notify/root/opt/yunion/share/local-templates/title@cn/CHECKSUM_TEST_FAILED.tmpl

@@ -0,0 +1 @@
+{{ .resource_type_display }}完整性校验失败

build/notify/root/opt/yunion/share/local-templates/title@cn/USER_LOCK.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+账号{{ $d.name }}已被锁定

build/notify/root/opt/yunion/share/local-templates/title@en/CHECKSUM_TEST_FAILED.tmpl

@@ -0,0 +1 @@
+The checksum of {{ .resource_type_display }} test failed

build/notify/root/opt/yunion/share/local-templates/title@en/USER_LOCK.tmpl

@@ -0,0 +1,2 @@
+{{- $d := .resource_details -}}
+Account {{ $d.name }} has been locked

cmd/climc/shell/notifyv2/notification.go

@@ -15,6 +15,8 @@
 package notifyv2
 
 import (
+	"fmt"
+
 	"yunion.io/x/jsonutils"
 
 	api "yunion.io/x/onecloud/pkg/apis/notify"
@@ -100,4 +102,30 @@ func init() {
 		printList(ret, modules.Notification.GetColumns(s))
 		return nil
 	})
+	type NotificationEventInput struct {
+		Event    string
+		Priority string
+		MsgBody  string
+	}
+	R(&NotificationEventInput{}, "notify-event-send", "Send notify event message", func(s *mcclient.ClientSession, args *NotificationEventInput) error {
+		body, err := jsonutils.ParseString(args.MsgBody)
+		if err != nil {
+			return err
+		}
+		dict, ok := body.(*jsonutils.JSONDict)
+		if !ok {
+			return fmt.Errorf("msg_body should be a json string, like '{'name': 'hello'}'")
+		}
+		params := api.NotificationManagerEventNotifyInput{
+			ReceiverIds:     []string{},
+			ResourceDetails: dict,
+			Event:           args.Event,
+			Priority:        args.Priority,
+		}
+		_, err = modules.Notification.PerformClassAction(s, "event-notify", jsonutils.Marshal(params))
+		if err != nil {
+			return fmt.Errorf("unable to EventNotify: %s", err)
+		}
+		return nil
+	})
 }

pkg/apis/notify/const.go

@@ -86,6 +86,7 @@ const (
 
 	TOPIC_TYPE_RESOURCE          = "resource"
 	TOPIC_TYPE_AUTOMATED_PROCESS = "automated_process"
+	TOPIC_TYPE_SECURITY          = "security"
 
 	TOPIC_RESOURCE_SERVER                   = "server"
 	TOPIC_RESOURCE_SCALINGGROUP             = "scalinggroup"
@@ -121,6 +122,9 @@ const (
 	TOPIC_RESOURCE_LOADBALANCERBACKEDNGROUP = "loadbalancerbackendgroup"
 	TOPIC_RESOURCE_HOST                     = "host"
 	TOPIC_RESOURCE_TASK                     = "task"
+	TOPIC_RESOURCE_DB_TABLE_RECORD          = "db_table_record"
+	TOPIC_RESOURCE_CLOUDPODS_COMPONENT      = "cloudpods_component"
+	TOPIC_RESOURCE_USER                     = "user"
 
 	SUBSCRIBER_TYPE_ROLE     = "role"
 	SUBSCRIBER_TYPE_ROBOT    = "robot"

pkg/apis/notify/event.go

@@ -48,6 +48,10 @@ var (
 	ActionSystemPanic     SAction = "panic"
 	ActionSystemException SAction = "exception"
 
+	ActionChecksumTest SAction = "checksum_test"
+
+	ActionLock SAction = "lock"
+
 	ResultFailed  SResult = "failed"
 	ResultSucceed SResult = "succeed"
 )

pkg/cloudcommon/database.go

@@ -17,8 +17,6 @@ package cloudcommon
 import (
 	"context"
 	"database/sql"
-	"fmt"
-	"net/http"
 	"time"
 
 	"github.com/mattn/go-sqlite3"
@@ -28,11 +26,13 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
-	"yunion.io/x/onecloud/pkg/appsrv"
+	noapi "yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
 	"yunion.io/x/onecloud/pkg/cloudcommon/informer"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 )
 
@@ -90,7 +90,7 @@ func InitDB(options *common_options.DBOptions) {
 		if err != nil {
 			panic(err)
 		}
-		sqlchemy.SetDBWithNameBackend(click, ClickhouseDB, sqlchemy.ClickhouseBackend)
+		sqlchemy.SetDBWithNameBackend(click, db.ClickhouseDB, sqlchemy.ClickhouseBackend)
 
 		if options.OpsLogWithClickhouse {
 			consts.OpsLogWithClickhouse = true
@@ -123,9 +123,16 @@ func InitDB(options *common_options.DBOptions) {
 	}
 	// lm := lockman.NewNoopLockManager()
 
+	initDBNotifier()
 	startInitInformer(options)
 }
 
+func initDBNotifier() {
+	db.SetChecksumTestFailedNotifier(func(obj *jsonutils.JSONDict) {
+		notifyclient.SystemExceptionNotifyWithResult(context.TODO(), noapi.ActionChecksumTest, noapi.TOPIC_RESOURCE_DB_TABLE_RECORD, noapi.ResultFailed, obj)
+	})
+}
+
 // startInitInformer starts goroutine init informer backend
 func startInitInformer(options *common_options.DBOptions) {
 	go func() {
@@ -167,28 +174,3 @@ func initInformer(options *common_options.DBOptions) error {
 func CloseDB() {
 	sqlchemy.CloseDB()
 }
-
-func AppDBInit(app *appsrv.Application) {
-	dbConn := sqlchemy.GetDB()
-	if dbConn != nil {
-		connMax := appsrv.GetDBConnectionCount()
-		if connMax < MIN_DB_CONN_MAX {
-			connMax = MIN_DB_CONN_MAX
-		}
-		log.Infof("Total %d db workers, set db connection max", connMax)
-		dbConn.SetMaxIdleConns(connMax)
-		dbConn.SetMaxOpenConns(connMax*2 + 1)
-	}
-
-	app.AddDefaultHandler("GET", "/db_stats", DBStatsHandler, "db_stats")
-}
-
-func DBStatsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
-	result := jsonutils.NewDict()
-	dbConn := sqlchemy.GetDB()
-	if dbConn != nil {
-		stats := dbConn.Stats()
-		result.Add(jsonutils.Marshal(&stats), "db_stats")
-	}
-	fmt.Fprintf(w, result.String())
-}

pkg/cloudcommon/db/checksum.go

@@ -20,12 +20,19 @@ import (
 	"reflect"
 	"sort"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/reflectutils"
 	"yunion.io/x/pkg/utils"
 )
 
+var checksumTestFailedNotifier func(obj *jsonutils.JSONDict)
+
+func SetChecksumTestFailedNotifier(notifier func(obj *jsonutils.JSONDict)) {
+	checksumTestFailedNotifier = notifier
+}
+
 type IRecordChecksumResourceBase interface {
 	GetRecordChecksum() string
 	SetRecordChecksum(checksum string)
@@ -93,6 +100,15 @@ func CheckRecordChecksumConsistent(model IModel) error {
 	savedChecksum := obj.GetRecordChecksum()
 	if calChecksum != savedChecksum {
 		log.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
+		ts := model.GetModelManager().TableSpec()
+		// notify
+		data := jsonutils.NewDict()
+		data.Set("db_name", jsonutils.NewString(string(ts.GetDBName())))
+		data.Set("table_name", jsonutils.NewString(ts.Name()))
+		data.Set("name", jsonutils.NewString(fmt.Sprintf("%s(%s)", obj.Keyword(), obj.GetId())))
+		if checksumTestFailedNotifier != nil {
+			checksumTestFailedNotifier(data)
+		}
 		return errors.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
 	}
 	return nil

pkg/cloudcommon/db/database.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package db
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/sqlchemy"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+)
+
+const (
+	MIN_DB_CONN_MAX = 5
+
+	ClickhouseDB = sqlchemy.DBName("clickhosue_db")
+)
+
+func AppDBInit(app *appsrv.Application) {
+	dbConn := sqlchemy.GetDB()
+	if dbConn != nil {
+		connMax := appsrv.GetDBConnectionCount()
+		if connMax < MIN_DB_CONN_MAX {
+			connMax = MIN_DB_CONN_MAX
+		}
+		log.Infof("Total %d db workers, set db connection max", connMax)
+		dbConn.SetMaxIdleConns(connMax)
+		dbConn.SetMaxOpenConns(connMax*2 + 1)
+	}
+
+	app.AddDefaultHandler("GET", "/db_stats", DBStatsHandler, "db_stats")
+}
+
+func DBStatsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	result := jsonutils.NewDict()
+	dbConn := sqlchemy.GetDB()
+	if dbConn != nil {
+		stats := dbConn.Stats()
+		result.Add(jsonutils.Marshal(&stats), "db_stats")
+	}
+	fmt.Fprintf(w, result.String())
+}

pkg/cloudcommon/db/models.go

@@ -24,7 +24,6 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/appsrv"
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 )
 
@@ -178,7 +177,7 @@ func EnsureAppSyncDB(app *appsrv.Application, opt *common_options.DBOptions, mod
 		os.Exit(0)
 	}
 
-	cloudcommon.AppDBInit(app)
+	AppDBInit(app)
 }
 
 func GetModelManager(keyword string) IModelManager {

pkg/cloudcommon/db/opslog.go

@@ -34,7 +34,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/appsrv"
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -89,7 +88,7 @@ func InitOpsLog() {
 			"opslog_tbl",
 			"event",
 			"events",
-			cloudcommon.ClickhouseDB,
+			ClickhouseDB,
 		)}
 		col := OpsLog.TableSpec().ColumnSpec("ops_time")
 		if clickCol, ok := col.(clickhouse.IClickhouseColumnSpec); ok {

pkg/cloudcommon/db/tablespec.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/sqlchemy"
 
+	api "yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/cloudcommon/informer"
 	"yunion.io/x/onecloud/pkg/util/nopanic"
 	"yunion.io/x/onecloud/pkg/util/splitable"
@@ -51,6 +52,8 @@ type ITableSpec interface {
 	GetSplitTable() *splitable.SSplitTableSpec
 
 	GetTableSpec() *sqlchemy.STableSpec
+
+	GetDBName() sqlchemy.DBName
 }
 
 type sTableSpec struct {
@@ -171,7 +174,28 @@ func (ts *sTableSpec) InsertOrUpdate(ctx context.Context, dt interface{}) error
 }
 
 func (ts *sTableSpec) CheckRecordChanged(dbObj IModel) error {
-	return CheckRecordChecksumConsistent(dbObj)
+	return ts.CheckRecordChecksumConsistent(dbObj)
+}
+
+func (ts *sTableSpec) CheckRecordChecksumConsistent(model IModel) error {
+	obj, ok := IsModelEnableRecordChecksum(model)
+	if !ok {
+		return nil
+	}
+	calChecksum, err := CalculateModelChecksum(obj)
+	if err != nil {
+		return errors.Wrap(err, "CalculateModelChecksum")
+	}
+	savedChecksum := obj.GetRecordChecksum()
+	if calChecksum != savedChecksum {
+		log.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
+		return errors.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
+	}
+	return nil
+}
+
+func checksumTestNotify(ctx context.Context, action api.SAction, resType string, obj jsonutils.JSONObject) {
+
 }
 
 func (ts *sTableSpec) Update(ctx context.Context, dt interface{}, doUpdate func() error) (sqlchemy.UpdateDiffs, error) {

pkg/cloudcommon/db/taskman/interface.go

@@ -17,16 +17,16 @@ package taskman
 import (
 	"context"
 	"net/http"
+	"time"
 
 	"yunion.io/x/jsonutils"
 
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
 type ITask interface {
-	cloudcommon.IStartable
+	GetStartTime() time.Time
 
 	ScheduleRun(data jsonutils.JSONObject) error
 	GetParams() *jsonutils.JSONDict

pkg/cloudcommon/interface.go

@@ -1,21 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cloudcommon
-
-import "time"
-
-type IStartable interface {
-	GetStartTime() time.Time
-}

pkg/cloudcommon/notifyclient/notify.go

@@ -299,13 +299,13 @@ func EventNotify(ctx context.Context, userCred mcclient.TokenCredential, ep SEve
 	notifyClientWorkerMan.Run(&t, nil, nil)
 }
 
-func SystemExceptionNotify(ctx context.Context, action api.SAction, resType string, obj jsonutils.JSONObject) {
-	event := api.Event.WithAction(action).WithResourceType(resType)
+func systemEventNotify(ctx context.Context, action api.SAction, resType string, result api.SResult, priority string, obj *jsonutils.JSONDict) {
+	event := api.Event.WithAction(action).WithResourceType(resType).WithResult(result)
 	params := api.NotificationManagerEventNotifyInput{
 		ReceiverIds:     []string{},
-		ResourceDetails: obj.(*jsonutils.JSONDict),
+		ResourceDetails: obj,
 		Event:           event.String(),
-		Priority:        string(npk.NotifyPriorityCritical),
+		Priority:        priority,
 	}
 	t := eventTask{
 		params: params,
@@ -313,6 +313,18 @@ func SystemExceptionNotify(ctx context.Context, action api.SAction, resType stri
 	notifyClientWorkerMan.Run(&t, nil, nil)
 }
 
+func SystemEventNotify(ctx context.Context, action api.SAction, resType string, obj *jsonutils.JSONDict) {
+	systemEventNotify(ctx, action, resType, api.ResultSucceed, string(npk.NotifyPriorityNormal), obj)
+}
+
+func SystemExceptionNotify(ctx context.Context, action api.SAction, resType string, obj *jsonutils.JSONDict) {
+	systemEventNotify(ctx, action, resType, api.ResultSucceed, string(npk.NotifyPriorityCritical), obj)
+}
+
+func SystemExceptionNotifyWithResult(ctx context.Context, action api.SAction, resType string, result api.SResult, obj *jsonutils.JSONDict) {
+	systemEventNotify(ctx, action, resType, result, string(npk.NotifyPriorityCritical), obj)
+}
+
 func RawNotifyWithCtx(ctx context.Context, recipientId []string, isGroup bool, channel npk.TNotifyChannel, priority npk.TNotifyPriority, event string, data jsonutils.JSONObject) {
 	rawNotify(ctx, sNotifyParams{
 		recipientId: recipientId,

pkg/cloudevent/models/cloudevents.go

@@ -26,7 +26,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/cloudevent"
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -50,7 +49,7 @@ func InitCloudevent() {
 				"cloudevents_tbl",
 				"cloudevent",
 				"cloudevents",
-				cloudcommon.ClickhouseDB,
+				db.ClickhouseDB,
 			),
 		}
 		col := CloudeventManager.TableSpec().ColumnSpec("created_at")

pkg/cloudir/service/service.go

@@ -19,8 +19,8 @@ import (
 
 	"yunion.io/x/log"
 
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloudir/options"
@@ -43,7 +43,7 @@ func StartService() {
 	}
 
 	app := app_common.InitApp(baseOpts, false)
-	cloudcommon.AppDBInit(app)
+	db.AppDBInit(app)
 	initHandlers(app)
 
 	app_common.ServeForeverWithCleanup(app, baseOpts, func() {

pkg/cloutpost/service/service.go

@@ -19,8 +19,8 @@ import (
 
 	"yunion.io/x/log"
 
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
 	"yunion.io/x/onecloud/pkg/cloudcommon/etcd/models"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
@@ -48,7 +48,7 @@ func StartService() {
 	defer etcd.CloseDefaultEtcdClient()
 
 	app := app_common.InitApp(baseOpts, false)
-	cloudcommon.AppDBInit(app)
+	db.AppDBInit(app)
 	initHandlers(app)
 
 	err = models.ServiceRegistryManager.Register(

pkg/hostman/guestman/qemu-kvm.go

@@ -35,8 +35,10 @@ import (
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	hostapi "yunion.io/x/onecloud/pkg/apis/host"
+	noapi "yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/deployclient"
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
@@ -1920,6 +1922,9 @@ func (s *SKVMGuestInstance) ExecMemorySnapshotResetTask(ctx context.Context, inp
 			return nil, handleErr(fmt.Sprintf("calculate statefile %s checksum: %v", memStatPath, err))
 		}
 		if checksum != input.Checksum {
+			data := jsonutils.NewDict()
+			data.Set("name", jsonutils.NewString(input.InstanceSnapshotId))
+			notifyclient.SystemExceptionNotifyWithResult(context.Background(), noapi.ActionChecksumTest, noapi.TOPIC_RESOURCE_SNAPSHOT, noapi.ResultFailed, data)
 			return nil, handleErr(fmt.Sprintf("calculate checksum %s != %s", checksum, input.Checksum))
 		}
 	}

pkg/image/models/image_subs.go

@@ -22,11 +22,14 @@ import (
 	"path/filepath"
 	"strings"
 
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/image"
+	noapi "yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
 	"yunion.io/x/onecloud/pkg/image/options"
 	"yunion.io/x/onecloud/pkg/image/torrent"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -312,11 +315,19 @@ func (self *SImageSubformat) GetDetails() SImageSubformatDetails {
 }
 
 func (self *SImageSubformat) isActive(useFast bool) bool {
-	return isActive(self.GetLocalLocation(), self.Size, self.Checksum, self.FastHash, useFast)
+	active, reason := isActive(self.GetLocalLocation(), self.Size, self.Checksum, self.FastHash, useFast)
+	if active || reason != FileChecksumMismatch {
+		return active
+	}
+	data := jsonutils.NewDict()
+	data.Set("name", jsonutils.NewString(self.ImageId))
+	notifyclient.SystemExceptionNotifyWithResult(context.TODO(), noapi.ActionChecksumTest, noapi.TOPIC_RESOURCE_IMAGE, noapi.ResultFailed, data)
+	return false
 }
 
 func (self *SImageSubformat) isTorrentActive() bool {
-	return isActive(self.getLocalTorrentLocation(), self.TorrentSize, self.TorrentChecksum, "", false)
+	active, _ := isActive(self.getLocalTorrentLocation(), self.TorrentSize, self.TorrentChecksum, "", false)
+	return active
 }
 
 func (self *SImageSubformat) SetStatus(status string) error {

pkg/image/models/images.go

@@ -37,6 +37,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/image"
+	noapi "yunion.io/x/onecloud/pkg/apis/notify"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
@@ -1340,40 +1341,49 @@ func (manager *SImageManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field
 	return q, httperrors.ErrNotFound
 }
 
-func isActive(localPath string, size int64, chksum string, fastHash string, useFastHash bool) bool {
+type sUnactiveReason int
+
+const (
+	FileNoExists sUnactiveReason = iota
+	FileSizeMismatch
+	FileChecksumMismatch
+	Others
+)
+
+func isActive(localPath string, size int64, chksum string, fastHash string, useFastHash bool) (bool, sUnactiveReason) {
 	if len(localPath) == 0 || !fileutils2.Exists(localPath) {
 		log.Errorf("invalid file: %s", localPath)
-		return false
+		return false, FileNoExists
 	}
 	if size != fileutils2.FileSize(localPath) {
 		log.Errorf("size mistmatch: %s", localPath)
-		return false
+		return false, FileSizeMismatch
 	}
 	if len(chksum) == 0 || len(fastHash) == 0 {
-		return true
+		return true, Others
 	}
 	if useFastHash && len(fastHash) > 0 {
 		fhash, err := fileutils2.FastCheckSum(localPath)
 		if err != nil {
 			log.Errorf("IsActive fastChecksum fail %s for %s", err, localPath)
-			return false
+			return false, Others
 		}
 		if fastHash != fhash {
 			log.Errorf("IsActive fastChecksum mismatch for %s", localPath)
-			return false
+			return false, FileChecksumMismatch
 		}
 	} else {
 		md5sum, err := fileutils2.MD5(localPath)
 		if err != nil {
 			log.Errorf("IsActive md5 fail %s for %s", err, localPath)
-			return false
+			return false, Others
 		}
 		if chksum != md5sum {
 			log.Errorf("IsActive checksum mismatch: %s", localPath)
-			return false
+			return false, FileChecksumMismatch
 		}
 	}
-	return true
+	return true, Others
 }
 
 func (self *SImage) IsIso() bool {
@@ -1381,7 +1391,14 @@ func (self *SImage) IsIso() bool {
 }
 
 func (self *SImage) isActive(useFast bool) bool {
-	return isActive(self.GetLocalLocation(), self.Size, self.Checksum, self.FastHash, useFast)
+	active, reason := isActive(self.GetLocalLocation(), self.Size, self.Checksum, self.FastHash, useFast)
+	if active || reason != FileChecksumMismatch {
+		return active
+	}
+	data := jsonutils.NewDict()
+	data.Set("name", jsonutils.NewString(self.Name))
+	notifyclient.SystemExceptionNotifyWithResult(context.TODO(), noapi.ActionChecksumTest, noapi.TOPIC_RESOURCE_IMAGE, noapi.ResultFailed, data)
+	return false
 }
 
 func (self *SImage) DoCheckStatus(ctx context.Context, userCred mcclient.TokenCredential, useFast bool) {

pkg/keystone/driver/sql/sql.go

@@ -74,6 +74,9 @@ func (sql *SSQLDriver) Authenticate(ctx context.Context, ident mcclient.SAuthent
 			// do not lock system account!!!
 			models.UserManager.LockUser(usrExt.Id, "too many failed auth attempts")
 			sql.alertNotify(ctx, usrExt, time.Now())
+			data := jsonutils.NewDict()
+			data.Set("name", jsonutils.NewString(usrExt.Name))
+			notifyclient.SystemEventNotify(ctx, noapi.ActionLock, noapi.TOPIC_RESOURCE_USER, data)
 			return nil, errors.Wrap(httperrors.ErrTooManyAttempts, "user locked")
 		}
 		return nil, errors.Wrap(err, "usrExt.VerifyPassword")

pkg/logger/models/actionlog.go

@@ -26,7 +26,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/apis"
 	api "yunion.io/x/onecloud/pkg/apis/logger"
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/logger/extern"
@@ -73,7 +72,7 @@ func InitActionLog() {
 					"action_tbl",
 					"action",
 					"actions",
-					cloudcommon.ClickhouseDB,
+					db.ClickhouseDB,
 				),
 			},
 		}

pkg/logger/models/baremetalevents.go

@@ -24,7 +24,6 @@ import (
 	"yunion.io/x/sqlchemy/backends/clickhouse"
 
 	api "yunion.io/x/onecloud/pkg/apis/logger"
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/mcclient"
@@ -60,7 +59,7 @@ func InitBaremetalEvent() {
 				"baremetal_event_tbl",
 				"baremetalevent",
 				"baremetalevents",
-				cloudcommon.ClickhouseDB,
+				db.ClickhouseDB,
 			),
 		}
 		col := BaremetalEventManager.TableSpec().ColumnSpec("ops_time")

pkg/notify/models/event.go

@@ -42,7 +42,7 @@ type SEvent struct {
 	db.SStandaloneAnonResourceBase
 
 	Message     string
-	Event       string `width:"32" nullable:"true"`
+	Event       string `width:"64" nullable:"true"`
 	AdvanceDays int
 }
 

pkg/notify/models/event_template.go

@@ -502,6 +502,16 @@ func init() {
 			"MongoDB",
 			"MongoDB",
 		},
+		sI18nElme{
+			api.TOPIC_RESOURCE_DB_TABLE_RECORD,
+			"database table record",
+			"数据库记录",
+		},
+		sI18nElme{
+			api.TOPIC_RESOURCE_CLOUDPODS_COMPONENT,
+			"cloudpods component",
+			"cloudpods服务组件",
+		},
 		sI18nElme{
 			string(api.ActionCreate),
 			"created",

pkg/notify/models/topic.go

@@ -92,6 +92,8 @@ const (
 	DefaultResourceOperationFailed = "resource operation failed"
 	DefaultResourceSync            = "resource sync"
 	DefaultSystemExceptionEvent    = "system exception event"
+	DefaultChecksumTestFailed      = "checksum test failed"
+	DefaultUserLock                = "user lock"
 )
 
 func (sm *STopicManager) InitializeData() error {
@@ -108,6 +110,8 @@ func (sm *STopicManager) InitializeData() error {
 		DefaultResourceOperationFailed,
 		DefaultResourceSync,
 		DefaultSystemExceptionEvent,
+		DefaultChecksumTestFailed,
+		DefaultUserLock,
 	)
 	q := sm.Query()
 	topics := make([]STopic, 0, initSNames.Len())
@@ -294,6 +298,25 @@ func (sm *STopicManager) InitializeData() error {
 				notify.ActionOffline,
 			)
 			t.Type = notify.TOPIC_TYPE_RESOURCE
+		case DefaultChecksumTestFailed:
+			t.addResources(
+				notify.TOPIC_RESOURCE_DB_TABLE_RECORD,
+				notify.TOPIC_RESOURCE_CLOUDPODS_COMPONENT,
+				notify.TOPIC_RESOURCE_SNAPSHOT,
+				notify.TOPIC_RESOURCE_IMAGE,
+			)
+			t.addAction(
+				notify.ActionChecksumTest,
+			)
+			t.Type = notify.TOPIC_TYPE_SECURITY
+		case DefaultUserLock:
+			t.addResources(
+				notify.TOPIC_RESOURCE_USER,
+			)
+			t.addAction(
+				notify.ActionLock,
+			)
+			t.Type = notify.TOPIC_TYPE_SECURITY
 		}
 		if topic == nil {
 			err := sm.TableSpec().Insert(ctx, t)
@@ -418,6 +441,7 @@ func (sm *STopicManager) TopicsByEvent(eventStr string, advanceDays int) ([]STop
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to parse event %q", event)
 	}
+	log.Infof("event: %s", event.String())
 	resourceV := converter.resourceValue(event.ResourceType())
 	if resourceV < 0 {
 		log.Warningf("unknown resource type: %s", event.ResourceType())
@@ -505,6 +529,9 @@ func init() {
 			notify.TOPIC_RESOURCE_LOADBALANCERBACKEDNGROUP: 31,
 			notify.TOPIC_RESOURCE_HOST:                     32,
 			notify.TOPIC_RESOURCE_TASK:                     33,
+			notify.TOPIC_RESOURCE_CLOUDPODS_COMPONENT:      34,
+			notify.TOPIC_RESOURCE_DB_TABLE_RECORD:          35,
+			notify.TOPIC_RESOURCE_USER:                     36,
 		},
 	)
 	converter.registerAction(
@@ -530,6 +557,8 @@ func init() {
 			notify.ActionOffline:            18,
 			notify.ActionSystemPanic:        19,
 			notify.ActionSystemException:    20,
+			notify.ActionChecksumTest:       21,
+			notify.ActionLock:               22,
 		},
 	)
 }

pkg/scheduler/service/service.go

@@ -93,7 +93,7 @@ func StartService() error {
 	common_options.StartOptionManager(&opts, opts.ConfigSyncPeriodSeconds, compute_api.SERVICE_TYPE, compute_api.SERVICE_VERSION, o.OnOptionsChange)
 
 	app := app_common.InitApp(&opts.BaseOptions, true)
-	cloudcommon.AppDBInit(app)
+	db.AppDBInit(app)
 
 	//InitHandlers(app)
 	return startHTTP(opts)

pkg/util/logclient/logclient.go

@@ -29,7 +29,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/appsrv"
-	"yunion.io/x/onecloud/pkg/cloudcommon"
 	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
@@ -67,6 +66,10 @@ type IModule interface {
 	Create(session *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error)
 }
 
+type IStartable interface {
+	GetStartTime() time.Time
+}
+
 // save log to db.
 func AddSimpleActionLog(model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
 	addLog(model, action, iNotes, userCred, success, time.Time{}, &logger.Actions)
@@ -76,7 +79,7 @@ func AddActionLogWithContext(ctx context.Context, model IObject, action string,
 	addLog(model, action, iNotes, userCred, success, appctx.AppContextStartTime(ctx), &logger.Actions)
 }
 
-func AddActionLogWithStartable(task cloudcommon.IStartable, model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
+func AddActionLogWithStartable(task IStartable, model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
 	addLog(model, action, iNotes, userCred, success, task.GetStartTime(), &logger.Actions)
 }
 

pkg/yunionconf/service/service.go

@@ -45,7 +45,7 @@ func StartService() {
 
 	app := app_common.InitApp(baseOpts, true)
 	InitHandlers(app)
-	cloudcommon.AppDBInit(app)
+	db.AppDBInit(app)
 
 	if db.CheckSync(opts.AutoSyncTable, opts.EnableDBChecksumTables, opts.DBChecksumSkipInit) {
 		err := models.InitDB()