opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-31 01:35:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cloudcommon): notify when the checksum of db table record test fail

Browse Source 
1. remove pkg/cloudcommon/db dependency on pkg/cloudcommon
2. init db notifier in pkg/cloudcommon.InitDB
3. move interface 'IStartable' to pkg/util/logclient package which alone references above interface
This commit is contained in:
rainzm
2022-04-01 18:44:12 +08:00
parent 0713624f0c
commit f5d58cdee9
16 changed files with 128 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/cloudcommon/database.go
+11 -29
View File
@@ -17,8 +17,6 @@ package cloudcommon
import (
"context"
"database/sql"
"fmt"
"net/http"
"time"
"github.com/mattn/go-sqlite3"
@@ -28,11 +26,13 @@ import (
"yunion.io/x/pkg/errors"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/appsrv"
noapi "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
"yunion.io/x/onecloud/pkg/cloudcommon/informer"
"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
)
@@ -90,7 +90,7 @@ func InitDB(options *common_options.DBOptions) {
if err != nil {
panic(err)
}
sqlchemy.SetDBWithNameBackend(click, ClickhouseDB, sqlchemy.ClickhouseBackend)
sqlchemy.SetDBWithNameBackend(click, db.ClickhouseDB, sqlchemy.ClickhouseBackend)
if options.OpsLogWithClickhouse {
consts.OpsLogWithClickhouse = true
@@ -123,9 +123,16 @@ func InitDB(options *common_options.DBOptions) {
}
// lm := lockman.NewNoopLockManager()
initDBNotifier()
startInitInformer(options)
}
func initDBNotifier() {
db.SetChecksumTestFailedNotifier(func(obj *jsonutils.JSONDict) {
notifyclient.SystemExceptionNotifyWithResult(context.TODO(), noapi.ActionChecksumTest, noapi.TOPIC_RESOURCE_DB_TABLE_RECORD, noapi.ResultFailed, obj)
})
}
// startInitInformer starts goroutine init informer backend
func startInitInformer(options *common_options.DBOptions) {
go func() {
@@ -167,28 +174,3 @@ func initInformer(options *common_options.DBOptions) error {
func CloseDB() {
sqlchemy.CloseDB()
}
func AppDBInit(app *appsrv.Application) {
dbConn := sqlchemy.GetDB()
if dbConn != nil {
connMax := appsrv.GetDBConnectionCount()
if connMax < MIN_DB_CONN_MAX {
connMax = MIN_DB_CONN_MAX
}
log.Infof("Total %d db workers, set db connection max", connMax)
dbConn.SetMaxIdleConns(connMax)
dbConn.SetMaxOpenConns(connMax*2 + 1)
}
app.AddDefaultHandler("GET", "/db_stats", DBStatsHandler, "db_stats")
}
func DBStatsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
result := jsonutils.NewDict()
dbConn := sqlchemy.GetDB()
if dbConn != nil {
stats := dbConn.Stats()
result.Add(jsonutils.Marshal(&stats), "db_stats")
}
fmt.Fprintf(w, result.String())
}
pkg/cloudcommon/db/checksum.go
+16
View File
@@ -20,12 +20,19 @@ import (
"reflect"
"sort"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/util/reflectutils"
"yunion.io/x/pkg/utils"
)
var checksumTestFailedNotifier func(obj *jsonutils.JSONDict)
func SetChecksumTestFailedNotifier(notifier func(obj *jsonutils.JSONDict)) {
checksumTestFailedNotifier = notifier
}
type IRecordChecksumResourceBase interface {
GetRecordChecksum() string
SetRecordChecksum(checksum string)
@@ -93,6 +100,15 @@ func CheckRecordChecksumConsistent(model IModel) error {
savedChecksum := obj.GetRecordChecksum()
if calChecksum != savedChecksum {
log.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
ts := model.GetModelManager().TableSpec()
// notify
data := jsonutils.NewDict()
data.Set("db_name", jsonutils.NewString(string(ts.GetDBName())))
data.Set("table_name", jsonutils.NewString(ts.Name()))
data.Set("name", jsonutils.NewString(fmt.Sprintf("%s(%s)", obj.Keyword(), obj.GetId())))
if checksumTestFailedNotifier != nil {
checksumTestFailedNotifier(data)
}
return errors.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
}
return nil
pkg/cloudcommon/db/database.go
+58
View File
@@ -0,0 +1,58 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package db
import (
"context"
"fmt"
"net/http"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/appsrv"
)
const (
MIN_DB_CONN_MAX = 5
ClickhouseDB = sqlchemy.DBName("clickhosue_db")
)
func AppDBInit(app *appsrv.Application) {
dbConn := sqlchemy.GetDB()
if dbConn != nil {
connMax := appsrv.GetDBConnectionCount()
if connMax < MIN_DB_CONN_MAX {
connMax = MIN_DB_CONN_MAX
}
log.Infof("Total %d db workers, set db connection max", connMax)
dbConn.SetMaxIdleConns(connMax)
dbConn.SetMaxOpenConns(connMax*2 + 1)
}
app.AddDefaultHandler("GET", "/db_stats", DBStatsHandler, "db_stats")
}
func DBStatsHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
result := jsonutils.NewDict()
dbConn := sqlchemy.GetDB()
if dbConn != nil {
stats := dbConn.Stats()
result.Add(jsonutils.Marshal(&stats), "db_stats")
}
fmt.Fprintf(w, result.String())
}
pkg/cloudcommon/db/models.go
+1 -2
View File
@@ -24,7 +24,6 @@ import (
"yunion.io/x/sqlchemy"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
)
@@ -178,7 +177,7 @@ func EnsureAppSyncDB(app *appsrv.Application, opt *common_options.DBOptions, mod
os.Exit(0)
}
cloudcommon.AppDBInit(app)
AppDBInit(app)
}
func GetModelManager(keyword string) IModelManager {
pkg/cloudcommon/db/opslog.go
+1 -2
View File
@@ -34,7 +34,6 @@ import (
"yunion.io/x/onecloud/pkg/apis"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/httperrors"
"yunion.io/x/onecloud/pkg/mcclient"
@@ -89,7 +88,7 @@ func InitOpsLog() {
"opslog_tbl",
"event",
"events",
cloudcommon.ClickhouseDB,
ClickhouseDB,
)}
col := OpsLog.TableSpec().ColumnSpec("ops_time")
if clickCol, ok := col.(clickhouse.IClickhouseColumnSpec); ok {
pkg/cloudcommon/db/tablespec.go
+25 -1
View File
@@ -24,6 +24,7 @@ import (
"yunion.io/x/pkg/errors"
"yunion.io/x/sqlchemy"
api "yunion.io/x/onecloud/pkg/apis/notify"
"yunion.io/x/onecloud/pkg/cloudcommon/informer"
"yunion.io/x/onecloud/pkg/util/nopanic"
"yunion.io/x/onecloud/pkg/util/splitable"
@@ -51,6 +52,8 @@ type ITableSpec interface {
GetSplitTable() *splitable.SSplitTableSpec
GetTableSpec() *sqlchemy.STableSpec
GetDBName() sqlchemy.DBName
}
type sTableSpec struct {
@@ -171,7 +174,28 @@ func (ts *sTableSpec) InsertOrUpdate(ctx context.Context, dt interface{}) error
}
func (ts *sTableSpec) CheckRecordChanged(dbObj IModel) error {
return CheckRecordChecksumConsistent(dbObj)
return ts.CheckRecordChecksumConsistent(dbObj)
}
func (ts *sTableSpec) CheckRecordChecksumConsistent(model IModel) error {
obj, ok := IsModelEnableRecordChecksum(model)
if !ok {
return nil
}
calChecksum, err := CalculateModelChecksum(obj)
if err != nil {
return errors.Wrap(err, "CalculateModelChecksum")
}
savedChecksum := obj.GetRecordChecksum()
if calChecksum != savedChecksum {
log.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
return errors.Errorf("Record %s(%s) checksum changed, expected(%s) != calculated(%s)", obj.Keyword(), obj.GetId(), savedChecksum, calChecksum)
}
return nil
}
func checksumTestNotify(ctx context.Context, action api.SAction, resType string, obj jsonutils.JSONObject) {
}
func (ts *sTableSpec) Update(ctx context.Context, dt interface{}, doUpdate func() error) (sqlchemy.UpdateDiffs, error) {
pkg/cloudcommon/db/taskman/interface.go
+2 -2
View File
@@ -17,16 +17,16 @@ package taskman
import (
"context"
"net/http"
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/db/quotas"
"yunion.io/x/onecloud/pkg/mcclient"
)
type ITask interface {
cloudcommon.IStartable
GetStartTime() time.Time
ScheduleRun(data jsonutils.JSONObject) error
GetParams() *jsonutils.JSONDict
pkg/cloudcommon/interface.go
-21
View File
@@ -1,21 +0,0 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cloudcommon
import "time"
type IStartable interface {
GetStartTime() time.Time
}
pkg/cloudevent/models/cloudevents.go
+1 -2
View File
@@ -26,7 +26,6 @@ import (
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/cloudevent"
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -50,7 +49,7 @@ func InitCloudevent() {
"cloudevents_tbl",
"cloudevent",
"cloudevents",
cloudcommon.ClickhouseDB,
db.ClickhouseDB,
),
}
col := CloudeventManager.TableSpec().ColumnSpec("created_at")
pkg/cloudir/service/service.go
+2 -2
View File
@@ -19,8 +19,8 @@ import (
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudcommon"
app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
"yunion.io/x/onecloud/pkg/cloudir/options"
@@ -43,7 +43,7 @@ func StartService() {
}
app := app_common.InitApp(baseOpts, false)
cloudcommon.AppDBInit(app)
db.AppDBInit(app)
initHandlers(app)
app_common.ServeForeverWithCleanup(app, baseOpts, func() {
pkg/cloutpost/service/service.go
+2 -2
View File
@@ -19,8 +19,8 @@ import (
"yunion.io/x/log"
"yunion.io/x/onecloud/pkg/cloudcommon"
app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/cloudcommon/etcd"
"yunion.io/x/onecloud/pkg/cloudcommon/etcd/models"
common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
@@ -48,7 +48,7 @@ func StartService() {
defer etcd.CloseDefaultEtcdClient()
app := app_common.InitApp(baseOpts, false)
cloudcommon.AppDBInit(app)
db.AppDBInit(app)
initHandlers(app)
err = models.ServiceRegistryManager.Register(
pkg/logger/models/actionlog.go
+1 -2
View File
@@ -26,7 +26,6 @@ import (
"yunion.io/x/onecloud/pkg/apis"
api "yunion.io/x/onecloud/pkg/apis/logger"
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/logger/extern"
@@ -73,7 +72,7 @@ func InitActionLog() {
"action_tbl",
"action",
"actions",
cloudcommon.ClickhouseDB,
db.ClickhouseDB,
),
},
}
pkg/logger/models/baremetalevents.go
+1 -2
View File
@@ -24,7 +24,6 @@ import (
"yunion.io/x/sqlchemy/backends/clickhouse"
api "yunion.io/x/onecloud/pkg/apis/logger"
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/cloudcommon/db"
"yunion.io/x/onecloud/pkg/mcclient"
@@ -60,7 +59,7 @@ func InitBaremetalEvent() {
"baremetal_event_tbl",
"baremetalevent",
"baremetalevents",
cloudcommon.ClickhouseDB,
db.ClickhouseDB,
),
}
col := BaremetalEventManager.TableSpec().ColumnSpec("ops_time")
pkg/scheduler/service/service.go
+1 -1
View File
@@ -93,7 +93,7 @@ func StartService() error {
common_options.StartOptionManager(&opts, opts.ConfigSyncPeriodSeconds, compute_api.SERVICE_TYPE, compute_api.SERVICE_VERSION, o.OnOptionsChange)
app := app_common.InitApp(&opts.BaseOptions, true)
cloudcommon.AppDBInit(app)
db.AppDBInit(app)
//InitHandlers(app)
return startHTTP(opts)
pkg/util/logclient/logclient.go
+5 -2
View File
@@ -29,7 +29,6 @@ import (
"yunion.io/x/onecloud/pkg/appctx"
"yunion.io/x/onecloud/pkg/appsrv"
"yunion.io/x/onecloud/pkg/cloudcommon"
"yunion.io/x/onecloud/pkg/cloudcommon/consts"
"yunion.io/x/onecloud/pkg/mcclient"
"yunion.io/x/onecloud/pkg/mcclient/auth"
@@ -67,6 +66,10 @@ type IModule interface {
Create(session *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error)
}
type IStartable interface {
GetStartTime() time.Time
}
// save log to db.
func AddSimpleActionLog(model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
addLog(model, action, iNotes, userCred, success, time.Time{}, &logger.Actions)
@@ -76,7 +79,7 @@ func AddActionLogWithContext(ctx context.Context, model IObject, action string,
addLog(model, action, iNotes, userCred, success, appctx.AppContextStartTime(ctx), &logger.Actions)
}
func AddActionLogWithStartable(task cloudcommon.IStartable, model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
func AddActionLogWithStartable(task IStartable, model IObject, action string, iNotes interface{}, userCred mcclient.TokenCredential, success bool) {
addLog(model, action, iNotes, userCred, success, task.GetStartTime(), &logger.Actions)
}
pkg/yunionconf/service/service.go
+1 -1
View File
@@ -45,7 +45,7 @@ func StartService() {
app := app_common.InitApp(baseOpts, true)
InitHandlers(app)
cloudcommon.AppDBInit(app)
db.AppDBInit(app)
if db.CheckSync(opts.AutoSyncTable, opts.EnableDBChecksumTables, opts.DBChecksumSkipInit) {
err := models.InitDB()