From 4476b9f70147e37ecb60efb5c744e1c32d57e332 Mon Sep 17 00:00:00 2001 From: Qiu Jian Date: Thu, 7 Jul 2022 00:13:16 +0800 Subject: [PATCH] fix: log admin action logs in separate logs --- pkg/apis/input.go | 8 +- pkg/cloudcommon/db/db_dispatcher.go | 133 ++++++++++++---------- pkg/cloudcommon/db/db_joint_dispatcher.go | 28 +++-- pkg/cloudcommon/db/interface.go | 3 + pkg/cloudcommon/db/modelbase.go | 64 ++++++++--- pkg/cloudcommon/db/models.go | 15 ++- pkg/logger/models/actionlog.go | 95 +++++++++++----- pkg/logger/options/options.go | 6 + pkg/logger/service/handlers.go | 5 + pkg/util/splitable/insert.go | 6 +- pkg/util/splitable/metadata.go | 15 +++ 11 files changed, 261 insertions(+), 117 deletions(-) diff --git a/pkg/apis/input.go b/pkg/apis/input.go index 427dd094ba..a688d7cd5c 100644 --- a/pkg/apis/input.go +++ b/pkg/apis/input.go @@ -399,6 +399,12 @@ type PurgeSplitTableInput struct { Tables []string `json:"tables"` } +const ( + MAX_SPLITABLE_EXPORT_LIMIT = 10000 +) + type SplitTableExportInput struct { - Table string `json:"table"` + Table string `json:"table"` + Offset int `json:"offset"` + Limit int `json:"limit"` } diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 894fb96f4b..70b6ba6c14 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -46,24 +46,24 @@ import ( ) type DBModelDispatcher struct { - modelManager IModelManager + manager IModelManager } func NewModelHandler(manager IModelManager) *DBModelDispatcher { // registerModelManager(manager) - return &DBModelDispatcher{modelManager: manager} + return &DBModelDispatcher{manager: manager} } func (dispatcher *DBModelDispatcher) Keyword() string { - return dispatcher.modelManager.Keyword() + return dispatcher.manager.Keyword() } func (dispatcher *DBModelDispatcher) KeywordPlural() string { - return dispatcher.modelManager.KeywordPlural() + return dispatcher.manager.KeywordPlural() } func (dispatcher *DBModelDispatcher) ContextKeywordPlurals() [][]string { - ctxMans := dispatcher.modelManager.GetContextManagers() + ctxMans := dispatcher.manager.GetContextManagers() if ctxMans != nil { keys := make([][]string, len(ctxMans)) for i := 0; i < len(ctxMans); i += 1 { @@ -82,7 +82,7 @@ func (dispatcher *DBModelDispatcher) Filter(f appsrv.FilterHandler) appsrv.Filte } func (dispatcher *DBModelDispatcher) CustomizeHandlerInfo(handler *appsrv.SHandlerInfo) { - dispatcher.modelManager.CustomizeHandlerInfo(handler) + dispatcher.manager.CustomizeHandlerInfo(handler) } func fetchUserCredential(ctx context.Context) mcclient.TokenCredential { @@ -844,13 +844,14 @@ func getExportCols(query jsonutils.JSONObject, retList []jsonutils.JSONObject) [ func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*modulebase.ListResult, error) { userCred := fetchUserCredential(ctx) + manager := dispatcher.manager.GetImmutableInstance(userCred) - items, err := ListItems(dispatcher.modelManager, ctx, userCred, query, ctxIds) + items, err := ListItems(manager, ctx, userCred, query, ctxIds) if err != nil { return nil, httperrors.NewGeneralError(errors.Wrapf(err, "ListItems")) } - if userCred != nil && userCred.HasSystemAdminPrivilege() && dispatcher.modelManager.ListSkipLog(ctx, userCred, query) { + if userCred != nil && userCred.HasSystemAdminPrivilege() && manager.ListSkipLog(ctx, userCred, query) { appParams := appsrv.AppContextGetParams(ctx) if appParams != nil { appParams.SkipLog = true @@ -909,7 +910,8 @@ func getItemDetails(manager IModelManager, item IModel, ctx context.Context, use func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, property string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) funcName := fmt.Sprintf("GetProperty%s", utils.Kebab2Camel(property, "-")) - modelValue := reflect.ValueOf(dispatcher.modelManager) + manager := dispatcher.manager.GetImmutableInstance(userCred) + modelValue := reflect.ValueOf(manager) params := []interface{}{ctx, userCred, query} funcValue := modelValue.MethodByName(funcName) @@ -917,7 +919,7 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr return nil, nil } - _, _, err, _ := FetchCheckQueryOwnerScope(ctx, userCred, query, dispatcher.modelManager, policy.PolicyActionList, true) + _, _, err, _ := FetchCheckQueryOwnerScope(ctx, userCred, query, manager, policy.PolicyActionList, true) if err != nil { return nil, err } @@ -946,13 +948,14 @@ func (dispatcher *DBModelDispatcher) tryGetModelProperty(ctx context.Context, pr func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, query jsonutils.JSONObject, isHead bool) (jsonutils.JSONObject, error) { // log.Debugf("Get %s", idStr) userCred := fetchUserCredential(ctx) + manager := dispatcher.manager.GetImmutableInstance(userCred) data, err := dispatcher.tryGetModelProperty(ctx, idStr, query) if err != nil { return nil, err } else if data != nil { if dataDict, ok := data.(*jsonutils.JSONDict); ok { - i18nDict := dispatcher.modelManager.GetI18N(ctx, idStr, data) + i18nDict := manager.GetI18N(ctx, idStr, data) if i18nDict != nil { dataDict.Set("_i18n", i18nDict) } @@ -960,9 +963,9 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer return data, nil } - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, query) + model, err := fetchItem(manager, ctx, userCred, idStr, query) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idStr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr) } else if err != nil { return nil, errors.Wrapf(err, "fetchItem") } @@ -972,20 +975,21 @@ func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, quer return nil, err } - if userCred.HasSystemAdminPrivilege() && dispatcher.modelManager.GetSkipLog(ctx, userCred, query) { + if userCred.HasSystemAdminPrivilege() && manager.GetSkipLog(ctx, userCred, query) { appParams := appsrv.AppContextGetParams(ctx) if appParams != nil { appParams.SkipLog = true } } - return getModelItemDetails(dispatcher.modelManager, model, ctx, userCred, query, isHead) + return getModelItemDetails(manager, model, ctx, userCred, query, isHead) } func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr string, spec string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, query) + manager := dispatcher.manager.GetImmutableInstance(userCred) + model, err := fetchItem(manager, ctx, userCred, idStr, query) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idStr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr) } else if err != nil { return nil, err } @@ -1264,12 +1268,12 @@ func _doCreateItem( } func (dispatcher *DBModelDispatcher) FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) { - return dispatcher.modelManager.FetchCreateHeaderData(ctx, header) + return dispatcher.manager.FetchCreateHeaderData(ctx, header) } func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - manager := dispatcher.modelManager + manager := dispatcher.manager.GetMutableInstance(userCred) ownerId, err := fetchOwnerId(ctx, manager, userCred, data) if err != nil { @@ -1281,14 +1285,14 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils if !ok { return nil, httperrors.NewGeneralError(fmt.Errorf("fail to parse body %s", data)) } - data, err = fetchContextObjectsIds(dispatcher.modelManager, ctx, userCred, ctxIds, dataDict) + data, err = fetchContextObjectsIds(manager, ctx, userCred, ctxIds, dataDict) if err != nil { return nil, httperrors.NewGeneralError(errors.Wrapf(err, "fetchContextObjectsIds")) } } var policyResult rbacutils.SPolicyResult - policyResult, err = isClassRbacAllowed(ctx, dispatcher.modelManager, userCred, ownerId, policy.PolicyActionCreate) + policyResult, err = isClassRbacAllowed(ctx, manager, userCred, ownerId, policy.PolicyActionCreate) if err != nil { return nil, errors.Wrap(err, "isClassRbacAllowed") } @@ -1299,7 +1303,7 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils data.(*jsonutils.JSONDict).Update(policyResult.Json()) - model, err := DoCreate(dispatcher.modelManager, ctx, userCred, query, data, ownerId) + model, err := DoCreate(manager, ctx, userCred, query, data, ownerId) if err != nil { if CancelPendingUsagesInContext != nil { e := CancelPendingUsagesInContext(ctx, userCred) @@ -1326,8 +1330,8 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils OpsLog.LogEvent(model, ACT_CREATE, notes, userCred) logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CREATE, notes, userCred, true) } - dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, data) - return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query) + manager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, data) + return getItemDetails(manager, model, ctx, userCred, query) } func expandMultiCreateParams(manager IModelManager, data jsonutils.JSONObject, count int) ([]jsonutils.JSONObject, error) { @@ -1355,8 +1359,7 @@ func expandMultiCreateParams(manager IModelManager, data jsonutils.JSONObject, c func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxIds []dispatcher.SResourceContext) ([]modulebase.SubmitResult, error) { userCred := fetchUserCredential(ctx) - - manager := dispatcher.modelManager + manager := dispatcher.manager.GetMutableInstance(userCred) ownerId, err := fetchOwnerId(ctx, manager, userCred, data) if err != nil { @@ -1488,7 +1491,7 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act } userCred := fetchUserCredential(ctx) - manager := dispatcher.modelManager + manager := dispatcher.manager.GetMutableInstance(userCred) ownerId, err := fetchOwnerId(ctx, manager, userCred, data) if err != nil { @@ -1498,15 +1501,16 @@ func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, act lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId)) defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId)) - managerValue := reflect.ValueOf(dispatcher.modelManager) - return objectPerformAction(dispatcher, nil, managerValue, ctx, userCred, action, query, data) + managerValue := reflect.ValueOf(manager) + return objectPerformAction(manager, nil, managerValue, ctx, userCred, action, query, data) } func (dispatcher *DBModelDispatcher) PerformAction(ctx context.Context, idStr string, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, nil) + manager := dispatcher.manager.GetMutableInstance(userCred) + model, err := fetchItem(manager, ctx, userCred, idStr, nil) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idStr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1517,15 +1521,16 @@ func (dispatcher *DBModelDispatcher) PerformAction(ctx context.Context, idStr st if err := model.PreCheckPerformAction(ctx, userCred, action, query, data); err != nil { return nil, err } - return objectPerformAction(dispatcher, model, reflect.ValueOf(model), ctx, userCred, action, query, data) + return objectPerformAction(manager, model, reflect.ValueOf(model), ctx, userCred, action, query, data) } -func objectPerformAction(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - return reflectDispatcher(dispatcher, model, modelValue, ctx, userCred, policy.PolicyActionPerform, "PerformAction", "Perform", action, query, data) +func objectPerformAction(manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return reflectDispatcher(manager, model, modelValue, ctx, userCred, policy.PolicyActionPerform, "PerformAction", "Perform", action, query, data) } func reflectDispatcher( - dispatcher *DBModelDispatcher, + // dispatcher *DBModelDispatcher, + manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, @@ -1538,15 +1543,16 @@ func reflectDispatcher( data jsonutils.JSONObject, ) (jsonutils.JSONObject, error) { result, err := reflectDispatcherInternal( - dispatcher, model, modelValue, ctx, userCred, operator, generalFuncName, funcPrefix, spec, query, data) + manager, model, modelValue, ctx, userCred, operator, generalFuncName, funcPrefix, spec, query, data) if model != nil && err == nil && result == nil { - return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query) + return getItemDetails(manager, model, ctx, userCred, query) } else { return result, err } } func reflectDispatcherInternal( - dispatcher *DBModelDispatcher, + // dispatcher *DBModelDispatcher, + manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, @@ -1566,7 +1572,7 @@ func reflectDispatcherInternal( funcValue = modelValue.MethodByName(generalFuncName) if !funcValue.IsValid() || funcValue.IsNil() { return nil, httperrors.NewActionNotFoundError("%s %s %s not found", - dispatcher.Keyword(), operator, spec) + manager.Keyword(), operator, spec) } else { isGeneral = true funcName = generalFuncName @@ -1583,11 +1589,11 @@ func reflectDispatcherInternal( var result rbacutils.SPolicyResult if model == nil { - ownerId, err := fetchOwnerId(ctx, dispatcher.modelManager, userCred, data) + ownerId, err := fetchOwnerId(ctx, manager, userCred, data) if err != nil { return nil, httperrors.NewGeneralError(err) } - _, err = isClassRbacAllowed(ctx, dispatcher.modelManager, userCred, ownerId, operator, spec) + _, err = isClassRbacAllowed(ctx, manager, userCred, ownerId, operator, spec) if err != nil { return nil, err } @@ -1674,14 +1680,15 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre } func (dispatcher *DBModelDispatcher) FetchUpdateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) { - return dispatcher.modelManager.FetchUpdateHeaderData(ctx, header) + return dispatcher.manager.FetchUpdateHeaderData(ctx, header) } func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, nil) + manager := dispatcher.manager.GetMutableInstance(userCred) + model, err := fetchItem(manager, ctx, userCred, idStr, nil) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idStr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1693,7 +1700,7 @@ func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, q data.(*jsonutils.JSONDict).Update(result.Json()) if len(ctxIds) > 0 { - ctxObjs, err := fetchContextObjects(dispatcher.modelManager, ctx, userCred, ctxIds) + ctxObjs, err := fetchContextObjects(manager, ctx, userCred, ctxIds) if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1703,15 +1710,17 @@ func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, q lockman.LockObject(ctx, model) defer lockman.ReleaseObject(ctx, model) - return updateItem(dispatcher.modelManager, model, ctx, userCred, query, data) + return updateItem(manager, model, ctx, userCred, query, data) } } func (dispatcher *DBModelDispatcher) UpdateSpec(ctx context.Context, idStr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idStr, nil) + manager := dispatcher.manager.GetMutableInstance(userCred) + + model, err := fetchItem(manager, ctx, userCred, idStr, nil) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idStr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1719,11 +1728,11 @@ func (dispatcher *DBModelDispatcher) UpdateSpec(ctx context.Context, idStr strin lockman.LockObject(ctx, model) defer lockman.ReleaseObject(ctx, model) - return objectUpdateSpec(dispatcher, model, reflect.ValueOf(model), ctx, userCred, spec, query, data) + return objectUpdateSpec(manager, model, reflect.ValueOf(model), ctx, userCred, spec, query, data) } -func objectUpdateSpec(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - return reflectDispatcher(dispatcher, model, modelValue, ctx, userCred, policy.PolicyActionUpdate, "UpdateSpec", "Update", spec, query, data) +func objectUpdateSpec(manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return reflectDispatcher(manager, model, modelValue, ctx, userCred, policy.PolicyActionUpdate, "UpdateSpec", "Update", spec, query, data) } func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error { @@ -1778,9 +1787,11 @@ func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCr func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idstr, nil) + manager := dispatcher.manager.GetMutableInstance(userCred) + + model, err := fetchItem(manager, ctx, userCred, idstr, nil) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idstr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idstr) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1791,7 +1802,7 @@ func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, q } if len(ctxIds) > 0 { - ctxObjs, err := fetchContextObjects(dispatcher.modelManager, ctx, userCred, ctxIds) + ctxObjs, err := fetchContextObjects(manager, ctx, userCred, ctxIds) if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1801,15 +1812,17 @@ func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, q lockman.LockObject(ctx, model) defer lockman.ReleaseObject(ctx, model) - return deleteItem(dispatcher.modelManager, model, ctx, userCred, query, data) + return deleteItem(manager, model, ctx, userCred, query, data) } } func (dispatcher *DBModelDispatcher) DeleteSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) - model, err := fetchItem(dispatcher.modelManager, ctx, userCred, idstr, nil) + manager := dispatcher.manager.GetMutableInstance(userCred) + + model, err := fetchItem(manager, ctx, userCred, idstr, nil) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), idstr) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idstr) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -1817,9 +1830,9 @@ func (dispatcher *DBModelDispatcher) DeleteSpec(ctx context.Context, idstr strin lockman.LockObject(ctx, model) defer lockman.ReleaseObject(ctx, model) - return objectDeleteSpec(dispatcher, model, reflect.ValueOf(model), ctx, userCred, spec, query, data) + return objectDeleteSpec(manager, model, reflect.ValueOf(model), ctx, userCred, spec, query, data) } -func objectDeleteSpec(dispatcher *DBModelDispatcher, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { - return reflectDispatcher(dispatcher, model, modelValue, ctx, userCred, policy.PolicyActionDelete, "DeleteSpec", "Delete", spec, query, data) +func objectDeleteSpec(manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { + return reflectDispatcher(manager, model, modelValue, ctx, userCred, policy.PolicyActionDelete, "DeleteSpec", "Delete", spec, query, data) } diff --git a/pkg/cloudcommon/db/db_joint_dispatcher.go b/pkg/cloudcommon/db/db_joint_dispatcher.go index 79f74bacd3..84b3231d1f 100644 --- a/pkg/cloudcommon/db/db_joint_dispatcher.go +++ b/pkg/cloudcommon/db/db_joint_dispatcher.go @@ -35,11 +35,11 @@ type DBJointModelDispatcher struct { func NewJointModelHandler(manager IJointModelManager) *DBJointModelDispatcher { // registerModelManager(manager) - return &DBJointModelDispatcher{DBModelDispatcher: DBModelDispatcher{modelManager: manager}} + return &DBJointModelDispatcher{DBModelDispatcher: DBModelDispatcher{manager: manager}} } func (dispatcher *DBJointModelDispatcher) JointModelManager() IJointModelManager { - return dispatcher.modelManager.(IJointModelManager) + return dispatcher.manager.(IJointModelManager) } func (dispatcher *DBJointModelDispatcher) MasterKeywordPlural() string { @@ -139,9 +139,11 @@ func fetchJointItem(dispatcher *DBJointModelDispatcher, ctx context.Context, use func (dispatcher *DBJointModelDispatcher) Get(ctx context.Context, id1 string, id2 string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) + manager := dispatcher.manager.GetImmutableInstance(userCred) + _, _, item, err := fetchJointItem(dispatcher, ctx, userCred, id1, id2, query) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), id1+"-"+id2) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), id1+"-"+id2) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -152,7 +154,15 @@ func (dispatcher *DBJointModelDispatcher) Get(ctx context.Context, id1 string, i return getItemDetails(dispatcher.JointModelManager(), item, ctx, userCred, query) } -func attachItems(dispatcher *DBJointModelDispatcher, master IStandaloneModel, slave IStandaloneModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { +func attachItems( + dispatcher *DBJointModelDispatcher, + master IStandaloneModel, + slave IStandaloneModel, + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, + data jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { err := isObjectRbacAllowed(ctx, master, userCred, policy.PolicyActionPerform, "attach") if err != nil { return nil, err @@ -180,7 +190,7 @@ func attachItems(dispatcher *DBJointModelDispatcher, master IStandaloneModel, sl } item.PostCreate(ctx, userCred, nil, query, data) OpsLog.LogAttachEvent(ctx, master, slave, userCred, jsonutils.Marshal(item)) - dispatcher.modelManager.OnCreateComplete(ctx, []IModel{item}, userCred, nil, query, data) + dispatcher.manager.OnCreateComplete(ctx, []IModel{item}, userCred, nil, query, data) return getItemDetails(dispatcher.JointModelManager(), item, ctx, userCred, query) } @@ -218,6 +228,8 @@ func (dispatcher *DBJointModelDispatcher) Attach(ctx context.Context, id1 string func (dispatcher *DBJointModelDispatcher) Update(ctx context.Context, id1 string, id2 string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) + manager := dispatcher.manager.GetMutableInstance(userCred) + master, slave, item, err := fetchJointItem(dispatcher, ctx, userCred, id1, id2, query) if err == sql.ErrNoRows { if jsonutils.QueryBoolean(query, "auto_create", false) { @@ -225,7 +237,7 @@ func (dispatcher *DBJointModelDispatcher) Update(ctx context.Context, id1 string queryDict.Remove("auto_create") return dispatcher.Attach(ctx, id1, id2, query, data) } - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), id1+"-"+id2) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), id1+"-"+id2) } else if err != nil { return nil, httperrors.NewGeneralError(err) } @@ -242,9 +254,11 @@ func (dispatcher *DBJointModelDispatcher) Update(ctx context.Context, id1 string func (dispatcher *DBJointModelDispatcher) Detach(ctx context.Context, id1 string, id2 string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) { userCred := fetchUserCredential(ctx) + manager := dispatcher.manager.GetMutableInstance(userCred) + master, slave, item, err := fetchJointItem(dispatcher, ctx, userCred, id1, id2, query) if err == sql.ErrNoRows { - return nil, httperrors.NewResourceNotFoundError2(dispatcher.modelManager.Keyword(), id1+"-"+id2) + return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), id1+"-"+id2) } else if err != nil { return nil, httperrors.NewGeneralError(err) } diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 56b7bc1fdc..64fafe0920 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -40,6 +40,9 @@ type IModelManager interface { GetIModelManager() IModelManager + GetMutableInstance(userCred mcclient.TokenCredential) IModelManager + GetImmutableInstance(userCred mcclient.TokenCredential) IModelManager + // Table() *sqlchemy.STable TableSpec() ITableSpec diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 0987343e99..11f0119a48 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -72,7 +72,11 @@ func NewModelBaseManagerWithSplitable(model interface{}, tableName string, keywo func NewModelBaseManagerWithSplitableDBName(model interface{}, tableName string, keyword string, keywordPlural string, indexField string, dateField string, maxDuration time.Duration, maxSegments int, dbName sqlchemy.DBName) SModelBaseManager { ts := newTableSpec(model, tableName, indexField, dateField, maxDuration, maxSegments, dbName) - modelMan := SModelBaseManager{tableSpec: ts, keyword: keyword, keywordPlural: keywordPlural} + modelMan := SModelBaseManager{ + tableSpec: ts, + keyword: keyword, + keywordPlural: keywordPlural, + } return modelMan } @@ -96,6 +100,14 @@ func (manager *SModelBaseManager) GetIModelManager() IModelManager { return r } +func (manager *SModelBaseManager) GetImmutableInstance(userCred mcclient.TokenCredential) IModelManager { + return manager.GetIModelManager() +} + +func (manager *SModelBaseManager) GetMutableInstance(userCred mcclient.TokenCredential) IModelManager { + return manager.GetIModelManager() +} + func (manager *SModelBaseManager) SetAlias(alias string, aliasPlural string) { manager.alias = alias manager.aliasPlural = aliasPlural @@ -106,7 +118,7 @@ func (manager *SModelBaseManager) TableSpec() ITableSpec { } func (manager *SModelBaseManager) GetSplitTable() *splitable.SSplitTableSpec { - return manager.tableSpec.GetSplitTable() + return manager.TableSpec().GetSplitTable() } func (manager *SModelBaseManager) Keyword() string { @@ -191,7 +203,7 @@ func (manager *SModelBaseManager) ExtraSearchConditions(ctx context.Context, q * // fetch hook func (manager *SModelBaseManager) getTable() *sqlchemy.STable { - return manager.tableSpec.Instance() + return manager.TableSpec().Instance() } func (manager *SModelBaseManager) Query(fieldNames ...string) *sqlchemy.SQuery { @@ -446,16 +458,32 @@ func (manager *SModelBaseManager) GetI18N(ctx context.Context, idstr string, res return nil } -func (manager *SModelBaseManager) AllowGetPropertySplitable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { - return true -} - func (manager *SModelBaseManager) GetPropertySplitable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { - splitable := manager.GetSplitTable() - if splitable == nil { - return nil, errors.Wrap(httperrors.ErrNotSupported, "not splitable") + stable := manager.GetImmutableInstance(userCred).GetSplitTable() + if stable == nil { + // generate a fake metadata tbl record + man := manager.GetImmutableInstance(userCred) + subq := man.Query().SubQuery() + q := subq.Query( + sqlchemy.MIN("start", subq.Field("id")), + sqlchemy.MAX("end", subq.Field("id")), + sqlchemy.MIN("start_date", subq.Field("ops_time")), + sqlchemy.MAX("end_date", subq.Field("ops_time")), + sqlchemy.COUNT("count", subq.Field("id")), + sqlchemy.MIN("created_at", subq.Field("ops_time")), + ) + meta := splitable.STableMetadata{ + Id: 1, + Table: "action_tbl", + } + err := q.First(&meta) + if err != nil { + return nil, errors.Wrap(err, "Query metadata") + } + metas := []splitable.STableMetadata{meta} + return jsonutils.Marshal(metas), nil } - metas, err := splitable.GetTableMetas() + metas, err := stable.GetTableMetas() if err != nil { return nil, errors.Wrap(err, "GetTableMetas") } @@ -463,7 +491,7 @@ func (manager *SModelBaseManager) GetPropertySplitable(ctx context.Context, user } func (manager *SModelBaseManager) GetPropertySplitableExport(ctx context.Context, userCred mcclient.TokenCredential, input apis.SplitTableExportInput) (jsonutils.JSONObject, error) { - splitable := manager.GetSplitTable() + splitable := manager.GetImmutableInstance(userCred).GetSplitTable() if splitable == nil { return nil, errors.Wrap(httperrors.ErrNotSupported, "not splitable") } @@ -477,7 +505,13 @@ func (manager *SModelBaseManager) GetPropertySplitableExport(ctx context.Context } for i := 0; i < len(metas); i += 1 { if metas[i].Table == input.Table { - q := splitable.GetTableSpec(metas[i]).Query() + if input.Limit <= 0 { + input.Limit = apis.MAX_SPLITABLE_EXPORT_LIMIT + } + if input.Offset < 0 { + input.Offset = 0 + } + q := splitable.GetTableSpec(metas[i]).Query().Limit(input.Limit).Offset(input.Offset) resp, err := q.AllStringMap() if err != nil { return nil, errors.Wrapf(err, "q.AllStringMap") @@ -493,9 +527,9 @@ func (manager *SModelBaseManager) AllowPerformPurgeSplitable(ctx context.Context } func (manager *SModelBaseManager) PerformPurgeSplitable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PurgeSplitTableInput) (jsonutils.JSONObject, error) { - splitable := manager.GetSplitTable() + splitable := manager.GetImmutableInstance(userCred).GetSplitTable() if splitable == nil { - return nil, errors.Wrap(httperrors.ErrNotSupported, "not splitable") + return jsonutils.Marshal(map[string][]string{"tables": []string{}}), nil } ret, err := splitable.Purge(input.Tables) if err != nil { diff --git a/pkg/cloudcommon/db/models.go b/pkg/cloudcommon/db/models.go index 7a37834f9f..7409a45ac9 100644 --- a/pkg/cloudcommon/db/models.go +++ b/pkg/cloudcommon/db/models.go @@ -34,14 +34,21 @@ func GlobalModelManagerTables() map[string]IModelManager { } func RegisterModelManager(modelMan IModelManager) { + RegisterModelManagerWithKeyword(modelMan, "") +} + +func RegisterModelManagerWithKeyword(modelMan IModelManager, keyword string) { if globalTables == nil { globalTables = make(map[string]IModelManager) } - mustCheckModelManager(modelMan) - if _, ok := globalTables[modelMan.Keyword()]; ok { - log.Fatalf("keyword %s exists in globalTables!", modelMan.Keyword()) + if len(keyword) == 0 { + keyword = modelMan.Keyword() } - globalTables[modelMan.Keyword()] = modelMan + mustCheckModelManager(modelMan) + if _, ok := globalTables[keyword]; ok { + log.Fatalf("keyword %s exists in globalTables!", keyword) + } + globalTables[keyword] = modelMan } func mustCheckModelManager(modelMan IModelManager) { diff --git a/pkg/logger/models/actionlog.go b/pkg/logger/models/actionlog.go index b660f6037a..78f920d94a 100644 --- a/pkg/logger/models/actionlog.go +++ b/pkg/logger/models/actionlog.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" "yunion.io/x/sqlchemy/backends/clickhouse" @@ -31,6 +32,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/logger/extern" + "yunion.io/x/onecloud/pkg/logger/options" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/logclient" ) @@ -72,44 +74,81 @@ type SActionlog struct { } var ActionLog *SActionlogManager +var AdminActionLog *SActionlogManager + var logQueue = make(chan *SActionlog, 50) func InitActionLog() { InitActionWhiteList() + var initTable func(tbname string) *SActionlogManager if consts.OpsLogWithClickhouse { - ActionLog = &SActionlogManager{ - SOpsLogManager: db.SOpsLogManager{ - SModelBaseManager: db.NewModelBaseManagerWithDBName( - SActionlog{}, - "action_tbl", - "action", - "actions", - db.ClickhouseDB, - ), - }, - } - col := ActionLog.TableSpec().ColumnSpec("ops_time") - if clickCol, ok := col.(clickhouse.IClickhouseColumnSpec); ok { - clickCol.SetTTL(consts.SplitableMaxKeepMonths(), "MONTH") + initTable = func(tbname string) *SActionlogManager { + tbl := &SActionlogManager{ + SOpsLogManager: db.SOpsLogManager{ + SModelBaseManager: db.NewModelBaseManagerWithDBName( + SActionlog{}, + tbname, + "action", + "actions", + db.ClickhouseDB, + ), + }, + } + col := tbl.TableSpec().ColumnSpec("ops_time") + if clickCol, ok := col.(clickhouse.IClickhouseColumnSpec); ok { + clickCol.SetTTL(consts.SplitableMaxKeepMonths(), "MONTH") + } + return tbl } } else { - ActionLog = &SActionlogManager{ - SOpsLogManager: db.SOpsLogManager{ - SModelBaseManager: db.NewModelBaseManagerWithSplitable( - SActionlog{}, - "action_tbl", - "action", - "actions", - "id", - "start_time", - consts.SplitableMaxDuration(), - consts.SplitableMaxKeepMonths(), - ), - }, - SRecordChecksumResourceBaseManager: *db.NewRecordChecksumResourceBaseManager(), + initTable = func(tbname string) *SActionlogManager { + tbl := &SActionlogManager{ + SOpsLogManager: db.SOpsLogManager{ + SModelBaseManager: db.NewModelBaseManagerWithSplitable( + SActionlog{}, + tbname, + "action", + "actions", + "id", + "start_time", + consts.SplitableMaxDuration(), + consts.SplitableMaxKeepMonths(), + ), + }, + SRecordChecksumResourceBaseManager: *db.NewRecordChecksumResourceBaseManager(), + } + return tbl } } + ActionLog = initTable("action_tbl") ActionLog.SetVirtualObject(ActionLog) + if options.Options.EnableSeparateAdminLog { + AdminActionLog = initTable("admin_action_tbl") + AdminActionLog.SetVirtualObject(AdminActionLog) + } +} + +func isRoleInNames(userCred mcclient.TokenCredential, roles []string) bool { + for _, r := range userCred.GetRoles() { + if utils.IsInStringArray(r, roles) { + return true + } + } + return false +} + +func (manager *SActionlogManager) GetImmutableInstance(userCred mcclient.TokenCredential) db.IModelManager { + if options.Options.EnableSeparateAdminLog && isRoleInNames(userCred, options.Options.AuditorRoleNames) { + return AdminActionLog + } + return ActionLog +} + +func (manager *SActionlogManager) GetMutableInstance(userCred mcclient.TokenCredential) db.IModelManager { + if options.Options.EnableSeparateAdminLog && (isRoleInNames(userCred, options.Options.SecadminRoleNames) || isRoleInNames(userCred, options.Options.OpsadminRoleNames)) { + return AdminActionLog + } + return ActionLog } func (action *SActionlog) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { diff --git a/pkg/logger/options/options.go b/pkg/logger/options/options.go index dc2090657c..50672fe326 100644 --- a/pkg/logger/options/options.go +++ b/pkg/logger/options/options.go @@ -25,6 +25,12 @@ type SLoggerOptions struct { common_options.DBOptions SyslogUrl string `help:"external syslog url, e.g. tcp://localhost:1234@cloud"` + + EnableSeparateAdminLog bool `help:"enable separate log for auditor admin" default:"false"` + + SecadminRoleNames []string `help:"role names of security admin" default:"sys_secadmin,domain_secadmin"` + OpsadminRoleNames []string `help:"role names of operation admin" default:"sys_opsadmin,domain_opsadmin"` + AuditorRoleNames []string `help:"role names of auditor admin" default:"sys_adtadmin,domain_adtadmin"` } var ( diff --git a/pkg/logger/service/handlers.go b/pkg/logger/service/handlers.go index e09fa1519c..bd08b459f5 100644 --- a/pkg/logger/service/handlers.go +++ b/pkg/logger/service/handlers.go @@ -19,6 +19,7 @@ import ( "yunion.io/x/onecloud/pkg/appsrv/dispatcher" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/logger/models" + "yunion.io/x/onecloud/pkg/logger/options" ) func initHandlers(app *appsrv.Application) { @@ -35,6 +36,10 @@ func initHandlers(app *appsrv.Application) { db.RegisterModelManager(manager) } + if options.Options.EnableSeparateAdminLog { + db.RegisterModelManagerWithKeyword(models.AdminActionLog, "admin-action") + } + for _, manager := range []db.IModelManager{ db.Metadata, diff --git a/pkg/util/splitable/insert.go b/pkg/util/splitable/insert.go index 600a9d89fd..a87db039ad 100644 --- a/pkg/util/splitable/insert.go +++ b/pkg/util/splitable/insert.go @@ -49,10 +49,11 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error { if !lastMeta.StartDate.IsZero() && lastDate.Sub(lastMeta.StartDate) > t.maxDuration { lastTable := t.GetTableSpec(lastMeta) ti := lastTable.Instance() - q := ti.Query(sqlchemy.MAX("last_index", ti.Field(t.indexField)), sqlchemy.MAX("last_date", ti.Field(t.dateField))) + q := ti.Query(sqlchemy.MAX("last_index", ti.Field(t.indexField)), sqlchemy.MAX("last_date", ti.Field(t.dateField)), sqlchemy.COUNT("total")) r := q.Row() var lastRecDateStr string - err := r.Scan(&lastRecIndex, &lastRecDateStr) + var total uint64 + err := r.Scan(&lastRecIndex, &lastRecDateStr, &total) if err != nil { return errors.Wrap(err, "scan lastRecIndex and lastRecDate") } @@ -62,6 +63,7 @@ func (t *SSplitTableSpec) Insert(dt interface{}) error { _, err = t.metaSpec.Update(&lastMeta, func() error { lastMeta.End = lastRecIndex lastMeta.EndDate = lastRecDate + lastMeta.Count = total return nil }) if err != nil { diff --git a/pkg/util/splitable/metadata.go b/pkg/util/splitable/metadata.go index 96c2883345..9f93d9d299 100644 --- a/pkg/util/splitable/metadata.go +++ b/pkg/util/splitable/metadata.go @@ -27,6 +27,7 @@ type STableMetadata struct { Table string `width:"64" charset:"ascii"` Start int64 `nullable:"true"` End int64 `nullable:"true"` + Count uint64 `nullable:"true"` StartDate time.Time `nullable:"true"` EndDate time.Time `nullable:"true"` Deleted bool `nullable:"false"` @@ -56,6 +57,20 @@ func (spec *SSplitTableSpec) GetTableMetas() ([]STableMetadata, error) { if err != nil && errors.Cause(err) != sql.ErrNoRows { return nil, errors.Wrap(err, "query metadata") } + for i := 0; i < len(metas); i++ { + if metas[i].Count <= 0 { + // fix count + tbl := spec.GetTableSpec(metas[i]).Instance() + cnt, err := tbl.Query().CountWithError() + if err != nil { + return nil, errors.Wrap(err, "CountWithError") + } + spec.metaSpec.Update(&metas[i], func() error { + metas[i].Count = uint64(cnt) + return nil + }) + } + } return metas, nil }