fix: fix the issue of MySql table partitioning not take effect

#IJULAW
This commit is contained in:
samwaf
2026-06-16 11:16:32 +08:00
parent 9366e0962b
commit 39dffafab6
12 changed files with 230 additions and 44 deletions
+9
View File
@@ -9,6 +9,7 @@ import (
response2 "SamWaf/model/response"
"SamWaf/utils"
"SamWaf/wafdb"
"SamWaf/wafdb/dialect"
"fmt"
"net/http"
"net/url"
@@ -88,6 +89,12 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) {
}
}
func (w *WafLogAPi) ExportDBApi(c *gin.Context) {
// 该导出走 BackupDatabase 备份 .db 文件,仅文件型数据库(SQLite)支持;
// MySQL 等无日志文件,直接屏蔽,避免进入后台 goroutine 后才失败。
if !dialect.Get().SupportsBackup() {
response.FailWithMessage("当前数据库不支持日志文件导出(仅 SQLite 支持)", c)
return
}
if global.GWAF_CAN_EXPORT_DOWNLOAD_LOG == false {
// 使用操作结果消息格式
serverName := global.GWAF_CUSTOM_SERVER_NAME
@@ -208,6 +215,7 @@ func (w *WafLogAPi) GetListByHostCodeApi(c *gin.Context) {
func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) {
wafShareList, _ := wafShareDbService.GetAllShareDbApi()
liveName := wafdb.LiveLogName() // 当前驱动的实时分片标识,用于标记默认选中项
allShareDbRep := make([]response2.AllShareDbRep, len(wafShareList)) // 创建数组
for i, _ := range wafShareList {
@@ -216,6 +224,7 @@ func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) {
EndTime: wafShareList[i].EndTime,
FileName: wafShareList[i].FileName,
Cnt: wafShareList[i].Cnt,
IsCurrent: wafShareList[i].FileName == liveName,
}
}
+14
View File
@@ -171,8 +171,22 @@ func (w *WafSysInfoApi) CheckVersionApi(c *gin.Context) {
// SystemParamsApi 返回认证后才能获取的系统参数(可扩展)
// GET /api/v1/sysinfo/systemparams
func (w *WafSysInfoApi) SystemParamsApi(c *gin.Context) {
// 当前数据库(sqlite|mysql),mysql 额外给出连接目标(不含密码)
database := gin.H{"driver": global.GWAF_DB_DRIVER}
if global.GWAF_DB_DRIVER == "mysql" {
database["host"] = global.GWAF_MYSQL_HOST
database["port"] = global.GWAF_MYSQL_PORT
}
// 当前缓存(memory|redis),redis 额外给出连接目标(不含密码)
cache := gin.H{"type": global.GCACHE_TYPE}
if global.GCACHE_TYPE == "redis" {
cache["host"] = global.GCACHE_REDIS_HOST
cache["port"] = global.GCACHE_REDIS_PORT
}
response.OkWithDetailed(gin.H{
"emergency_path": "/" + global.GWAF_SECURITY_EMERGENCY_PATH,
"database": database,
"cache": cache,
}, "获取成功", c)
}
+5 -3
View File
@@ -3,9 +3,11 @@ package request
import "SamWaf/model/common/request"
type WafAttackLogDetailReq struct {
CurrrentDbName string `json:"current_db_name"`
REQ_UUID string `json:"req_uuid"`
OutputFormat string `json:"output_format"` //输出格式 raw,curl
// detail/httpcopymask 为 GET 请求,需 form tag 才能从 query 绑定;
// 前端发送的 query 键为 current_db_name / REQ_UUID / output_format
CurrrentDbName string `json:"current_db_name" form:"current_db_name"`
REQ_UUID string `json:"req_uuid" form:"REQ_UUID"`
OutputFormat string `json:"output_format" form:"output_format"` //输出格式 raw,curl
}
type WafAttackLogDoExport struct {
+1
View File
@@ -13,6 +13,7 @@ type AllShareDbRep struct {
EndTime customtype.JsonTime `json:"end_time"` //结束时间
FileName string `json:"file_name"` //文件名
Cnt int64 `json:"cnt"` //当前数量
IsCurrent bool `json:"is_current"` //是否为当前(实时)分片:前端据此设默认选中项
}
// AllDomainRep 域名信息
+10 -19
View File
@@ -80,13 +80,9 @@ func (receiver *WafLogService) ModifyApi(log innerbean.WebLog) error {
}
func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) (innerbean.WebLog, error) {
var weblog innerbean.WebLog
if len(req.CurrrentDbName) == 0 || req.CurrrentDbName == "local_log.db" {
global.GWAF_LOCAL_LOG_DB.Select(getWebLogDetailSelect()).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog)
} else {
wafdb.InitManaulLogDb("", req.CurrrentDbName)
global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Select(getWebLogDetailSelect()).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog)
}
// 解析当前应查询的日志连接与表(live 或历史分片:SQLite 历史文件 / MySQL 历史表)
logDB, logTable := wafdb.ResolveLogDB(req.CurrrentDbName)
logDB.Table(logTable).Select(getWebLogDetailSelect()).Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog)
return weblog, nil
}
func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) {
@@ -95,8 +91,10 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn
splitFilterBys := strings.Split(req.FilterBy, "|")
splitFilterValues := strings.Split(req.FilterValue, "|")
// 解析当前应查询的日志连接与表(live 或历史分片:SQLite 历史文件 / MySQL 历史表)
logDB, logTable := wafdb.ResolveLogDB(req.CurrrentDbName)
/*强制索引*/
var forceIndex = "web_logs"
var forceIndex = logTable
/*where条件*/
var whereField = ""
var whereValues []interface{}
@@ -171,9 +169,9 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn
//强制索引
{
if strings.Contains(whereField, "unix_add_time") && !strings.Contains(whereField, "src_ip") {
forceIndex = dialect.Get().ForceIndexClause("web_logs", "idx_web_time_desc_tenant_user_code")
forceIndex = dialect.Get().ForceIndexClause(logTable, "idx_web_time_desc_tenant_user_code")
} else if strings.Contains(whereField, "src_ip") {
forceIndex = dialect.Get().ForceIndexClause("web_logs", "idx_web_time_desc_tenant_user_code_ip")
forceIndex = dialect.Get().ForceIndexClause(logTable, "idx_web_time_desc_tenant_user_code_ip")
}
}
@@ -239,15 +237,8 @@ func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]inn
} else {
return nil, 0, errors.New("输入排序字段不合法")
}
if len(req.CurrrentDbName) == 0 || req.CurrrentDbName == "local_log.db" {
global.GWAF_LOCAL_LOG_DB.Select(getWebLogListSelect()).Table(forceIndex).Limit(req.PageSize).Where(whereField, whereValues...).Offset(req.PageSize * (req.PageIndex - 1)).Order(orderInfo).Find(&weblogs)
global.GWAF_LOCAL_LOG_DB.Table(forceIndex).Where(whereField, whereValues...).Count(&total)
} else {
wafdb.InitManaulLogDb("", req.CurrrentDbName)
global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Select(getWebLogListSelect()).Table(forceIndex).Limit(req.PageSize).Where(whereField, whereValues...).Offset(req.PageSize * (req.PageIndex - 1)).Order(orderInfo).Find(&weblogs)
global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Table(forceIndex).Where(whereField, whereValues...).Count(&total)
}
logDB.Select(getWebLogListSelect()).Table(forceIndex).Limit(req.PageSize).Where(whereField, whereValues...).Offset(req.PageSize * (req.PageIndex - 1)).Order(orderInfo).Find(&weblogs)
logDB.Table(forceIndex).Where(whereField, whereValues...).Count(&total)
return weblogs, total, nil
}
func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) {
+16 -3
View File
@@ -4,6 +4,7 @@ import (
"SamWaf/global"
"SamWaf/model"
"SamWaf/model/request"
"SamWaf/wafdb/dialect"
)
type WafShareDbService struct{}
@@ -33,10 +34,22 @@ func (receiver *WafShareDbService) GetListApi(req request.WafShareDbReq) ([]mode
return list, total, nil
}
// 获取所有db
// 获取所有db(按当前驱动过滤分片类型)
// 历史遗留:share_dbs 表会同时存在 SQLite 文件分片(local_log*.db)与 MySQL 表分片(web_logs*)
// 用户切换驱动后另一种分片对当前库无意义,这里按 ShareDb.IsTableShard() 与当前驱动是否文件型过滤:
// - SQLite(文件型) 只保留文件分片(.db)
// - MySQL(非文件型) 只保留表分片(无 .db 后缀)
func (receiver *WafShareDbService) GetAllShareDbApi() ([]model.ShareDb, error) {
var list []model.ShareDb
global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Find(&list)
var all []model.ShareDb
global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Find(&all)
fileBased := dialect.Get().IsFileBased()
list := make([]model.ShareDb, 0, len(all))
for _, s := range all {
// 文件型驱动保留非表分片(.db),非文件型驱动保留表分片
if s.IsTableShard() != fileBased {
list = append(list, s)
}
}
return list, nil
}
+13
View File
@@ -96,6 +96,19 @@ type DBDialect interface {
// for server databases (MySQL: RENAME TABLE; MSSQL: sp_rename).
RenameTable(db *gorm.DB, src, dst string) error
// ShardSwapTable atomically archives liveTable to archiveTable and leaves an
// empty liveTable in place (same structure + indexes), used for log table
// sharding on server databases.
// MySQL: CREATE TABLE <live>_tmp LIKE <live>; RENAME TABLE <live> TO <archive>, <live>_tmp TO <live>
// File-based databases (SQLite) shard via OS-level file rename instead and
// return an error here ("not supported").
ShardSwapTable(db *gorm.DB, liveTable, archiveTable string) error
// TableSizeMB returns the on-disk size (data + index) of a table in MB,
// used by the sharding task to detect the size threshold on server databases.
// File-based databases (SQLite) return 0 (the caller uses the file size instead).
TableSizeMB(db *gorm.DB, table string) (int64, error)
// ListTables returns all user-defined table names in the current schema.
ListTables(db *gorm.DB) ([]string, error)
+41 -1
View File
@@ -34,7 +34,47 @@ func (d *MySQLDialect) FormatTimeWithOffset(colExpr string, offsetMin int) strin
}
func (d *MySQLDialect) RenameTable(db *gorm.DB, src, dst string) error {
return db.Exec(fmt.Sprintf("RENAME TABLE `%s` TO `%s`", src, dst)).Error
return db.Exec(fmt.Sprintf("RENAME TABLE %s TO %s", mysqlQuote(src), mysqlQuote(dst))).Error
}
// ShardSwapTable archives liveTable to archiveTable and leaves an empty liveTable.
// It first creates an empty clone of the live table (structure + all indexes via
// CREATE TABLE ... LIKE), then atomically swaps the two names in a single
// RENAME TABLE statement so there is no window where the live table is missing
// (avoids losing concurrent log inserts).
func (d *MySQLDialect) ShardSwapTable(db *gorm.DB, liveTable, archiveTable string) error {
tmpTable := liveTable + "_shardtmp"
// Clean up any leftover temp table from a previously failed attempt.
if err := db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", mysqlQuote(tmpTable))).Error; err != nil {
return fmt.Errorf("mysql: 清理临时分表 %s 失败: %w", tmpTable, err)
}
// Create an empty clone with identical structure and indexes.
if err := db.Exec(fmt.Sprintf("CREATE TABLE %s LIKE %s", mysqlQuote(tmpTable), mysqlQuote(liveTable))).Error; err != nil {
return fmt.Errorf("mysql: 创建临时分表 %s 失败: %w", tmpTable, err)
}
// Atomic swap: live → archive, tmp → live (single statement).
swapSQL := fmt.Sprintf("RENAME TABLE %s TO %s, %s TO %s",
mysqlQuote(liveTable), mysqlQuote(archiveTable),
mysqlQuote(tmpTable), mysqlQuote(liveTable))
if err := db.Exec(swapSQL).Error; err != nil {
// Best-effort cleanup so a retry can succeed.
db.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s", mysqlQuote(tmpTable)))
return fmt.Errorf("mysql: 原子换表失败 (%s→%s): %w", liveTable, archiveTable, err)
}
return nil
}
// TableSizeMB returns (DATA_LENGTH + INDEX_LENGTH) of the table in MB.
func (d *MySQLDialect) TableSizeMB(db *gorm.DB, table string) (int64, error) {
var sizeMB int64
err := db.Raw(
"SELECT COALESCE((DATA_LENGTH + INDEX_LENGTH), 0) DIV (1024*1024) "+
"FROM information_schema.TABLES "+
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?",
table,
).Scan(&sizeMB).Error
return sizeMB, err
}
func (d *MySQLDialect) ListTables(db *gorm.DB) ([]string, error) {
+12
View File
@@ -36,6 +36,18 @@ func (d *SQLiteDialect) RenameTable(db *gorm.DB, src, dst string) error {
return db.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s", sqliteQuote(src), sqliteQuote(dst))).Error
}
// ShardSwapTable is not used for SQLite: sharding is done by renaming the whole
// .db file at the OS level (see waftask.TaskShareDbInfo). Returns an error to
// guard against accidental use.
func (d *SQLiteDialect) ShardSwapTable(db *gorm.DB, liveTable, archiveTable string) error {
return fmt.Errorf("ShardSwapTable 不适用于 SQLite(按文件改名分库),当前驱动: sqlite")
}
// TableSizeMB returns 0 for SQLite; the caller uses the .db file size instead.
func (d *SQLiteDialect) TableSizeMB(db *gorm.DB, table string) (int64, error) {
return 0, nil
}
func (d *SQLiteDialect) ListTables(db *gorm.DB) ([]string, error) {
rows, err := db.Raw(
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name",
+61
View File
@@ -0,0 +1,61 @@
package wafdb
import (
"SamWaf/enums"
"SamWaf/global"
"SamWaf/wafdb/dialect"
"gorm.io/gorm"
)
// LogTableName is the live web access log table name (model innerbean.WebLog).
const LogTableName = "web_logs"
// LiveLogName returns the ShareDb.FileName that identifies the current/live log
// store for the active driver: SQLite → "local_log.db", others (MySQL) → "web_logs".
// Used to mark the default selection in the front-end archive dropdown.
func LiveLogName() string {
if dialect.Get().IsFileBased() {
return enums.DB_LOG // "local_log.db"
}
return LogTableName // "web_logs"
}
// ResolveLogDB returns the *gorm.DB connection and table name to query for the
// given log shard identifier (ShareDb.FileName, passed from the front-end as
// current_db_name).
//
// - empty / live identifier → live log DB + "web_logs"
// - SQLite historical shard → on-demand opened shard .db file + "web_logs"
// - MySQL historical shard → same log DB connection + shard table name
//
// It never returns a nil *gorm.DB: if a SQLite shard cannot be opened it falls
// back to the live connection, guarding against the nil-map dereference panic
// that the previous inline read paths were exposed to under MySQL.
func ResolveLogDB(currentDbName string) (*gorm.DB, string) {
// Treat as "live" (current log store): the empty value, the legacy default
// "local_log.db" (still sent by the front-end as its default selection under
// any driver), and the MySQL live table name "web_logs".
if len(currentDbName) == 0 || currentDbName == enums.DB_LOG || currentDbName == LogTableName {
return global.GWAF_LOCAL_LOG_DB, LogTableName
}
// Historical shard.
if dialect.Get().IsFileBased() {
// SQLite: open the archived .db file on demand and query its web_logs table.
InitManaulLogDb("", currentDbName)
if db := global.GDATA_CURRENT_LOG_DB_MAP[currentDbName]; db != nil {
return db, LogTableName
}
// Shard file unavailable — degrade to live DB instead of panicking.
return global.GWAF_LOCAL_LOG_DB, LogTableName
}
// MySQL: the archived shard is a table (web_logs_<ts>) in the same database.
// Guard against a non-existent table name (e.g. stale share_dbs rows that
// stored the database name instead of a table name) by falling back to live.
if dialect.Get().TableExists(global.GWAF_LOCAL_LOG_DB, currentDbName) {
return global.GWAF_LOCAL_LOG_DB, currentDbName
}
return global.GWAF_LOCAL_LOG_DB, LogTableName
}
+9 -6
View File
@@ -269,10 +269,11 @@ func InitLogDbMySQL() (bool, error) {
pathLogSql(db)
// Ensure a ShareDb entry exists for the MySQL log database.
var total int64
global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Count(&total)
if total == 0 {
// 确保存在一条 live 分片记录(web_logs)。幂等:仅当该记录不存在时创建。
// 不能用 share_dbs 总数判断——从 SQLite 迁移过来时表里已有 .db 历史分片,总数!=0 会导致 live 记录缺失。
var liveCount int64
global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Where("file_name = ?", "web_logs").Count(&liveCount)
if liveCount == 0 {
var logTotal int64
global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Count(&logTotal)
@@ -287,8 +288,10 @@ func InitLogDbMySQL() (bool, error) {
DbLogicType: "log",
StartTime: customtype.JsonTime(time.Now()),
EndTime: customtype.JsonTime(time.Now()),
FileName: dbName, // MySQL DB name (no .db extension)
Cnt: logTotal,
// live 分片标识用 web_logs 表名(与 ResolveLogDB 的 live 判定一致);
// 历史分片由分表任务写入 web_logs_<ts> 表名。不能用库名,否则读取会误入历史分支。
FileName: "web_logs",
Cnt: logTotal,
}
global.GWAF_LOCAL_DB.Create(sharDbBean)
}
+39 -12
View File
@@ -50,23 +50,41 @@ func TaskShareDbInfo() {
shardingReason = fmt.Sprintf("记录数量(%d)超过限制(%d)", total, global.GDATA_SHARE_DB_SIZE)
}
// 检查文件大小是否超过限制
fileInfo, err := os.Stat(dbFilePath)
if err == nil {
fileSizeMB := fileInfo.Size() / (1024 * 1024) // 转换为MB
if fileSizeMB > global.GDATA_SHARE_DB_FILE_SIZE {
needSharding = true
shardingReason = fmt.Sprintf("文件大小(%dMB)超过限制(%dMB)", fileSizeMB, global.GDATA_SHARE_DB_FILE_SIZE)
// 检查大小是否超过限制SQLite 用 .db 文件大小,MySQL 用 web_logs 表(数据+索引)大小
if dialect.Get().IsFileBased() {
fileInfo, err := os.Stat(dbFilePath)
if err == nil {
fileSizeMB := fileInfo.Size() / (1024 * 1024) // 转换为MB
if fileSizeMB > global.GDATA_SHARE_DB_FILE_SIZE {
needSharding = true
shardingReason = fmt.Sprintf("文件大小(%dMB)超过限制(%dMB)", fileSizeMB, global.GDATA_SHARE_DB_FILE_SIZE)
}
} else {
zlog.Error(innerLogName, "获取数据库文件大小失败:", err)
}
} else {
zlog.Error(innerLogName, "获取数据库文件大小失败:", err)
tableSizeMB, err := dialect.Get().TableSizeMB(global.GWAF_LOCAL_LOG_DB, "web_logs")
if err == nil {
if tableSizeMB > global.GDATA_SHARE_DB_FILE_SIZE {
needSharding = true
shardingReason = fmt.Sprintf("表大小(%dMB)超过限制(%dMB)", tableSizeMB, global.GDATA_SHARE_DB_FILE_SIZE)
}
} else {
zlog.Error(innerLogName, "获取web_logs表大小失败:", err)
}
}
if needSharding {
global.GDATA_CURRENT_CHANGE = true
zlog.Info(innerLogName, "开始分库,原因:", shardingReason)
newDBFilename := fmt.Sprintf("local_log_%v.db", time.Now().Format("20060102150405"))
ts := time.Now().Format("20060102150405")
newDBFilename := fmt.Sprintf("local_log_%v.db", ts)
// 归档标识:SQLite 为新文件名(.db)MySQL 为归档表名(web_logs_<ts>)
archiveName := newDBFilename
if !dialect.Get().IsFileBased() {
archiveName = fmt.Sprintf("web_logs_%v", ts)
}
var lastedDb model.ShareDb
err := global.GWAF_LOCAL_DB.Limit(1).Order("create_time desc").Find(&lastedDb).Error
@@ -85,7 +103,7 @@ func TaskShareDbInfo() {
DbLogicType: "log",
StartTime: startTime,
EndTime: customtype.JsonTime(time.Now()),
FileName: newDBFilename,
FileName: archiveName,
Cnt: total,
}
@@ -127,8 +145,17 @@ func TaskShareDbInfo() {
global.GWAF_LOCAL_LOG_DB = nil
wafdb.InitLogDb("")
} else {
// MySQL / SQL ServerRENAME TABLE + AutoMigrate 重建空表(Milestone 2 实现)
zlog.Warn(innerLogName, "当前数据库驱动暂不支持自动分库,跳过:", dialect.Get().Name())
// MySQLCREATE TABLE LIKE + 单语句原子 RENAME 换表。
// 把 web_logs 归档为 archiveName 并重建同结构空表,换表期间无写入空窗,
// 同一连接、表已重建为空,无需像 SQLite 那样置 nil 重连。
// 注意:归档表(web_logs_<ts>)不再被 gormigrate 跟踪,今后给 web_logs 加列时
// 需同步 ALTER 历史分表,否则读旧分片可能缺列(见 ResolveLogDB 注释)。
if err := dialect.Get().ShardSwapTable(global.GWAF_LOCAL_LOG_DB, "web_logs", archiveName); err != nil {
zlog.Error(innerLogName, "分表失败:", err)
} else {
global.GWAF_LOCAL_DB.Create(sharDbBean)
zlog.Info(innerLogName, "分表完成,归档表:", archiveName)
}
}
global.GDATA_CURRENT_CHANGE = false
zlog.Info(innerLogName, "切库完成...")