Merge pull request #301 from samwafgo/feat_291_maxlog

feat:add max log db file sharding

global/global.go

@@ -129,6 +129,7 @@ var (
 	/******数据库处理参数*****/
 	GDATA_BATCH_INSERT       int                 = 1000        //最大批量插入
 	GDATA_SHARE_DB_SIZE      int64               = 100 * 10000 //100w 进行分库 100*10000
+	GDATA_SHARE_DB_FILE_SIZE int64               = 1024        //1024M 进行分库
 	GDATA_CURRENT_CHANGE     bool                = false       //当前是否正在切换
 	GDATA_CURRENT_LOG_DB_MAP map[string]*gorm.DB               //当前自定义的数据库连接 TODO 如果用户打开了多个 会不会影响内存
 	/******WebSocket*********/

waftask/task_config.go

@@ -25,6 +25,9 @@ func setConfigIntValue(name string, value int64, change int) {
 	case "log_db_size":
 		global.GDATA_SHARE_DB_SIZE = value
 		break
+	case "db_file_size":
+		global.GDATA_SHARE_DB_FILE_SIZE = value
+		break
 	case "auto_load_ssl_file":
 		global.GCONFIG_RECORD_AUTO_LOAD_SSL = value
 		break
@@ -185,6 +188,7 @@ func TaskLoadSetting(initLoad bool) {
 	updateConfigIntItem(initLoad, "system", "record_resp", global.GCONFIG_RECORD_RESP, "是否记录响应报文", "int", "")
 	updateConfigIntItem(initLoad, "system", "delete_history_log_day", global.GDATA_DELETE_INTERVAL, "删除多少天前的日志数据(单位:天)", "int", "")
 	updateConfigIntItem(initLoad, "system", "log_db_size", global.GDATA_SHARE_DB_SIZE, "日志归档最大记录数量", "int", "")
+	updateConfigIntItem(initLoad, "system", "db_file_size", global.GDATA_SHARE_DB_FILE_SIZE, "日志归档最大文件大小(MB)", "int", "")
 	updateConfigIntItem(initLoad, "system", "auto_load_ssl_file", global.GCONFIG_RECORD_AUTO_LOAD_SSL, "是否每天凌晨3点自动加载ssl证书", "int", "")
 
 	updateConfigStringItem(initLoad, "system", "dns_server", global.GWAF_RUNTIME_DNS_SERVER, "DNS服务器", "options", "119.29.29.29|腾讯DNS,8.8.8.8|谷歌DNS")

waftask/task_db_sharding.go

@@ -30,12 +30,41 @@ func TaskShareDbInfo() {
 		zlog.Debug(innerLogName, "数据库没有初始化完成呢")
 		return
 	}
+
 	//获取当前日志数量
 	var total int64 = 0
 	global.GWAF_LOCAL_LOG_DB.Table("web_logs INDEXED BY  idx_tenant_usercode_web_logs").Count(&total)
+
+	//获取当前数据库文件大小
+	currentDir := utils.GetCurrentDir()
+	oldDBFilename := "local_log.db"
+	dbFilePath := currentDir + "/data/" + oldDBFilename
+
+	needSharding := false
+	var shardingReason string
+
+	// 检查记录数量是否超过限制
 	if total > global.GDATA_SHARE_DB_SIZE {
+		needSharding = true
+		shardingReason = fmt.Sprintf("记录数量(%d)超过限制(%d)", total, global.GDATA_SHARE_DB_SIZE)
+	}
+
+	// 检查文件大小是否超过限制
+	fileInfo, err := os.Stat(dbFilePath)
+	if err == nil {
+		fileSizeMB := fileInfo.Size() / (1024 * 1024) // 转换为MB
+		if fileSizeMB > global.GDATA_SHARE_DB_FILE_SIZE {
+			needSharding = true
+			shardingReason = fmt.Sprintf("文件大小(%dMB)超过限制(%dMB)", fileSizeMB, global.GDATA_SHARE_DB_FILE_SIZE)
+		}
+	} else {
+		zlog.Error(innerLogName, "获取数据库文件大小失败:", err)
+	}
+
+	if needSharding {
 		global.GDATA_CURRENT_CHANGE = true
-		oldDBFilename := "local_log.db"
+		zlog.Info(innerLogName, "开始分库，原因:", shardingReason)
+
 		newDBFilename := fmt.Sprintf("local_log_%v.db", time.Now().Format("20060102150405"))
 
 		var lastedDb model.ShareDb
@@ -105,6 +134,7 @@ func TaskShareDbInfo() {
 		global.GWAF_LOCAL_DB.Create(sharDbBean)
 		global.GWAF_LOCAL_LOG_DB = nil
 		wafdb.InitLogDb("")
+		createLogDbIndex() //重新创建索引
 		global.GDATA_CURRENT_CHANGE = false
 		zlog.Info(innerLogName, "切库完成...")
 	}