mirror of
https://gitee.com/samwaf/SamWaf.git
synced 2026-11-02 19:21:44 +08:00
* fix: make archived log shard connections concurrency-safe and recoverable * fix: build the WebSocket online table before its consumers start * perf: cap stored payloads, index log lookups by IP and rule, batch IP tag writes * fix: keep archived log shards readable when the live table gains a column * feat: merge IP tag history when their database location changes * refactor: move request payloads out of the access log into their own table * feat: scope log database export to a time range and selected tiers * feat: add an IP watchlist that full-captures watched IPs * feat: split log storage into security events, access rows, and payload tiers * feat: roll up requests per actor and path into daily analysis tables * fix:actor key by ip * feat: add a source and path analysis page over the daily rollups * feat: exclude chosen source IPs from web log recording * docs: add upgrade notes for the analysis page and log IP exclusion * feat: derive the log partition from the time range and look up identifiers across all of them * feat: add time partition verbs and the period naming they share * feat: cut log archives per month and expire them by dropping partitions * feat: manage log partitions, and answer why an IP has no logs * fix: read archived SQLite log shards read-only and choose tiers by data * fix: count the live log shard from access_log instead of a stale snapshot * fix:export db bug * fix: default backend keep-alive pool (512/128/90s) to stop connect-close churn under load * chore: gen_swagger.bat outputs json only, syncs doc site, runs route test * feat: pin web frontend version and embed it into the binary
165 lines
5.1 KiB
Go
165 lines
5.1 KiB
Go
//go:build crossdb
|
|
|
|
// 报文落库这一步的三库回归。splitForStore 的纯逻辑由 log_split_test.go 覆盖,
|
|
// 这里盯的是真正压到数据库上的那条语句:报文表以 req_uuid 为主键,
|
|
// 同一请求分两批入队就会撞主键,三个引擎的"冲突即跳过"写法各不相同,
|
|
// 撞坏了整批报文都写不进去(日志行还在,用户看到的是报文凭空消失)。
|
|
//
|
|
// go test -tags crossdb ./wafqueue/ -run TestStorePayloadsCrossEngine -v
|
|
package wafqueue
|
|
|
|
import (
|
|
"SamWaf/common/zlog"
|
|
"SamWaf/global"
|
|
"SamWaf/model"
|
|
"SamWaf/wafdb/dialect"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
sqlitedriver "github.com/samwafgo/sqlitedriver"
|
|
mysqldriver "gorm.io/driver/mysql"
|
|
pgdriver "gorm.io/driver/postgres"
|
|
"gorm.io/gorm"
|
|
"gorm.io/gorm/logger"
|
|
)
|
|
|
|
var silentCfg = &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}
|
|
|
|
type payloadEngine struct {
|
|
name string
|
|
setup func(t *testing.T) (*gorm.DB, func())
|
|
}
|
|
|
|
func payloadEngines() []payloadEngine {
|
|
return []payloadEngine{
|
|
{"sqlite", func(t *testing.T) (*gorm.DB, func()) {
|
|
dialect.Register(&dialect.SQLiteDialect{})
|
|
path := filepath.Join(t.TempDir(), "payload_test.db")
|
|
db, err := gorm.Open(sqlitedriver.Open(path), silentCfg)
|
|
if err != nil {
|
|
t.Logf("sqlite 打开失败(跳过): %v", err)
|
|
return nil, nil
|
|
}
|
|
return db, func() {
|
|
if s, e := db.DB(); e == nil {
|
|
s.Close()
|
|
}
|
|
}
|
|
}},
|
|
{"mysql", func(t *testing.T) (*gorm.DB, func()) {
|
|
dsn := os.Getenv("SAMWAF_TEST_MYSQL_DSN")
|
|
if dsn == "" {
|
|
dsn = "root:canteen1@tcp(127.0.0.1:3306)/"
|
|
}
|
|
name := fmt.Sprintf("samwaf_payload_%d", time.Now().UnixNano()%100000)
|
|
root, err := gorm.Open(mysqldriver.Open(dsn+"?charset=utf8mb4&parseTime=True&loc=Local"), silentCfg)
|
|
if err != nil {
|
|
t.Logf("mysql 连接失败(跳过): %v", err)
|
|
return nil, nil
|
|
}
|
|
if err := root.Exec("CREATE DATABASE " + name + " DEFAULT CHARACTER SET utf8mb4").Error; err != nil {
|
|
t.Logf("mysql 建库失败(跳过): %v", err)
|
|
return nil, nil
|
|
}
|
|
dialect.Register(&dialect.MySQLDialect{})
|
|
db, err := gorm.Open(mysqldriver.Open(dsn+name+"?charset=utf8mb4&parseTime=True&loc=Local"), silentCfg)
|
|
if err != nil {
|
|
t.Fatalf("mysql 打开 %s: %v", name, err)
|
|
}
|
|
return db, func() {
|
|
if s, e := db.DB(); e == nil {
|
|
s.Close()
|
|
}
|
|
root.Exec("DROP DATABASE IF EXISTS " + name)
|
|
if s, e := root.DB(); e == nil {
|
|
s.Close()
|
|
}
|
|
}
|
|
}},
|
|
{"postgres", func(t *testing.T) (*gorm.DB, func()) {
|
|
base := os.Getenv("SAMWAF_TEST_PG_DSN")
|
|
if base == "" {
|
|
base = "postgres://postgres:postgres@127.0.0.1:5432/"
|
|
}
|
|
name := fmt.Sprintf("samwaf_payload_%d", time.Now().UnixNano()%100000)
|
|
root, err := gorm.Open(pgdriver.Open(base+"postgres?sslmode=disable"), silentCfg)
|
|
if err != nil {
|
|
t.Logf("postgres 连接失败(跳过): %v", err)
|
|
return nil, nil
|
|
}
|
|
if err := root.Exec("CREATE DATABASE " + name + " ENCODING 'UTF8'").Error; err != nil {
|
|
t.Logf("postgres 建库失败(跳过): %v", err)
|
|
return nil, nil
|
|
}
|
|
dialect.Register(&dialect.PostgresDialect{})
|
|
db, err := gorm.Open(pgdriver.Open(base+name+"?sslmode=disable&TimeZone=Asia/Shanghai"), silentCfg)
|
|
if err != nil {
|
|
t.Fatalf("postgres 打开 %s: %v", name, err)
|
|
}
|
|
return db, func() {
|
|
if s, e := db.DB(); e == nil {
|
|
s.Close()
|
|
}
|
|
root.Exec("DROP DATABASE IF EXISTS " + name)
|
|
if s, e := root.DB(); e == nil {
|
|
s.Close()
|
|
}
|
|
}
|
|
}},
|
|
}
|
|
}
|
|
|
|
func TestStorePayloadsCrossEngine(t *testing.T) {
|
|
zlog.InitZLog(false, "console")
|
|
saved := global.GWAF_LOCAL_LOG_DB
|
|
defer func() { global.GWAF_LOCAL_LOG_DB = saved }()
|
|
|
|
for _, e := range payloadEngines() {
|
|
e := e
|
|
t.Run(e.name, func(t *testing.T) {
|
|
db, teardown := e.setup(t)
|
|
if db == nil {
|
|
t.Skipf("%s 未就绪,跳过", e.name)
|
|
return
|
|
}
|
|
defer teardown()
|
|
if err := db.AutoMigrate(&model.EventPayload{}); err != nil {
|
|
t.Fatalf("建 event_payload 失败: %v", err)
|
|
}
|
|
global.GWAF_LOCAL_LOG_DB = db
|
|
|
|
rows := []*model.EventPayload{
|
|
{ReqUUID: "u1", Kind: PayloadKindEvent, BODY: "a=1", CreateTime: "2026-09-17 10:00:00"},
|
|
{ReqUUID: "u2", Kind: PayloadKindEvent, BODY: "b=2", CreateTime: "2026-09-17 10:00:00"},
|
|
}
|
|
storePayloads(rows)
|
|
|
|
var n int64
|
|
db.Model(&model.EventPayload{}).Count(&n)
|
|
if n != 2 {
|
|
t.Fatalf("首次写入应有 2 行,实际 %d", n)
|
|
}
|
|
|
|
// 同一请求分两批入队:主键撞上必须整批跳过而不是整批失败
|
|
again := []*model.EventPayload{
|
|
{ReqUUID: "u1", Kind: PayloadKindEvent, BODY: "a=1-late", CreateTime: "2026-09-17 10:00:01"},
|
|
{ReqUUID: "u3", Kind: PayloadKindEvent, BODY: "c=3", CreateTime: "2026-09-17 10:00:01"},
|
|
}
|
|
storePayloads(again)
|
|
|
|
db.Model(&model.EventPayload{}).Count(&n)
|
|
if n != 3 {
|
|
t.Fatalf("撞主键应跳过冲突行、写入新行,期望 3 行实际 %d(整批失败了?)", n)
|
|
}
|
|
var got model.EventPayload
|
|
db.Where("req_uuid = ?", "u1").First(&got)
|
|
if got.BODY != "a=1" {
|
|
t.Fatalf("冲突行应保留先到的那条,实际 %q", got.BODY)
|
|
}
|
|
})
|
|
}
|
|
}
|