diff --git a/api/waf_ip_lookup.go b/api/waf_ip_lookup.go index 3d70893..abab13c 100644 --- a/api/waf_ip_lookup.go +++ b/api/waf_ip_lookup.go @@ -11,10 +11,10 @@ import ( type WafIPLookupApi struct { } -// LookupApi IP归属查询 -// @Summary IP归属查询 +// LookupApi IP策略命中查询 +// @Summary IP策略命中查询 // @Description 查一个IP当前落在哪些名单里:IP黑/白名单、IP组、威胁情报IP、IP失败封禁、CC封禁、系统防火墙封禁、CDN回源段 -// @Tags 网站防护-IP归属查询 +// @Tags 网站防护-IP策略命中查询 // @Produce json // @Param ip query string true "待查询的IP" // @Param sources query string false "只查指定来源(逗号分隔),留空查全部" diff --git a/api/waf_log.go b/api/waf_log.go index 6e8591a..1aca1b0 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -26,6 +26,19 @@ import ( type WafLogAPi struct { } +var logRequestIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$`) + +func normalizeLogRequestID(value string, required bool) (string, error) { + value = strings.TrimSpace(value) + if value == "" && !required { + return "", nil + } + if !logRequestIDPattern.MatchString(value) { + return "", fmt.Errorf("访问识别码格式不正确(需为 1–64 位字母、数字或分隔符)") + } + return value, nil +} + // GetDetailApi 获取攻击日志详情 // @Summary 获取攻击日志详情 // @Description 根据 req_uuid 获取单条攻击日志详情(含请求体、响应体) @@ -47,7 +60,16 @@ func (w *WafLogAPi) GetDetailApi(c *gin.Context) { response.FailWithMessage("正在切换数据库请等待", c) return } - wafLog, _ := wafLogService.GetDetailApi(req) + req.REQ_UUID, err = normalizeLogRequestID(req.REQ_UUID, true) + if err != nil { + response.FailWithMessage(err.Error(), c) + return + } + wafLog, err := wafLogService.GetDetailApi(req) + if err != nil { + response.FailWithMessage("获取日志详情失败:"+err.Error(), c) + return + } response.OkWithDetailed(wafLog, "获取成功", c) } else { response.FailWithMessage("解析失败", c) @@ -73,6 +95,11 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) { response.FailWithMessage("正在切换数据库请等待", c) return } + req.ReqUuid, err = normalizeLogRequestID(req.ReqUuid, false) + if err != nil { + response.FailWithMessage(err.Error(), c) + return + } wafLogs, total, meta, err2 := wafLogService.GetListApiWithMeta(req) if err2 != nil { response.FailWithMessage("访问列表失败:"+err2.Error(), c) @@ -317,7 +344,16 @@ func (w *WafLogAPi) GetHttpCopyMaskApi(c *gin.Context) { response.FailWithMessage("正在切换数据库请等待", c) return } - wafLog, _ := wafLogService.GetDetailApi(req) + req.REQ_UUID, err = normalizeLogRequestID(req.REQ_UUID, true) + if err != nil { + response.FailWithMessage(err.Error(), c) + return + } + wafLog, err := wafLogService.GetDetailApi(req) + if err != nil { + response.FailWithMessage("获取日志详情失败:"+err.Error(), c) + return + } if req.OutputFormat == "curl" { response.OkWithDetailed(GenerateCurlRequest(wafLog), "获取成功", c) diff --git a/api/waf_log_test.go b/api/waf_log_test.go new file mode 100644 index 0000000..9d3a4dc --- /dev/null +++ b/api/waf_log_test.go @@ -0,0 +1,172 @@ +package api + +import ( + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/common/response" + "SamWaf/service/waf_service" + "SamWaf/wafdb/dialect" + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +func setupLogAPITest(t *testing.T) *gorm.DB { + t.Helper() + priorDialect := func() (d dialect.DBDialect) { + defer func() { _ = recover() }() + return dialect.Get() + }() + dialect.Register(&dialect.SQLiteDialect{}) + db := openNotifyTestDB(t) + if err := db.AutoMigrate(&model.ShareDb{}, &model.AccessLog{}, &model.SecurityEvent{}, &model.EventPayload{}, &innerbean.WebLog{}); err != nil { + t.Fatal(err) + } + oldCore, oldLog := global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB + oldTenant, oldUser, oldChanging := global.GWAF_TENANT_ID, global.GWAF_USER_CODE, global.GDATA_CURRENT_CHANGE + global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB = db, db + global.GWAF_TENANT_ID, global.GWAF_USER_CODE, global.GDATA_CURRENT_CHANGE = "api-log-tenant", "api-log-user", false + waf_service.InvalidateShardCounts() + t.Cleanup(func() { + global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB = oldCore, oldLog + global.GWAF_TENANT_ID, global.GWAF_USER_CODE, global.GDATA_CURRENT_CHANGE = oldTenant, oldUser, oldChanging + waf_service.InvalidateShardCounts() + dialect.Register(priorDialect) + }) + if err := db.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", func(d *gorm.DB) { + d.Where("tenant_id = ? and user_code = ?", global.GWAF_TENANT_ID, global.GWAF_USER_CODE) + }); err != nil { + t.Fatal(err) + } + row := model.SecurityEvent{LogNarrow: model.LogNarrow{ + ReqUUID: "req_nonstandard_0", TenantId: global.GWAF_TENANT_ID, UserCode: global.GWAF_USER_CODE, + URL: "https://example.test/check", Method: "GET", HostCode: "api-host", UNIX_ADD_TIME: 1, + }} + if err := db.Create(&row).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.AccessLog{LogNarrow: row.LogNarrow}).Error; err != nil { + t.Fatal(err) + } + return db +} + +func callLogTestAPI(t *testing.T, handler gin.HandlerFunc, method, target, body string) response.Response { + t.Helper() + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(func(c *gin.Context) { c.Set("is_openapi", true) }) + r.Handle(method, "/t", handler) + req := httptest.NewRequest(method, target, bytes.NewBufferString(body)) + if method == http.MethodPost { + req.Header.Set("Content-Type", "application/json") + } + rec := httptest.NewRecorder() + r.ServeHTTP(rec, req) + var result response.Response + if err := json.Unmarshal(rec.Body.Bytes(), &result); err != nil { + t.Fatalf("response decode: %v, body=%s", err, rec.Body.String()) + } + return result +} + +func TestLogDetailAPIsHandleErrorsAndLegacyIDs(t *testing.T) { + db := setupLogAPITest(t) + api := &WafLogAPi{} + for _, handler := range []struct { + name string + call gin.HandlerFunc + }{{"detail", api.GetDetailApi}, {"copy", api.GetHttpCopyMaskApi}} { + for _, format := range []string{"raw", "curl"} { + for _, tc := range []struct { + name, id, shard, wantMsg string + wantCode int + }{ + {"legacy-form", "req_nonstandard_0", "local_log.db", "获取成功", response.SUCCESS}, + {"trimmed", " req_nonstandard_0 ", "local_log.db", "获取成功", response.SUCCESS}, + {"missing", "req_missing", "local_log.db", "未找到该识别码", response.ERROR}, + {"missing-auto", "req_missing", "auto", "未找到该识别码", response.ERROR}, + {"missing-shard", "req_missing", "local_log_199901.db", "存储已不存在", response.ERROR}, + {"empty", "", "local_log.db", "解析失败", response.ERROR}, + {"whitespace", " ", "local_log.db", "格式不正确", response.ERROR}, + {"too-long", strings.Repeat("a", 65), "local_log.db", "解析失败", response.ERROR}, + {"invalid", "req_bad\nvalue", "local_log.db", "格式不正确", response.ERROR}, + } { + t.Run(handler.name+"/"+format+"/"+tc.name, func(t *testing.T) { + query := url.Values{"REQ_UUID": {tc.id}, "current_db_name": {tc.shard}, "output_format": {format}} + got := callLogTestAPI(t, handler.call, http.MethodGet, "/t?"+query.Encode(), "") + if got.Code != tc.wantCode || !strings.Contains(got.Msg, tc.wantMsg) { + t.Fatalf("expected code=%d msg contains %q, got %+v", tc.wantCode, tc.wantMsg, got) + } + if got.Code != response.SUCCESS { + data, ok := got.Data.(map[string]interface{}) + if !ok || len(data) != 0 { + t.Fatalf("failure must not return empty detail or generated request: %+v", got.Data) + } + } else if handler.name == "detail" { + data, ok := got.Data.(map[string]interface{}) + if !ok || data["req_uuid"] != "req_nonstandard_0" { + t.Fatalf("REQ_UUID form binding failed: %+v", got.Data) + } + } else if data, ok := got.Data.(string); !ok || !strings.Contains(data, "GET") || strings.Contains(data, "curl -X ") { + t.Fatalf("copy should contain a populated request: %+v", got.Data) + } + }) + } + } + } + if err := db.Callback().Query().Before("gorm:query").Register("test:read_error", func(db *gorm.DB) { db.AddError(errors.New("test read failure")) }); err != nil { + t.Fatal(err) + } + for _, handler := range []gin.HandlerFunc{api.GetDetailApi, api.GetHttpCopyMaskApi} { + got := callLogTestAPI(t, handler, http.MethodGet, "/t?REQ_UUID=req_nonstandard_0¤t_db_name=local_log.db&output_format=curl", "") + if got.Code != response.ERROR || !strings.Contains(got.Msg, "查询日志详情失败") { + t.Fatalf("read failure must propagate: %+v", got) + } + } +} + +func TestLogListAPIUUIDValidationAndSelectedShard(t *testing.T) { + setupLogAPITest(t) + api := &WafLogAPi{} + for _, tc := range []struct { + name, id string + wantCode int + wantTotal float64 + }{ + {"legacy-trimmed", " req_nonstandard_0 ", response.SUCCESS, 1}, + {"missing", "req_missing", response.SUCCESS, 0}, + {"too-long", strings.Repeat("a", 65), response.ERROR, 0}, + {"invalid", "req_bad value", response.ERROR, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + body, err := json.Marshal(map[string]interface{}{ + "req_uuid": tc.id, "current_db_name": "local_log_199901.db", + "unix_add_time_begin": "4102444800000", "unix_add_time_end": "4102531200000", + "sort_by": "unix_add_time", "sort_descending": "desc", "pageIndex": 1, "pageSize": 10, + }) + if err != nil { + t.Fatal(err) + } + got := callLogTestAPI(t, api.GetListApi, http.MethodPost, "/t", string(body)) + if got.Code != tc.wantCode { + t.Fatalf("expected code=%d got %+v", tc.wantCode, got) + } + if got.Code == response.SUCCESS { + data, ok := got.Data.(map[string]interface{}) + if !ok || data["total"] != tc.wantTotal || data["uuid_lookup"] != true { + t.Fatalf("selected shard/date UUID query: %+v", got.Data) + } + } + }) + } +} diff --git a/global/template_config.go b/global/template_config.go index 7d5102e..efadf0f 100644 --- a/global/template_config.go +++ b/global/template_config.go @@ -1,4 +1,4 @@ package global // GLOBAL_DEFAULT_BLOCK_INFO 默认被拦截的HTML的内容 -var GLOBAL_DEFAULT_BLOCK_INFO string = "\n \n \n \n \n 您的访问被阻止\n \n \n \n
\n

您的访问被阻止

\n

由于安全策略,您的请求被阻止。
如果有疑问,请将下面的访问识别码发给管理员,以便进一步排查。

\n

访问识别码:[[.SAMWAF_REQ_UUID]]

\n\t\t
\n 安全防护由SamWAF提供。\n
\n
\n \n " +var GLOBAL_DEFAULT_BLOCK_INFO string = "\n \n \n \n \n 您的访问被阻止\n \n \n \n
\n

您的访问被阻止

\n

由于安全策略,您的请求被阻止。
如果有疑问,请将下面的访问识别码和您的 IP 发给管理员,以便进一步排查。

\n

访问识别码:[[.SAMWAF_REQ_UUID]]

\n

您的 IP:[[.SAMWAF_IP]]

\n\t\t
\n 安全防护由SamWAF提供。\n
\n
\n \n " diff --git a/innerbean/web_log.go b/innerbean/web_log.go index 87a2fad..1225b39 100644 --- a/innerbean/web_log.go +++ b/innerbean/web_log.go @@ -65,6 +65,8 @@ type WebLog struct { // 截断只发生在落库这一步,内存对象与 Kafka 出口始终是原文。 Truncated int `json:"truncated"` + SkipPayloadStore bool `gorm:"-" json:"-"` // 仅保存窄行 + // GeoUnresolved 本次请求的地区无法判定(没有可用的地区库,或查询失败), // 区别于"查出来是未知"。为 true 时规则引擎会跳过引用了 COUNTRY/PROVINCE/CITY 的规则, // 避免 `MF.COUNTRY != "中国"` 这类规则在 IPv6 地区库缺失时把访客整片误杀。 diff --git a/model/request/waf_attack_log_req.go b/model/request/waf_attack_log_req.go index 718b8b4..51c761e 100644 --- a/model/request/waf_attack_log_req.go +++ b/model/request/waf_attack_log_req.go @@ -6,7 +6,7 @@ type WafAttackLogDetailReq struct { // detail/httpcopymask 为 GET 请求,需 form tag 才能从 query 绑定; // 前端发送的 query 键为 current_db_name / REQ_UUID / output_format CurrrentDbName string `json:"current_db_name" form:"current_db_name"` - REQ_UUID string `json:"req_uuid" form:"REQ_UUID"` + REQ_UUID string `json:"req_uuid" form:"REQ_UUID" binding:"required,max=64"` OutputFormat string `json:"output_format" form:"output_format"` //输出格式 raw,curl } @@ -19,7 +19,7 @@ type WafAttackLogSearch struct { ViewType string `json:"view_type" form:"view_type"` //视图:access=访问日志(默认) event=安全事件 HostCode string `json:"host_code" form:"host_code"` //主机码 Rule string `json:"rule" form:"rule"` //规则名 - ReqUuid string `json:"req_uuid" form:"req_uuid"` //请求UUID + ReqUuid string `json:"req_uuid" form:"req_uuid" binding:"max=64"` //请求UUID Action string `json:"action" form:"action"` //状态 SrcIp string `json:"src_ip" form:"src_ip"` //请求IP StatusCode string `json:"status_code" form:"status_code"` //响应码 diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index 84011f7..6c78d72 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -22,6 +22,8 @@ type WafLogService struct{} var WafLogServiceApp = new(WafLogService) +var ErrLogNotFound = errors.New("未找到该识别码的日志,记录可能未保存或已过保留期") + // listExcludeColumns 列表查询排除的列:大文本字段和原始 blob 字段 var listExcludeColumns = map[string]bool{ "body": true, "res_body": true, "post_form": true, @@ -123,6 +125,10 @@ func (receiver *WafLogService) ModifyApi(log innerbean.WebLog) error { } func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) (innerbean.WebLog, error) { var weblog innerbean.WebLog + req.REQ_UUID = strings.TrimSpace(req.REQ_UUID) + if req.REQ_UUID == "" { + return weblog, ErrLogNotFound + } // 解析当前分片的三层表:安全事件 → 访问日志 → 存量 web_logs,按 req_uuid 逐层点查。 // 事件双写了窄行,内容一致,但事件表保留期更长,优先从它读。 // 分区标识为空或 auto 时先按识别码定位分区:详情最常见的来路就是 @@ -151,7 +157,7 @@ func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) ( } } if !found { - return weblog, nil + return weblog, ErrLogNotFound } // 告诉界面这条是在哪个分区找到的:auto 时用户并没有选分区,得有个地方说清楚 weblog.ShardName = shardName diff --git a/service/waf_service/waf_log_lookup_test.go b/service/waf_service/waf_log_lookup_test.go new file mode 100644 index 0000000..e914264 --- /dev/null +++ b/service/waf_service/waf_log_lookup_test.go @@ -0,0 +1,189 @@ +package waf_service + +import ( + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/model/request" + "SamWaf/wafdb" + "SamWaf/wafdb/dialect" + "errors" + "os" + "path/filepath" + "testing" + "time" + + sqlite "github.com/samwafgo/sqlitedriver" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +func setupLogLookupTest(t *testing.T) (*gorm.DB, *gorm.DB, string) { + t.Helper() + priorDialect := func() (d dialect.DBDialect) { + defer func() { _ = recover() }() + return dialect.Get() + }() + dialect.Register(&dialect.SQLiteDialect{}) + oldCore, oldLog := global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB + oldTenant, oldUser, oldKey := global.GWAF_TENANT_ID, global.GWAF_USER_CODE, global.GWAF_PWD_LOGDB + global.GWAF_TENANT_ID, global.GWAF_USER_CODE, global.GWAF_PWD_LOGDB = "lookup-tenant", "lookup-user", "" + InvalidateShardCounts() + t.Cleanup(func() { + global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB = oldCore, oldLog + global.GWAF_TENANT_ID, global.GWAF_USER_CODE, global.GWAF_PWD_LOGDB = oldTenant, oldUser, oldKey + InvalidateShardCounts() + dialect.Register(priorDialect) + }) + open := func(path string) *gorm.DB { + db, err := gorm.Open(sqlite.Open(path), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { sqlDB, _ := db.DB(); _ = sqlDB.Close() }) + return db + } + root := t.TempDir() + core, live := open(filepath.Join(root, "core.db")), open(filepath.Join(root, "live.db")) + global.GWAF_LOCAL_DB, global.GWAF_LOCAL_LOG_DB = core, live + if err := core.AutoMigrate(&model.ShareDb{}); err != nil { + t.Fatal(err) + } + if err := live.AutoMigrate(&model.AccessLog{}, &model.SecurityEvent{}, &model.EventPayload{}, &innerbean.WebLog{}); err != nil { + t.Fatal(err) + } + for _, db := range []*gorm.DB{core, live} { + if err := db.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", func(d *gorm.DB) { + d.Where("tenant_id = ? and user_code = ?", global.GWAF_TENANT_ID, global.GWAF_USER_CODE) + }); err != nil { + t.Fatal(err) + } + } + return core, live, root +} + +func TestLogLookupUUIDIgnoresSelectedShardAndDate(t *testing.T) { + core, _, root := setupLogLookupTest(t) + const archiveName = "local_log_202001.db" + if err := os.Mkdir(filepath.Join(root, "data"), 0700); err != nil { + t.Fatal(err) + } + archive, err := gorm.Open(sqlite.Open(filepath.Join(root, "data", archiveName)), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)}) + if err != nil { + t.Fatal(err) + } + archiveSQL, _ := archive.DB() + t.Cleanup(func() { _ = archiveSQL.Close() }) + if err = archive.AutoMigrate(&model.AccessLog{}, &model.SecurityEvent{}, &model.EventPayload{}); err != nil { + t.Fatal(err) + } + at := time.Date(2020, 1, 10, 12, 0, 0, 0, time.Local) + narrow := model.LogNarrow{ + ReqUUID: "fan_nonstandard_202001_0", TenantId: global.GWAF_TENANT_ID, UserCode: global.GWAF_USER_CODE, + HostCode: "host-old", Method: "GET", SRC_IP: "203.0.113.8", RULE: "CC封禁", ACTION: "阻止", + UNIX_ADD_TIME: at.UnixMilli(), CREATE_TIME: at.Format("2006-01-02 15:04:05"), USER_AGENT: "lookup-agent", + } + for _, row := range []interface{}{&model.AccessLog{LogNarrow: narrow}, &model.SecurityEvent{LogNarrow: narrow}} { + if err = archive.Create(row).Error; err != nil { + t.Fatal(err) + } + } + foreign := narrow + foreign.ReqUUID, foreign.TenantId = "req_foreign_tenant", "other-tenant" + if err = archive.Create(&model.AccessLog{LogNarrow: foreign}).Error; err != nil { + t.Fatal(err) + } + if err = core.Create(&model.ShareDb{ + BaseOrm: baseorm.BaseOrm{Id: "lookup-archive", Tenant_ID: global.GWAF_TENANT_ID, USER_CODE: global.GWAF_USER_CODE}, + DbLogicType: "log", FileName: archiveName, Cnt: 2, + StartTime: customtype.JsonTime(at.Add(-time.Hour)), EndTime: customtype.JsonTime(at.Add(time.Hour)), + }).Error; err != nil { + t.Fatal(err) + } + // 通过现有连接缓存装入临时归档,避免读取工作区 data/。 + if err = wafdb.InitManaulLogDb(root, archiveName); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { wafdb.CloseManualLogDb(archiveName) }) + + base := request.WafAttackLogSearch{ + ReqUuid: " " + narrow.ReqUUID + " ", UnixAddTimeBegin: "4102444800000", UnixAddTimeEnd: "4102531200000", + HostCode: narrow.HostCode, SrcIp: narrow.SRC_IP, Rule: narrow.RULE, FilterBy: "user_agent", FilterValue: "lookup-agent", + SortBy: "unix_add_time", SortDescending: "desc", + } + base.PageIndex, base.PageSize = 1, 10 + for _, selected := range []string{"", AutoShard, wafdb.LiveLogName(), "local_log_199901.db"} { + for _, view := range []string{"access", "event"} { + t.Run(selected+"/"+view, func(t *testing.T) { + req := base + req.CurrrentDbName, req.ViewType = selected, view + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req) + if err != nil || total != 1 || len(rows) != 1 || rows[0].REQ_UUID != narrow.ReqUUID || meta.FoundIn != archiveName || !meta.UuidLookup { + t.Fatalf("archive UUID lookup: total=%d rows=%+v meta=%+v err=%v", total, rows, meta, err) + } + }) + } + } + for _, filter := range []string{"host", "ip", "rule", "header-filter", "tenant", "user", "missing"} { + t.Run(filter, func(t *testing.T) { + req := base + req.CurrrentDbName = "local_log_199901.db" + switch filter { + case "host": + req.HostCode = "other-host" + case "ip": + req.SrcIp = "203.0.113.9" + case "rule": + req.Rule = "other-rule" + case "header-filter": + req.FilterValue = "different-agent" + case "tenant": + req.ReqUuid = foreign.ReqUUID + case "user": + global.GWAF_USER_CODE = "other-user" + defer func() { global.GWAF_USER_CODE = narrow.UserCode }() + case "missing": + req.ReqUuid = "req_missing" + } + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(req) + if err != nil || total != 0 || len(rows) != 0 || !meta.UuidLookup { + t.Fatalf("expected filtered empty lookup: total=%d rows=%+v meta=%+v err=%v", total, rows, meta, err) + } + }) + } + // 无识别码仍按显式分区、日期查询。 + base.ReqUuid, base.CurrrentDbName = " ", wafdb.LiveLogName() + rows, total, meta, err := WafLogServiceApp.GetListApiWithMeta(base) + if err != nil || len(rows) != 0 || total != 0 || meta.UuidLookup || len(meta.Shards) != 1 { + t.Fatalf("ordinary selected query changed: total=%d meta=%+v err=%v", total, meta, err) + } +} + +func TestLogDetailMissingAndReadError(t *testing.T) { + _, live, _ := setupLogLookupTest(t) + for _, id := range []string{"req_missing", " "} { + got, err := WafLogServiceApp.GetDetailApi(request.WafAttackLogDetailReq{REQ_UUID: id, CurrrentDbName: wafdb.LiveLogName()}) + if !errors.Is(err, ErrLogNotFound) || got.REQ_UUID != "" { + t.Fatalf("expected identifiable missing error: got=%+v err=%v", got, err) + } + } + if _, err := WafLogServiceApp.GetDetailApi(request.WafAttackLogDetailReq{REQ_UUID: "req_missing", CurrrentDbName: "local_log_199901.db"}); err == nil || errors.Is(err, ErrLogNotFound) { + t.Fatalf("missing shard must return storage error, got %v", err) + } + row := model.SecurityEvent{LogNarrow: model.LogNarrow{ReqUUID: "req_detail", TenantId: global.GWAF_TENANT_ID, UserCode: global.GWAF_USER_CODE}} + if err := live.Create(&row).Error; err != nil { + t.Fatal(err) + } + got, err := WafLogServiceApp.GetDetailApi(request.WafAttackLogDetailReq{REQ_UUID: " req_detail ", CurrrentDbName: wafdb.LiveLogName()}) + if err != nil || got.REQ_UUID != row.ReqUUID { + t.Fatalf("trimmed detail lookup: got=%+v err=%v", got, err) + } + if err := live.Callback().Query().Before("gorm:query").Register("test:read_error", func(db *gorm.DB) { db.AddError(errors.New("test read failure")) }); err != nil { + t.Fatal(err) + } + if _, err := WafLogServiceApp.GetDetailApi(request.WafAttackLogDetailReq{REQ_UUID: row.ReqUUID, CurrrentDbName: wafdb.LiveLogName()}); err == nil || errors.Is(err, ErrLogNotFound) { + t.Fatalf("read failure must remain an error, got %v", err) + } +} diff --git a/service/waf_service/waf_log_query.go b/service/waf_service/waf_log_query.go index 1f006be..cb359d7 100644 --- a/service/waf_service/waf_log_query.go +++ b/service/waf_service/waf_log_query.go @@ -491,7 +491,8 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch // 悄悄改成扇出会让它们的每次查询都多打几个分区 auto := shardName == AutoShard - if auto && strings.TrimSpace(req.ReqUuid) != "" { + req.ReqUuid = strings.TrimSpace(req.ReqUuid) + if req.ReqUuid != "" { return receiver.lookupByUuid(req) } @@ -718,7 +719,9 @@ func (receiver *WafLogService) GetListApiWithMeta(req request.WafAttackLogSearch // // 识别码是三张日志表的主键,一次点查就是一次索引命中;最常见的来路是访客把拦截页上的 // 识别码报给管理员——他手里只有这串码,不知道是哪天,更不知道在哪个分区。 -// 查不到不是错误:调用方按空结果处理,界面负责解释可能的原因(已过保留期 / 抄错 / 本就没记录)。 +// 查不到不是错误:所有分区都查询成功但没有行时,调用方按空结果处理,界面负责解释可能的原因 +// (已过保留期 / 抄错 / 本就没记录)。有分区查询失败则返回首个错误——明明可能是扫描失败, +// 不能伪装成「未找到」。 func (receiver *WafLogService) lookupByUuid(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, LogQueryMeta, error) { meta := LogQueryMeta{UuidLookup: true} ctx, cancel := context.WithTimeout(context.Background(), fanoutBudget) @@ -763,6 +766,9 @@ func (receiver *WafLogService) lookupByUuid(req request.WafAttackLogSearch) ([]i return found, int64(len(found)), meta, nil } } + if firstErr != nil { + return []innerbean.WebLog{}, 0, meta, firstErr + } return []innerbean.WebLog{}, 0, meta, nil } diff --git a/wafenginecore/anticc_ban.go b/wafenginecore/anticc_ban.go index ced320f..b36acb3 100644 --- a/wafenginecore/anticc_ban.go +++ b/wafenginecore/anticc_ban.go @@ -31,6 +31,37 @@ func matchCCBan(hostCode, netSrcIP, srcIP, ipMode string) (hitKey string, client return "", clientIP } +func ccBanLog(log *innerbean.WebLog, responseCode int) *innerbean.WebLog { + recorded := *log + recorded.SkipPayloadStore = true + recorded.HEADER = "" + recorded.BODY = "" + recorded.RES_BODY = "" + recorded.COOKIES = "" + recorded.POST_FORM = "" + recorded.ResHeader = "" + recorded.SrcByteBody = nil + recorded.SrcByteResBody = nil + recorded.SrcURL = nil + recorded.BodyDecoded = "" + recorded.CookiesDecoded = "" + recorded.BodyValues = nil + recorded.BodyFields = nil + recorded.BodyHash = "" + recorded.RULE = "CC封禁" + recorded.ACTION = "阻止" + recorded.STATUS = "阻止访问" + if responseCode == 444 { + recorded.STATUS = "连接关闭" + } + recorded.STATUS_CODE = responseCode + recorded.TASK_FLAG = 1 + recorded.RISK_LEVEL = 1 + recorded.GUEST_IDENTIFICATION = "可疑用户" + recorded.TimeSpent = time.Now().UnixMilli() - recorded.UNIX_ADD_TIME + return &recorded +} + // notifyCCBanOnce 推送 CC 封禁提醒,同一封禁键在节流窗口内只推一条。 func notifyCCBanOnce(hitKey, clientIP string, region []string) { throttleKey := "cc_ban_notify_" + hitKey diff --git a/wafenginecore/anticc_ban_http_test.go b/wafenginecore/anticc_ban_http_test.go new file mode 100644 index 0000000..00dbde7 --- /dev/null +++ b/wafenginecore/anticc_ban_http_test.go @@ -0,0 +1,244 @@ +package wafenginecore + +import ( + "bufio" + "fmt" + "net" + "net/http" + "net/http/httptest" + "regexp" + "testing" + "time" + + "SamWaf/cache" + "SamWaf/common/queue" + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/wafenginmodel" +) + +// 本文件是 CC 封禁拦截链路的 HTTP 级用例:真实走 ServeHTTP 全链路 +// (路由解析 → 流量计量 → CC 封禁命中 → 拦截页渲染 → 窄行入队 → 提醒节流), +// 验证「拦截页展示的识别码/IP 与落库窄行一一对应」这一 #1001 修复的核心承诺。 + +var ccBanPageUUIDRe = regexp.MustCompile(`访问识别码:([0-9a-fA-F-]{36})`) + +// hijackRecorder 让 httptest 记录器具备 Hijack 能力,用于覆盖 444 关闭连接分支。 +type hijackRecorder struct { + *httptest.ResponseRecorder + conn net.Conn +} + +func (h *hijackRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { + server, client := net.Pipe() + h.conn = server + go func() { _, _ = bufio.NewReader(client).ReadString(0) }() + return server, bufio.NewReadWriter(bufio.NewReader(server), bufio.NewWriter(server)), nil +} + +// setupCCBanEngine 搭一套可跑 ServeHTTP 的最小引擎:站点 + 全局网站 + 封禁键 + 空队列。 +// 返回的清理函数恢复被替换的全局对象。 +func setupCCBanEngine(t *testing.T, banIP string, mutateSite func(h *wafenginmodel.HostSafe)) (*WafEngine, func()) { + t.Helper() + + waf := &WafEngine{} + waf.InitRouting() + waf.ServerOnline = wafenginmodel.NewSafeServerMap() + + site := &wafenginmodel.HostSafe{ + Host: model.Hosts{ + Code: "ccb-code", + Host: "ccb-test.com", + Port: 80, + GUARD_STATUS: 1, + IPSourceMode: "nic", + }, + BlockingPage: map[string]model.BlockingPage{}, + RuleData: []model.Rules{}, + IPBlockLists: []model.IPBlockList{}, + LoadBalanceRuntime: &wafenginmodel.LoadBalanceRuntime{}, + } + if mutateSite != nil { + mutateSite(site) + } + globalHost := &wafenginmodel.HostSafe{ + Host: model.Hosts{Code: global.GWAF_GLOBAL_HOST_CODE}, + BlockingPage: map[string]model.BlockingPage{}, + RuleData: []model.Rules{}, + IPBlockLists: []model.IPBlockList{}, + LoadBalanceRuntime: &wafenginmodel.LoadBalanceRuntime{}, + } + waf.withWriteTable(func(nt *routingTable) { + nt.HostTarget["ccb-test.com:80"] = site + nt.HostCode["ccb-code"] = "ccb-test.com:80" + nt.HostTarget["global:80"] = globalHost + nt.HostCode[global.GWAF_GLOBAL_HOST_CODE] = "global:80" + }) + + oldCache, oldLogQ, oldMsgQ := global.GCACHE_WAFCACHE, global.GQEQUE_LOG_DB, global.GQEQUE_MESSAGE_DB + global.GCACHE_WAFCACHE = cache.InitWafCache() + global.GQEQUE_LOG_DB = queue.NewQueue() + global.GQEQUE_MESSAGE_DB = queue.NewQueue() + + banKey := model.BuildCCBanKey(model.CCBanScopeHost, "ccb-code", banIP) + global.GCACHE_WAFCACHE.SetWithTTl(banKey, 300, 10*time.Minute) + + return waf, func() { + global.GCACHE_WAFCACHE = oldCache + global.GQEQUE_LOG_DB = oldLogQ + global.GQEQUE_MESSAGE_DB = oldMsgQ + } +} + +func doCCBanRequest(waf *WafEngine, w http.ResponseWriter, path string) *http.Request { + req := httptest.NewRequest("GET", "http://ccb-test.com"+path, nil) + req.RemoteAddr = "10.1.2.3:40001" + waf.ServeHTTP(w, req) + return req +} + +// 封禁期每次请求:拦截页给新的识别码且展示访客 IP,每个识别码对应一条窄行落库。 +// 这正是 #1001 的验收口径——页面上看到的识别码必须能在日志里查到。 +func TestCCBanServeHTTP_PageIdMatchesNarrowLog(t *testing.T) { + waf, cleanup := setupCCBanEngine(t, "10.1.2.3", nil) + defer cleanup() + + const n = 3 + pageUUIDs := make([]string, 0, n) + for i := 0; i < n; i++ { + rec := httptest.NewRecorder() + doCCBanRequest(waf, rec, fmt.Sprintf("/ccban?page=%d", i)) + if rec.Code != 403 { + t.Fatalf("第 %d 次请求应返回 403,实际 %d", i, rec.Code) + } + body := rec.Body.String() + m := ccBanPageUUIDRe.FindStringSubmatch(body) + if m == nil { + t.Fatalf("拦截页缺少访问识别码:%s", body[:min(200, len(body))]) + } + if body != "" && !containsAll(body, "您的 IP:10.1.2.3") { + t.Fatalf("拦截页应展示访客 IP 10.1.2.3") + } + pageUUIDs = append(pageUUIDs, m[1]) + } + for i := 1; i < n; i++ { + if pageUUIDs[i] == pageUUIDs[0] { + t.Fatalf("每次请求的识别码应独立生成,第 %d 次与第 0 次相同", i) + } + } + + if got := global.GQEQUE_LOG_DB.Size(); got != n { + t.Fatalf("应入队 %d 条窄行,实际 %d", n, got) + } + for i := 0; i < n; i++ { + v, ok := global.GQEQUE_LOG_DB.Dequeue() + if !ok { + t.Fatalf("第 %d 条窄行取不到", i) + } + lg, ok := v.(*innerbean.WebLog) + if !ok { + t.Fatalf("入队对象类型不对: %T", v) + } + if lg.REQ_UUID != pageUUIDs[i] { + t.Fatalf("第 %d 条窄行识别码 %s 与页面展示的 %s 不一致", i, lg.REQ_UUID, pageUUIDs[i]) + } + if lg.RULE != "CC封禁" || lg.ACTION != "阻止" || lg.STATUS != "阻止访问" || lg.STATUS_CODE != 403 { + t.Fatalf("第 %d 条窄行归因不对: rule=%q action=%q status=%q code=%d", i, lg.RULE, lg.ACTION, lg.STATUS, lg.STATUS_CODE) + } + if !lg.SkipPayloadStore { + t.Fatalf("第 %d 条窄行应标记跳过重报文留存", i) + } + if lg.HEADER != "" || lg.BODY != "" || lg.COOKIES != "" || lg.SrcByteBody != nil || lg.SrcURL != nil { + t.Fatalf("第 %d 条窄行应只留元数据,实际 HEADER=%d 字节 BODY=%d 字节", i, len(lg.HEADER), len(lg.BODY)) + } + if lg.SRC_IP != "10.1.2.3" { + t.Fatalf("第 %d 条窄行来源 IP 应为 10.1.2.3,实际 %q", i, lg.SRC_IP) + } + } +} + +// 站点给 cc_attack 配了自定义拦截页时:响应码以页面配置为准,窄行记录实际下发的码。 +// 444(关闭连接)在可劫持的真实连接上生效,且窄行状态写明「连接关闭」。 +func TestCCBanServeHTTP_CustomStatusAnd444(t *testing.T) { + mkPage := func(code string) func(h *wafenginmodel.HostSafe) { + return func(h *wafenginmodel.HostSafe) { + h.BlockingPage = map[string]model.BlockingPage{ + "cc_attack": {AttackType: "cc_attack", ResponseCode: code, ResponseContent: "CC BANNED [[.SAMWAF_REQ_UUID]] [[.SAMWAF_IP]]"}, + } + } + } + // 两个子场景各建一套引擎;任一 t.Fatalf 都要把全局替换恢复掉,不能泄漏给同包后续用例 + cleanups := []func(){} + defer func() { + for i := len(cleanups) - 1; i >= 0; i-- { + cleanups[i]() + } + }() + setup := func(ip string, mutate func(h *wafenginmodel.HostSafe)) *WafEngine { + waf, cleanup := setupCCBanEngine(t, ip, mutate) + cleanups = append(cleanups, cleanup) + return waf + } + dequeueLog := func(tag string) *innerbean.WebLog { + v, ok := global.GQEQUE_LOG_DB.Dequeue() + if !ok { + t.Fatalf("%s 应有一条窄行入队", tag) + } + lg, ok := v.(*innerbean.WebLog) + if !ok { + t.Fatalf("%s 入队对象类型不对: %T", tag, v) + } + return lg + } + + // 429 自定义码 + waf := setup("10.1.2.3", mkPage("429")) + rec := httptest.NewRecorder() + doCCBanRequest(waf, rec, "/ccban") + if rec.Code != 429 { + t.Fatalf("自定义拦截页 429 应原样下发,实际 %d", rec.Code) + } + if !containsAll(rec.Body.String(), "CC BANNED", "10.1.2.3") { + t.Fatalf("自定义拦截页应渲染识别码与 IP 变量: %s", rec.Body.String()) + } + lg := dequeueLog("429") + if lg.STATUS_CODE != 429 || lg.RULE != "CC封禁" { + t.Fatalf("窄行应记录实际下发的 429,实际 code=%d rule=%q", lg.STATUS_CODE, lg.RULE) + } + + // 444 关闭连接(用可劫持的记录器模拟真实连接) + waf = setup("10.1.2.3", mkPage("444")) + hj := &hijackRecorder{ResponseRecorder: httptest.NewRecorder()} + doCCBanRequest(waf, hj, "/ccban") + if hj.Body.Len() != 0 { + t.Fatalf("444 分支不应写出任何响应体,实际 %d 字节", hj.Body.Len()) + } + lg = dequeueLog("444") + if lg.STATUS_CODE != 444 || lg.STATUS != "连接关闭" { + t.Fatalf("444 窄行应写明连接关闭,实际 code=%d status=%q", lg.STATUS_CODE, lg.STATUS) + } +} + +// 封禁期内持续重试不会刷爆提醒队列:同一封禁键只推一条。 +func TestCCBanServeHTTP_NotifyThrottled(t *testing.T) { + waf, cleanup := setupCCBanEngine(t, "10.1.2.3", nil) + defer cleanup() + + for i := 0; i < 5; i++ { + rec := httptest.NewRecorder() + doCCBanRequest(waf, rec, "/ccban") + } + if got := global.GQEQUE_MESSAGE_DB.Size(); got != 1 { + t.Fatalf("同一封禁键 5 次请求应只推 1 条提醒,实际 %d", got) + } +} + +func containsAll(s string, subs ...string) bool { + for _, sub := range subs { + if !regexp.MustCompile(regexp.QuoteMeta(sub)).MatchString(s) { + return false + } + } + return true +} diff --git a/wafenginecore/anticc_ban_test.go b/wafenginecore/anticc_ban_test.go new file mode 100644 index 0000000..1dff88d --- /dev/null +++ b/wafenginecore/anticc_ban_test.go @@ -0,0 +1,44 @@ +package wafenginecore + +import ( + "SamWaf/innerbean" + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +func TestCCBanLogNarrowCopy(t *testing.T) { + original := innerbean.WebLog{ + REQ_UUID: "ban-request", HOST_CODE: "host", SRC_IP: "203.0.113.10", NetSrcIp: "127.0.0.1", + URL: "/test", METHOD: "POST", ACTION: "通过", UNIX_ADD_TIME: time.Now().UnixMilli() - 10, + HEADER: "header", BODY: "body", RES_BODY: "response", COOKIES: "cookie", POST_FORM: "form", + ResHeader: "res-header", SrcByteBody: []byte("body"), SrcByteResBody: []byte("response"), + SrcURL: []byte("url"), BodyDecoded: "decoded", CookiesDecoded: "decoded-cookie", + BodyValues: []string{"value"}, BodyFields: []string{"field"}, BodyHash: "hash", + } + before := original + for _, code := range []int{403, 429, 444} { + got := ccBanLog(&original, code) + if got.REQ_UUID != original.REQ_UUID || got.SRC_IP != original.SRC_IP || got.NetSrcIp != original.NetSrcIp || got.URL != original.URL { + t.Fatal("request identifiers changed") + } + if got.STATUS_CODE != code || got.ACTION != "阻止" || got.RULE != "CC封禁" || !got.SkipPayloadStore || !got.IsSecurityEvent() { + t.Fatalf("incorrect event: %+v", got) + } + if code == 444 && got.STATUS != "连接关闭" { + t.Fatal("444 must record connection closure") + } + if got.HEADER != "" || got.BODY != "" || got.RES_BODY != "" || got.COOKIES != "" || got.POST_FORM != "" || got.ResHeader != "" || got.BodyDecoded != "" || got.CookiesDecoded != "" || got.BodyHash != "" || len(got.SrcByteBody)+len(got.SrcByteResBody)+len(got.SrcURL)+len(got.BodyValues)+len(got.BodyFields) != 0 { + t.Fatal("narrow log retains payload") + } + if got.TimeSpent < 0 || !reflect.DeepEqual(original, before) { + t.Fatal("original request changed or negative duration") + } + encoded, err := json.Marshal(got) + if err != nil || strings.Contains(string(encoded), "SkipPayloadStore") || strings.Contains(string(encoded), "skip_payload") { + t.Fatal("runtime marker leaked to JSON") + } + } +} diff --git a/wafenginecore/custom_error_page.go b/wafenginecore/custom_error_page.go index 3a4d6f4..73cf370 100644 --- a/wafenginecore/custom_error_page.go +++ b/wafenginecore/custom_error_page.go @@ -92,7 +92,7 @@ func resolveCustomErrorPage(hostSafe, globalHostSafe *wafenginmodel.HostSafe, ba // - 「优先后端响应」且后端确有内容 → 原样还原(状态码/响应头/压缩编码/Content-Type 全不动) // - 其余情况 → 渲染模版覆盖 func applyCustomErrorPage(resp *http.Response, page *model.BlockingPage, - backendStatusCode int, backendStatus string, reqUUID string) customErrorPageResult { + backendStatusCode int, backendStatus string, reqUUID string, srcIP string) customErrorPageResult { var rawBody []byte if resp.Body != nil && resp.Body != http.NoBody { @@ -118,6 +118,7 @@ func applyCustomErrorPage(resp *http.Response, page *model.BlockingPage, renderData := map[string]interface{}{ "SAMWAF_REQ_UUID": reqUUID, + "SAMWAF_IP": templateSourceIP(srcIP), "SAMWAF_BACKEND_STATUS": backendStatus, "SAMWAF_BACKEND_CODE": backendStatusCode, "SAMWAF_BACKEND_BODY": string(plainBody), diff --git a/wafenginecore/custom_error_page_test.go b/wafenginecore/custom_error_page_test.go index 0694074..504029f 100644 --- a/wafenginecore/custom_error_page_test.go +++ b/wafenginecore/custom_error_page_test.go @@ -136,7 +136,7 @@ func TestApplyCustomErrorPage_DefaultOverwrites(t *testing.T) { backendBody := []byte(`{"code":` + strconv.Itoa(code) + `,"message":"backend says no"}`) resp := newBackendResp(code, strconv.Itoa(code)+" Err", "application/json", backendBody) - result := applyCustomErrorPage(resp, page("", priority), code, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", priority), code, resp.Status, "uuid-1", "") if !result.TemplateApplied { t.Fatalf("priority=%q code=%d 应使用模版覆盖", priority, code) @@ -162,7 +162,7 @@ func TestApplyCustomErrorPage_BackendFirstPassthrough(t *testing.T) { backendBody := []byte(`{"code":` + strconv.Itoa(code) + `,"message":"no permission"}`) resp := newBackendResp(code, strconv.Itoa(code)+" Err", "application/json", backendBody) - result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), code, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), code, resp.Status, "uuid-1", "") if result.TemplateApplied { t.Fatalf("code=%d 后端有响应体时应透传,不应套模版", code) @@ -192,7 +192,7 @@ func TestApplyCustomErrorPage_BackendFirstFallback(t *testing.T) { t.Run(c.name, func(t *testing.T) { resp := newBackendResp(500, "500 Internal Server Error", "text/html", c.body) - result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 500, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 500, resp.Status, "uuid-1", "") if !result.TemplateApplied { t.Fatal("后端无响应体时应回落到自定义模版") @@ -210,7 +210,7 @@ func TestApplyCustomErrorPage_NilBody(t *testing.T) { resp := newBackendResp(502, "502 Bad Gateway", "text/html", nil) resp.Body = body - result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 502, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 502, resp.Status, "uuid-1", "") if !result.TemplateApplied { t.Fatal("无 body 时应使用模版") @@ -225,7 +225,7 @@ func TestApplyCustomErrorPage_NilBody(t *testing.T) { func TestApplyCustomErrorPage_ResponseCodeOverride(t *testing.T) { resp := newBackendResp(500, "500 Internal Server Error", "text/html", []byte("")) - applyCustomErrorPage(resp, page("503", ""), 500, resp.Status, "uuid-1") + applyCustomErrorPage(resp, page("503", ""), 500, resp.Status, "uuid-1", "") if resp.StatusCode != 503 { t.Fatalf("配置 response_code=503 时应下发 503,实际 %d", resp.StatusCode) @@ -242,7 +242,7 @@ func TestApplyCustomErrorPage_ResponseHeaderApplied(t *testing.T) { p.ResponseHeader = `[{"name":"Content-Type","value":"text/html"},{"name":"X-Waf-Page","value":"custom"},{"name":"X-Empty","value":""}]` resp := newBackendResp(404, "404 Not Found", "application/json", []byte("")) - applyCustomErrorPage(resp, p, 404, resp.Status, "uuid-1") + applyCustomErrorPage(resp, p, 404, resp.Status, "uuid-1", "") body := readBody(t, resp) if resp.Header.Get("Content-Type") != "text/html" { @@ -267,7 +267,7 @@ func TestApplyCustomErrorPage_GzipBackend_TemplateDropsContentEncoding(t *testin resp := newBackendResp(500, "500 Internal Server Error", "application/json", gzipBytes(t, plain)) resp.Header.Set("Content-Encoding", "gzip") - result := applyCustomErrorPage(resp, page("", ""), 500, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", ""), 500, resp.Status, "uuid-1", "") if !result.TemplateApplied { t.Fatal("默认模式下应套模版") @@ -299,7 +299,7 @@ func TestApplyCustomErrorPage_GzipBackend_PassthroughKeepsBytes(t *testing.T) { resp := newBackendResp(403, "403 Forbidden", "application/json", raw) resp.Header.Set("Content-Encoding", "gzip") - result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 403, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 403, resp.Status, "uuid-1", "") if result.TemplateApplied { t.Fatal("后端有内容时应透传") @@ -326,7 +326,7 @@ func TestApplyCustomErrorPage_GzipEmptyBody_FallsBackToTemplate(t *testing.T) { resp := newBackendResp(500, "500 Internal Server Error", "text/html", raw) resp.Header.Set("Content-Encoding", "gzip") - result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 500, resp.Status, "uuid-1") + result := applyCustomErrorPage(resp, page("", model.BlockingPageContentPriorityBackend), 500, resp.Status, "uuid-1", "") if !result.TemplateApplied { t.Fatal("后端 body 解压后为空,应回落到自定义模版") diff --git a/wafenginecore/error_process.go b/wafenginecore/error_process.go index 04679c2..09152a6 100644 --- a/wafenginecore/error_process.go +++ b/wafenginecore/error_process.go @@ -11,13 +11,23 @@ import ( "encoding/json" "fmt" "go.uber.org/zap" + "html" "io" + "net" "net/http" "strconv" "text/template" "time" ) +func templateSourceIP(srcIP string) string { + ip := net.ParseIP(srcIP) + if ip == nil { + return "" + } + return html.EscapeString(ip.String()) +} + func renderTemplate(templateContent string, data map[string]interface{}) ([]byte, error) { tmpl, err := template.New("blockingPageTemplate").Delims("[[", "]]").Parse(templateContent) if err != nil { @@ -47,6 +57,7 @@ func EchoErrorInfo(w http.ResponseWriter, r *http.Request, weblogbean *innerbean renderData := map[string]interface{}{ "SAMWAF_REQ_UUID": weblogbean.REQ_UUID, "SAMWAF_BLOCK_INFO": blockInfo, + "SAMWAF_IP": templateSourceIP(weblogbean.SRC_IP), } // 优先根据攻击类型查找对应的拦截页面 @@ -198,6 +209,7 @@ func EchoResponseErrorInfo(resp *http.Response, weblogbean *innerbean.WebLog, ru renderData := map[string]interface{}{ "SAMWAF_REQ_UUID": weblogbean.REQ_UUID, "SAMWAF_BLOCK_INFO": blockInfo, + "SAMWAF_IP": templateSourceIP(weblogbean.SRC_IP), } // 优先根据攻击类型查找对应的拦截页面 diff --git a/wafenginecore/error_process_test.go b/wafenginecore/error_process_test.go new file mode 100644 index 0000000..8807181 --- /dev/null +++ b/wafenginecore/error_process_test.go @@ -0,0 +1,143 @@ +package wafenginecore + +import ( + "SamWaf/global" + "SamWaf/innerbean" + "SamWaf/model" + "SamWaf/model/wafenginmodel" + "net/http/httptest" + "strconv" + "strings" + "testing" +) + +func TestTemplateSourceIP(t *testing.T) { + cases := []struct { + name string + ip string + want string + }{ + {"IPv4", "203.0.113.10", "203.0.113.10"}, + {"IPv6", "2001:0DB8:0000:0000:0000:0000:0000:0001", "2001:db8::1"}, + {"mapped IPv4", "::ffff:203.0.113.10", "203.0.113.10"}, + {"empty", "", ""}, + {"invalid", "unknown", ""}, + {"HTML suffix", "203.0.113.10", ""}, + {"quoted value", `"203.0.113.10"`, ""}, + {"IPv4 port", "203.0.113.10:443", ""}, + {"IPv6 port", "[2001:db8::1]:443", ""}, + {"zone", "fe80::1%eth0", ""}, + {"CIDR", "203.0.113.0/24", ""}, + {"list", "203.0.113.10, 203.0.113.11", ""}, + {"spaces", " 203.0.113.10 ", ""}, + {"newline", "203.0.113.10\n", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := templateSourceIP(tc.ip); got != tc.want { + t.Fatalf("templateSourceIP(%q) = %q, want %q", tc.ip, got, tc.want) + } + }) + } +} + +func TestErrorPageTemplatesSourceIP(t *testing.T) { + const reqUUID = "12345678-1234-1234-1234-123456789abc" + const customTemplate = `uuid=[[.SAMWAF_REQ_UUID]] ip=[[.SAMWAF_IP]]` + const legacyTemplate = `

legacy uuid=[[.SAMWAF_REQ_UUID]]

` + inputs := []struct { + name string + ip string + want string + }{ + {"IPv4", "203.0.113.10", "203.0.113.10"}, + {"IPv6", "2001:0DB8:0000:0000:0000:0000:0000:0001", "2001:db8::1"}, + {"invalid", "203.0.113.10", ""}, + {"empty", "", ""}, + } + + for _, entry := range []string{"request", "response", "backend"} { + for _, mode := range []string{"default", "host", "global", "legacy"} { + for _, input := range inputs { + t.Run(entry+"/"+mode+"/"+input.name, func(t *testing.T) { + hostSafe := &wafenginmodel.HostSafe{BlockingPage: map[string]model.BlockingPage{}} + globalSafe := &wafenginmodel.HostSafe{BlockingPage: map[string]model.BlockingPage{}} + content := customTemplate + if mode == "default" { + content = global.GLOBAL_DEFAULT_BLOCK_INFO + } else if mode == "legacy" { + content = legacyTemplate + } + configuredPage := model.BlockingPage{ResponseCode: "403", ResponseContent: content} + if mode == "global" { + globalSafe.BlockingPage["403"] = configuredPage + } else if mode != "default" || entry == "backend" { + hostSafe.BlockingPage["403"] = configuredPage + } + log := &innerbean.WebLog{REQ_UUID: reqUUID, SRC_IP: input.ip} + var got string + switch entry { + case "request": + r := httptest.NewRequest("GET", "http://example.test/", nil) + r.RemoteAddr = "198.51.100.20:12345" + r.Header.Set("X-Forwarded-For", "192.0.2.30") + w := httptest.NewRecorder() + code := EchoErrorInfo(w, r, log, "test", "blocked", hostSafe, globalSafe, false, "") + if code != 403 || w.Code != 403 { + t.Fatalf("unexpected status: returned=%d response=%d", code, w.Code) + } + got = w.Body.String() + case "response": + resp := newBackendResp(200, "200 OK", "text/plain", []byte("backend")) + EchoResponseErrorInfo(resp, log, "test", "blocked", hostSafe, globalSafe, false, "") + got = string(readBody(t, resp)) + if resp.StatusCode != 403 || resp.ContentLength != int64(len(got)) || resp.Header.Get("Content-Length") != strconv.Itoa(len(got)) { + t.Fatalf("response metadata differs from rendered page: %+v", resp) + } + case "backend": + resp := newBackendResp(403, "403 Forbidden", "text/html", nil) + p := resolveCustomErrorPage(hostSafe, globalSafe, 403) + if p == nil { + t.Fatal("configured backend page not resolved") + } + result := applyCustomErrorPage(resp, p, 403, resp.Status, log.REQ_UUID, log.SRC_IP) + if !result.TemplateApplied { + t.Fatal("backend page not rendered") + } + got = string(readBody(t, resp)) + if resp.StatusCode != 403 || resp.ContentLength != int64(len(got)) || resp.Header.Get("Content-Length") != strconv.Itoa(len(got)) { + t.Fatalf("backend response metadata differs from rendered page: %+v", resp) + } + } + + if mode == "legacy" { + if want := "

legacy uuid=" + reqUUID + "

"; got != want { + t.Fatalf("legacy template changed: got %q, want %q", got, want) + } + } else if mode == "default" { + for _, want := range []string{ + "访问识别码:" + reqUUID, + `

您的 IP:` + input.want + "

", + "访问识别码和您的 IP 发给管理员", + "overflow-wrap: anywhere;", + "width: calc(100% - 32px);", + "box-sizing: border-box;", + } { + if !strings.Contains(got, want) { + t.Fatalf("default page missing %q", want) + } + } + if strings.Index(got, "您的 IP:") < strings.Index(got, "访问识别码:") { + t.Fatal("IP should appear below request UUID") + } + } else if want := "uuid=" + reqUUID + " ip=" + input.want; got != want { + t.Fatalf("rendered page = %q, want %q", got, want) + } + if log.SRC_IP != input.ip { + t.Fatalf("log source IP changed: %q", log.SRC_IP) + } + }) + } + } + } +} diff --git a/wafenginecore/wafengine.go b/wafenginecore/wafengine.go index 493c7fb..d9962ef 100644 --- a/wafenginecore/wafengine.go +++ b/wafenginecore/wafengine.go @@ -599,7 +599,8 @@ func (waf *WafEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { weblogbean.RULE = "CC封禁" } else { notifyCCBanOnce(hitKey, banIP, region) - EchoErrorInfo(w, r, &weblogbean, "", "当前IP由于访问频次太高暂时无法访问", hostTarget, waf.rt().HostTarget[waf.rt().HostCode[global.GWAF_GLOBAL_HOST_CODE]], false, "cc_attack") + responseCode := EchoErrorInfo(w, r, &weblogbean, "", "当前IP由于访问频次太高暂时无法访问", hostTarget, waf.rt().HostTarget[waf.rt().HostCode[global.GWAF_GLOBAL_HOST_CODE]], false, "cc_attack") + global.GQEQUE_LOG_DB.Enqueue(ccBanLog(&weblogbean, responseCode)) return } } @@ -1422,7 +1423,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { ) if customBlockingPage != nil { - pageResult := applyCustomErrorPage(resp, customBlockingPage, backendStatusCode, backendStatus, weblogfrist.REQ_UUID) + pageResult := applyCustomErrorPage(resp, customBlockingPage, backendStatusCode, backendStatus, weblogfrist.REQ_UUID, weblogfrist.SRC_IP) if pageResult.TemplateApplied { // 记录响应Header信息 diff --git a/wafqueue/log_tier.go b/wafqueue/log_tier.go index 53143d9..ba7d494 100644 --- a/wafqueue/log_tier.go +++ b/wafqueue/log_tier.go @@ -70,8 +70,10 @@ func tierForStore(logs []*innerbean.WebLog) (events []*model.SecurityEvent, acce if isEvent { events = dedupByUUID(events, seenEvent, lg.REQ_UUID, &model.SecurityEvent{LogNarrow: ln, PayloadHash: PayloadHash(lg.RULE, lg.BodyHash)}) - if p := extractPayload(lg, PayloadKindEvent); p != nil { - payloads = dedupByUUID(payloads, seenPayload, p.ReqUUID, p) + if !lg.SkipPayloadStore { + if p := extractPayload(lg, PayloadKindEvent); p != nil { + payloads = dedupByUUID(payloads, seenPayload, p.ReqUUID, p) + } } } if watched { @@ -105,48 +107,48 @@ func narrowFromLog(lg *innerbean.WebLog) model.LogNarrow { truncated = 1 } return model.LogNarrow{ - ReqUUID: lg.REQ_UUID, - TenantId: lg.TenantId, - UserCode: lg.USER_CODE, - HostCode: lg.HOST_CODE, - Host: lg.HOST, - URL: url, - RawQuery: rawQuery, - Method: lg.METHOD, - Scheme: lg.Scheme, - REFERER: cutUTF8N(lg.REFERER, 1024), - USER_AGENT: cutUTF8N(lg.USER_AGENT, 500), - SRC_IP: lg.SRC_IP, - SRC_PORT: lg.SRC_PORT, - NetSrcIp: lg.NetSrcIp, - COUNTRY: lg.COUNTRY, - PROVINCE: lg.PROVINCE, - CITY: lg.CITY, - ACTION: lg.ACTION, - RULE: lg.RULE, - STATUS: lg.STATUS, - STATUS_CODE: lg.STATUS_CODE, - RISK_LEVEL: lg.RISK_LEVEL, - AI_SCORE: lg.AI_SCORE, - IsBot: lg.IsBot, - LogOnlyMode: lg.LogOnlyMode, + ReqUUID: lg.REQ_UUID, + TenantId: lg.TenantId, + UserCode: lg.USER_CODE, + HostCode: lg.HOST_CODE, + Host: lg.HOST, + URL: url, + RawQuery: rawQuery, + Method: lg.METHOD, + Scheme: lg.Scheme, + REFERER: cutUTF8N(lg.REFERER, 1024), + USER_AGENT: cutUTF8N(lg.USER_AGENT, 500), + SRC_IP: lg.SRC_IP, + SRC_PORT: lg.SRC_PORT, + NetSrcIp: lg.NetSrcIp, + COUNTRY: lg.COUNTRY, + PROVINCE: lg.PROVINCE, + CITY: lg.CITY, + ACTION: lg.ACTION, + RULE: lg.RULE, + STATUS: lg.STATUS, + STATUS_CODE: lg.STATUS_CODE, + RISK_LEVEL: lg.RISK_LEVEL, + AI_SCORE: lg.AI_SCORE, + IsBot: lg.IsBot, + LogOnlyMode: lg.LogOnlyMode, GUEST_IDENTIFICATION: lg.GUEST_IDENTIFICATION, - TimeSpent: lg.TimeSpent, - CONTENT_LENGTH: lg.CONTENT_LENGTH, - ResContentLength: lg.RES_CONTENT_LENGTH, - PreCheckCost: lg.PreCheckCost, - ForwardCost: lg.ForwardCost, - BackendCheckCost: lg.BackendCheckCost, - IsBalance: lg.IsBalance, - BalanceInfo: lg.BalanceInfo, - BodyHash: lg.BodyHash, - Truncated: truncated, - ActorKey: ActorKey(lg.SRC_IP), - UaHash: UaHash(lg.USER_AGENT), - PathNorm: NormalizePath(lg.URL), - CREATE_TIME: lg.CREATE_TIME, - UNIX_ADD_TIME: lg.UNIX_ADD_TIME, - Day: lg.Day, + TimeSpent: lg.TimeSpent, + CONTENT_LENGTH: lg.CONTENT_LENGTH, + ResContentLength: lg.RES_CONTENT_LENGTH, + PreCheckCost: lg.PreCheckCost, + ForwardCost: lg.ForwardCost, + BackendCheckCost: lg.BackendCheckCost, + IsBalance: lg.IsBalance, + BalanceInfo: lg.BalanceInfo, + BodyHash: lg.BodyHash, + Truncated: truncated, + ActorKey: ActorKey(lg.SRC_IP), + UaHash: UaHash(lg.USER_AGENT), + PathNorm: NormalizePath(lg.URL), + CREATE_TIME: lg.CREATE_TIME, + UNIX_ADD_TIME: lg.UNIX_ADD_TIME, + Day: lg.Day, } } diff --git a/wafqueue/log_tier_db_test.go b/wafqueue/log_tier_db_test.go index 44ca0c8..9919e51 100644 --- a/wafqueue/log_tier_db_test.go +++ b/wafqueue/log_tier_db_test.go @@ -91,6 +91,23 @@ func TestStoreTieredCrossEngine(t *testing.T) { if offAccess != 0 { t.Fatalf("off 档正常请求不应有窄行,实际 %d", offAccess) } + for _, mode := range []string{"db", "sample", "off"} { + global.GDATA_ACCESS_LOG_MODE = mode + uid := "cc-ban-" + mode + lg := mk(uid, "203.0.113.10", "阻止", "CC封禁") + lg.SkipPayloadStore = true + storeTiered([]*innerbean.WebLog{lg}) + for _, table := range []string{model.AccessLogTableName, model.SecurityEventTableName} { + var count int64 + if err := db.Table(table).Where("req_uuid = ?", uid).Count(&count).Error; err != nil || count != 1 { + t.Fatalf("%s %s narrow row: count=%d err=%v", mode, table, count, err) + } + } + var payloadCount int64 + if err := db.Model(&model.EventPayload{}).Where("req_uuid = ?", uid).Count(&payloadCount).Error; err != nil || payloadCount != 0 { + t.Fatalf("%s payload: count=%d err=%v", mode, payloadCount, err) + } + } global.GDATA_ACCESS_LOG_MODE = "db" }) } diff --git a/wafqueue/log_tier_test.go b/wafqueue/log_tier_test.go index 60aaa0c..84fbbdd 100644 --- a/wafqueue/log_tier_test.go +++ b/wafqueue/log_tier_test.go @@ -53,6 +53,25 @@ func TestTier_EventWritesAllLayers(t *testing.T) { } } +func TestTier_CCBanNarrowOnly(t *testing.T) { + for _, mode := range []string{"db", "sample", "off"} { + t.Run(mode, func(t *testing.T) { + defer resetTier(mode)() + lg := mkLog("cc-ban-"+mode, "203.0.113.10") + lg.ACTION = "阻止" + lg.RULE = "CC封禁" + lg.SkipPayloadStore = true + events, accesses, payloads := tierForStore([]*innerbean.WebLog{lg, lg}) + if len(events) != 1 || len(accesses) != 1 || len(payloads) != 0 { + t.Fatalf("CC narrow layers: %d/%d/%d", len(events), len(accesses), len(payloads)) + } + if events[0].ReqUUID != lg.REQ_UUID || accesses[0].ReqUUID != lg.REQ_UUID || lg.BODY != "a=1" || lg.HEADER != "User-Agent: curl" { + t.Fatal("identifiers or original payload changed") + } + }) + } +} + // db 档:正常请求窄行全量进 access_log;报文不进本批——被蓄水池采中的攒在池里定期刷库(D2) func TestTier_DBModeNormalNoPayload(t *testing.T) { defer resetTier("db")() diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 8c6ad9e..2faa32b 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -939,6 +939,35 @@ notes: the Access IP Details dialog on the risk log page gained a time range (last 30 days by default) so it no longer takes going through archives one by one. + - id: v1_3_25_cc_ban_request_logging + version: v1.3.25 + kind: notice + level: normal + page: /waf/wafvisitlog + doc: https://doc.samwaf.com/guide/BlockingPage.html + apply: + type: navigate + zh: + title: CC 封禁期间的阻断请求可按访问识别码查询 + detail: >- + CC 封禁期间,每次实际阻断请求保留独立的访问识别码,并记录来源 IP、站点、路径、方法、响应码与耗时等窄行信息。 + 这些事件同时出现在访问日志和安全事件中,不保存重复请求的完整报文;首次触发封禁的日志仍按现有规则保留报文。 + 封禁期间的请求数、拦截数与窄行磁盘占用会相应增加,通知仍沿用封禁提醒节流。 + 日志异步写入,查询需开启日志持久化、写入成功且记录尚在对应视图保留期内。 + 内置拦截页同时显示访客 IP;已有自定义模板可以自行加入 [[.SAMWAF_IP]],其值与日志来源 IP 一致。 + 顶部访问识别码查询忽略时间范围与所选分区,其余筛选条件仍生效;详情与请求复制会反馈未命中或读取错误。 + en: + title: Requests blocked during a CC ban can be looked up by visit identifier + detail: >- + Each request blocked during a CC ban retains its own visit identifier and a narrow log row containing source IP, + site, path, method, response status and duration. These events appear in both access logs and security events, + without retaining repeated full payloads. The request that initially triggers the ban keeps its payload under the existing rules. + Request counts, blocked counts and narrow-row disk usage increase accordingly; ban notifications retain their throttling. + Logs are written asynchronously and require persistence to be enabled, a successful write and an unexpired record in the selected view. + The built-in block page also shows the visitor IP. Existing custom templates can add [[.SAMWAF_IP]], using the same source IP as the logs. + The top visit-identifier lookup ignores the date range and selected partition while other filters still apply. + Request details and request-copy actions report missing records or read errors. + - id: v1_3_25_httpauth_path_scope version: v1.3.25 kind: notice