From bedcbf5c04a1de456a81a6a19879c8b4b45324d8 Mon Sep 17 00:00:00 2001 From: samwaf Date: Thu, 23 Jan 2025 13:53:00 +0800 Subject: [PATCH] feat:add focus attack log #24 --- api/waf_log.go | 33 +++++++ global/config.go | 1 + global/global.go | 2 + innerbean/web_log.go | 4 + model/IPTag.go | 15 +++ model/request/waf_attack_log_req.go | 6 ++ router/waf_log.go | 2 + service/waf_service/waf_log.go | 136 ++++++++++++++++++++++++++++ utils/common.go | 11 +++ wafenginecore/wafengine.go | 12 ++- waftask/task_config.go | 5 + 11 files changed, 226 insertions(+), 1 deletion(-) diff --git a/api/waf_log.go b/api/waf_log.go index 89e42fe..c6d0f05 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -196,6 +196,39 @@ func (w *WafLogAPi) GetHttpCopyMaskApi(c *gin.Context) { response.FailWithMessage("解析失败", c) } } + +// GetAttackIPListApi 获取风险数据列表 +func (w *WafLogAPi) GetAttackIPListApi(c *gin.Context) { + var req request.WafAttackIpTagSearch + err := c.ShouldBindJSON(&req) + if err == nil { + ipAttackTags, total, err2 := wafLogService.GetAttackIpListApi(req) + if err2 != nil { + response.FailWithMessage("访问列表失败:"+err2.Error(), c) + } else { + response.OkWithDetailed(response.PageResult{ + List: ipAttackTags, + Total: total, + PageIndex: req.PageIndex, + PageSize: req.PageSize, + }, "获取成功", c) + } + + } else { + response.FailWithMessage("解析失败", c) + } +} + +// GetAllIpTagApi 获取所有ip tag +func (w *WafLogAPi) GetAllIpTagApi(c *gin.Context) { + + ipAttackTags, err2 := wafLogService.GetAllAttackIPTagListApi() + if err2 != nil { + response.FailWithMessage("访问ip tag 失败:"+err2.Error(), c) + } else { + response.OkWithDetailed(ipAttackTags, "获取成功", c) + } +} func GenerateCurlRequest(weblog innerbean.WebLog) string { headers := strings.Split(weblog.HEADER, "\n") diff --git a/global/config.go b/global/config.go index 7609edc..f5eb33f 100644 --- a/global/config.go +++ b/global/config.go @@ -22,4 +22,5 @@ var ( GCONFIG_RECORD_KEEPALIVE_TIME_OUT int64 = 30 // 保持活动超时 默认30s //GCONFIG_RECORD_PATCH_VERSION_CORE int64 = 20250106 // 核心数据库补丁日期 //GCONFIG_RECORD_PATCH_VERSION_LOG int64 = 20250106 // 日志数据库补丁日期 + GCONFIG_RECORD_ALL_SRC_BYTE_INFO int64 = 0 //记录原始信息(默认不开启) ) diff --git a/global/global.go b/global/global.go index 61f494f..7fd9ba6 100644 --- a/global/global.go +++ b/global/global.go @@ -108,6 +108,8 @@ var ( GNOTIFY_KAKFA_SERVICE *wafnotify.WafNotifyService //通知服务 GNOTIFY_SEND_MAX_LIMIT_MINTUTES = time.Duration(5) * time.Minute // 规则相关信息最大发送抑止 默认5分钟 + /*******日志记录相关*************/ + GWEBLOG_VERSION = 20250112 //weblog 日志版本号 /*********SSL相关*************/ GSSL_HTTP_CHANGLE_PATH string = "/.well-known/acme-challenge/" // http01证书验证路径 diff --git a/innerbean/web_log.go b/innerbean/web_log.go index b00b3c6..09f1ef6 100644 --- a/innerbean/web_log.go +++ b/innerbean/web_log.go @@ -33,6 +33,10 @@ type WebLog struct { RISK_LEVEL int `json:"risk_level"` //危险等级 0:正常 1:轻微 2:有害 3:严重 4:特别严重 GUEST_IDENTIFICATION string `json:"guest_identification"` //访客身份识别 TimeSpent int64 `json:"time_spent"` //用时 + NetSrcIp string `json:"net_src_ip"` //获取的原始IP + SrcByteBody []byte `json:"src_byte_body"` //原始body信息 + SrcByteResBody []byte `json:"src_byte_res_body"` //返回body bytes信息 + WebLogVersion int `json:"web_log_version"` //日志版本信息早期的是空和0，后期实时增加 } // 在 GORM 的 Model 方法中定义复合索引 diff --git a/model/IPTag.go b/model/IPTag.go index ce99343..04e2822 100644 --- a/model/IPTag.go +++ b/model/IPTag.go @@ -13,3 +13,18 @@ type IPTag struct { Cnt int64 `json:"cnt"` //触发次数 Remarks string `json:"remarks"` //备注 } +type AttackIPTag struct { + TenantID string `json:"tenant_id"` + UserCode string `json:"user_code"` + IP string `json:"ip"` + PassNum int64 `json:"pass_num"` + DenyNum int64 `json:"deny_num"` + FirstTime string `json:"first_time"` + LatestTime string `json:"latest_time"` + IpTotalTag string `json:"ip_total_tag"` +} + +type AllIPTag struct { + Label string `json:"label"` + Value string `json:"value"` +} diff --git a/model/request/waf_attack_log_req.go b/model/request/waf_attack_log_req.go index 66395de..22019f8 100644 --- a/model/request/waf_attack_log_req.go +++ b/model/request/waf_attack_log_req.go @@ -28,3 +28,9 @@ type WafAttackLogSearch struct { FilterValue string `json:"filter_value" form:"filter_value"` //筛选值 request.PageInfo } + +type WafAttackIpTagSearch struct { + Rule string `json:"rule" form:"rule"` //规则名 + SrcIp string `json:"src_ip" form:"src_ip"` //请求IP + request.PageInfo +} diff --git a/router/waf_log.go b/router/waf_log.go index c21fb40..3968309 100644 --- a/router/waf_log.go +++ b/router/waf_log.go @@ -18,4 +18,6 @@ func (receiver *LogRouter) InitLogRouter(group *gin.RouterGroup) { wafLogRouter.GET("/samwaf/waflog/attack/detail", logApi.GetDetailApi) wafLogRouter.GET("/samwaf/waflog/attack/allsharedb", logApi.GetAllShareDbApi) wafLogRouter.GET("/samwaf/waflog/attack/httpcopymask", logApi.GetHttpCopyMaskApi) + wafLogRouter.POST("/samwaf/waflog/attack/attackiplist", logApi.GetAttackIPListApi) + wafLogRouter.GET("/samwaf/waflog/attack/alliptag", logApi.GetAllIpTagApi) } diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index 736de25..fcfbe5f 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -4,6 +4,7 @@ import ( "SamWaf/common/validfield" "SamWaf/global" "SamWaf/innerbean" + "SamWaf/model" "SamWaf/model/request" "SamWaf/wafdb" "errors" @@ -182,6 +183,141 @@ func (receiver *WafLogService) GetUnixTimeByCounter(lastStartCreateUnix int64, l return weblog } +// GetAttackIpListApi 访问IP列表 +func (receiver *WafLogService) GetAttackIpListApi(req request.WafAttackIpTagSearch) ([]model.AttackIPTag, int64, error) { + var results []model.AttackIPTag + var total int64 + + // 基础查询部分 + query := ` + SELECT + tenant_id, + user_code, + ip, + SUM(CASE WHEN ip_tag = '正常' THEN cnt ELSE 0 END) AS pass_num, + SUM(CASE WHEN ip_tag <> '正常' THEN cnt ELSE 0 END) AS deny_num, + strftime('%Y-%m-%d %H:%M:%S', MIN(update_time)) AS first_time, + strftime('%Y-%m-%d %H:%M:%S', MAX(update_time)) AS latest_time, + GROUP_CONCAT(DISTINCT CASE WHEN ip_tag <> '正常' THEN ip_tag END) AS ip_total_tag + FROM + ip_tags + WHERE tenant_id=? and user_code=?` + + // 动态添加过滤条件 + if req.Rule != "" { + query += " AND ip_tag = ?" + } + if req.SrcIp != "" { + query += " AND ip = ?" + } + + // 完成查询的其他部分 + query += ` + GROUP BY + tenant_id, + user_code, + ip + HAVING + SUM(CASE WHEN ip_tag <> '正常' THEN cnt ELSE 0 END) > 0 + ORDER BY + MAX(update_time) DESC + LIMIT ? OFFSET ?` + + // 构建查询参数 + params := []interface{}{global.GWAF_TENANT_ID, global.GWAF_USER_CODE} + + // 添加 Rule 和 SrcIp 作为参数（如果提供了） + if req.Rule != "" { + params = append(params, req.Rule) + } + if req.SrcIp != "" { + params = append(params, req.SrcIp) + } + + // 分页参数 + params = append(params, req.PageSize, req.PageSize*(req.PageIndex-1)) + + // 执行查询 + if err := global.GWAF_LOCAL_DB.Raw(query, params...).Scan(&results).Error; err != nil { + return nil, 0, err + } + + // 获取总记录数 + countQuery := ` + SELECT + COUNT(*) AS total + FROM ( + SELECT + tenant_id, + user_code, + ip + FROM + ip_tags + WHERE tenant_id=? and user_code=?` + + // 动态添加过滤条件 + if req.Rule != "" { + countQuery += " AND ip_tag = ?" + } + if req.SrcIp != "" { + countQuery += " AND ip = ?" + } + + countQuery += ` + GROUP BY + tenant_id, + user_code, + ip + HAVING + SUM(CASE WHEN ip_tag <> '正常' THEN cnt ELSE 0 END) > 0 + ) AS subquery` + + // 获取总记录数参数 + countParams := []interface{}{global.GWAF_TENANT_ID, global.GWAF_USER_CODE} + if req.Rule != "" { + countParams = append(countParams, req.Rule) + } + if req.SrcIp != "" { + countParams = append(countParams, req.SrcIp) + } + + // 执行记录数查询 + if err := global.GWAF_LOCAL_DB.Raw(countQuery, countParams...).Scan(&total).Error; err != nil { + return nil, 0, err + } + + return results, total, nil +} + +// GetAllAttackIPTagListApi 获取所有攻击Tag +func (receiver *WafLogService) GetAllAttackIPTagListApi() ([]model.AllIPTag, error) { + var results []model.AllIPTag + + // 基础查询部分 + query := ` +SELECT + ip_tag as value, + ip_tag || ' (' || sum(cnt) || ')' as label + FROM + "ip_tags" +WHERE ip_tag<>'正常' and tenant_id=? and user_code=? + GROUP BY + tenant_id, + ip_tag +order by sum(cnt) desc +` + + // 构建查询参数 + params := []interface{}{global.GWAF_TENANT_ID, global.GWAF_USER_CODE} + + // 执行查询 + if err := global.GWAF_LOCAL_DB.Raw(query, params...).Scan(&results).Error; err != nil { + return nil, err + } + + return results, nil +} + /* * 判断是否合法 diff --git a/utils/common.go b/utils/common.go index a09338e..afdf6e7 100644 --- a/utils/common.go +++ b/utils/common.go @@ -425,3 +425,14 @@ func CheckSSLCertificateExpiry(host string) (time.Time, error) { // 返回到期时间 return expiryDate, nil } + +// GetSourceClientIP 获取原始IP +func GetSourceClientIP(ipAndPort string) string { + ip, _, err := net.SplitHostPort(ipAndPort) + if err != nil { + return "" + } else { + return ip + } + +} diff --git a/wafenginecore/wafengine.go b/wafenginecore/wafengine.go index 9cb076c..8233736 100644 --- a/wafenginecore/wafengine.go +++ b/wafenginecore/wafengine.go @@ -226,6 +226,9 @@ func (waf *WafEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { RISK_LEVEL: 0, //危险等级 GUEST_IDENTIFICATION: "正常访客", //访客身份识别 TimeSpent: 0, + NetSrcIp: utils.GetSourceClientIP(r.RemoteAddr), + SrcByteBody: bodyByte, + WebLogVersion: global.GWEBLOG_VERSION, } formValues := url.Values{} @@ -343,6 +346,10 @@ func (waf *WafEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } + //如果正常的流量不记录请求原始包 + if global.GCONFIG_RECORD_ALL_SRC_BYTE_INFO == 0 { + weblogbean.SrcByteBody = nil + } //基本验证是否开关是否开启 if hostTarget.Host.IsEnableHttpAuthBase == 1 { bHttpAuthBaseResult, sHttpAuthBaseResult := waf.DoHttpAuthBase(hostTarget, w, r) @@ -436,6 +443,8 @@ func (waf *WafEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { RISK_LEVEL: 1, //危险等级 GUEST_IDENTIFICATION: "未解析域名", //访客身份识别 TimeSpent: 0, + NetSrcIp: utils.GetSourceClientIP(r.RemoteAddr), + WebLogVersion: global.GWEBLOG_VERSION, } //记录响应body @@ -613,7 +622,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { } //文本资源 if respContentType == "text/html" || respContentType == "application/html" || - respContentType == "application/json" || respContentType == "application/xml" { + respContentType == "application/json" || respContentType == "application/xml" || respContentType == "text/plain" { contentType = "text" } //记录静态日志 @@ -650,6 +659,7 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { resp.Header.Set("Content-Length", strconv.FormatInt(int64(len(finalCompressBytes)), 10)) if resp.ContentLength < global.GCONFIG_RECORD_MAX_RES_BODY_LENGTH { weblogfrist.RES_BODY = string(orgContentBytes) + weblogfrist.SrcByteResBody = orgContentBytes } } zlog.Debug("TESTChanllage", weblogfrist.HOST, weblogfrist.URL) diff --git a/waftask/task_config.go b/waftask/task_config.go index e744773..93f22b1 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -51,6 +51,9 @@ func setConfigIntValue(name string, value int64, change int) { case "keepalive_time_out": global.GCONFIG_RECORD_KEEPALIVE_TIME_OUT = value break + case "record_all_src_byte_info": + global.GCONFIG_RECORD_ALL_SRC_BYTE_INFO = value + break default: zlog.Warn("Unknown config item:", name) } @@ -158,4 +161,6 @@ func TaskLoadSetting(initLoad bool) { updateConfigIntItem(initLoad, "network", "connect_time_out", global.GCONFIG_RECORD_CONNECT_TIME_OUT, "连接超时（默认30s）", "int", "") updateConfigIntItem(initLoad, "network", "keepalive_time_out", global.GCONFIG_RECORD_KEEPALIVE_TIME_OUT, "保持活动超时（默认30s）", "int", "") + updateConfigIntItem(initLoad, "system", "record_all_src_byte_info", global.GCONFIG_RECORD_ALL_SRC_BYTE_INFO, "启动记录原始请求BODY报文（1启动 0关闭）", "int", "") + }