mirror of
https://gitee.com/samwaf/SamWaf.git
synced 2026-10-31 02:01:44 +08:00
samwafgo
64aaab16fa
* fix: make archived log shard connections concurrency-safe and recoverable * fix: build the WebSocket online table before its consumers start * perf: cap stored payloads, index log lookups by IP and rule, batch IP tag writes * fix: keep archived log shards readable when the live table gains a column * feat: merge IP tag history when their database location changes * refactor: move request payloads out of the access log into their own table * feat: scope log database export to a time range and selected tiers * feat: add an IP watchlist that full-captures watched IPs * feat: split log storage into security events, access rows, and payload tiers * feat: roll up requests per actor and path into daily analysis tables * fix:actor key by ip * feat: add a source and path analysis page over the daily rollups * feat: exclude chosen source IPs from web log recording * docs: add upgrade notes for the analysis page and log IP exclusion * feat: derive the log partition from the time range and look up identifiers across all of them * feat: add time partition verbs and the period naming they share * feat: cut log archives per month and expire them by dropping partitions * feat: manage log partitions, and answer why an IP has no logs * fix: read archived SQLite log shards read-only and choose tiers by data * fix: count the live log shard from access_log instead of a stale snapshot * fix:export db bug * fix: default backend keep-alive pool (512/128/90s) to stop connect-close churn under load * chore: gen_swagger.bat outputs json only, syncs doc site, runs route test * feat: pin web frontend version and embed it into the binary
86 lines
2.7 KiB
Go
86 lines
2.7 KiB
Go
package wafqueue
|
|
|
|
import (
|
|
"SamWaf/common/zlog"
|
|
"SamWaf/global"
|
|
"SamWaf/innerbean"
|
|
"SamWaf/wafipban"
|
|
"SamWaf/waftask"
|
|
"strconv"
|
|
"time"
|
|
)
|
|
|
|
/*
|
|
*
|
|
处理Log队列信息
|
|
*/
|
|
func ProcessLogDequeEngine() {
|
|
for {
|
|
select {
|
|
case <-global.GWAF_QUEUE_SHUTDOWN_SIGNAL:
|
|
zlog.Info("日志队列处理协程收到关闭信号，正在退出...")
|
|
return
|
|
default:
|
|
global.GWAF_MEASURE_PROCESS_DEQUEENGINE.WriteData(time.Now().UnixNano() / 1e6)
|
|
if global.GDATA_CURRENT_CHANGE {
|
|
//如果正在切换库 跳过
|
|
zlog.Debug("正在切换数据库等待中队列")
|
|
|
|
} else {
|
|
var webLogArray []*innerbean.WebLog
|
|
batchCount := 0
|
|
for !global.GQEQUE_LOG_DB.Empty() {
|
|
global.IncrementLogQPS() // 使用统一的日志QPS增量函数
|
|
weblogbean, ok := global.GQEQUE_LOG_DB.Dequeue()
|
|
if !ok {
|
|
continue
|
|
}
|
|
if weblogbean != nil {
|
|
// 进行类型断言将其转为具体的结构
|
|
if logValue, ok := weblogbean.(*innerbean.WebLog); ok {
|
|
webLogArray = append(webLogArray, logValue)
|
|
batchCount++
|
|
if batchCount > int(global.GDATA_BATCH_INSERT) {
|
|
break
|
|
}
|
|
} else {
|
|
//插入其他类型内容
|
|
global.GWAF_LOCAL_LOG_DB.Create(weblogbean)
|
|
}
|
|
}
|
|
}
|
|
if len(webLogArray) > 0 {
|
|
zlog.Debug("日志队列处理协程处理日志数量:" + strconv.Itoa(len(webLogArray)))
|
|
// 检查失败状态码并记录IP失败
|
|
if global.GCONFIG_IP_FAILURE_BAN_ENABLED == 1 {
|
|
ipManager := wafipban.GetIPFailureManager()
|
|
for _, log := range webLogArray {
|
|
if ipManager.IsFailureStatusCode(log.STATUS_CODE) {
|
|
ipManager.RecordFailure(log)
|
|
}
|
|
}
|
|
}
|
|
if global.GCONFIG_LOG_PERSIST_ENABLED == 1 {
|
|
// 分层落库：安全事件进 security_event（带报文），窄行进 access_log（按档位），
|
|
// web_logs 不再写入（旧数据只读到期删）。
|
|
// 全部在副本上动手，下面的统计与出口仍拿完整的原始对象。
|
|
storeTiered(webLogArray)
|
|
// 蓄水池采到的正常请求报文攒在内存里，到点整池刷成 event_payload(kind=sample)
|
|
negSampler.FlushIfDue()
|
|
}
|
|
// 日志流做统计
|
|
waftask.CollectStatsFromLogs(webLogArray)
|
|
// 分析层天级汇总（行为 / 目标 / 手法三视角）。键在本包算好再传过去：
|
|
// waftask 不能反向 import 本包，而键必须与窄行同源。
|
|
waftask.CollectAnalysisStats(AnalysisRowsFromLogs(webLogArray))
|
|
global.GNOTIFY_KAKFA_SERVICE.ProcessBatchLogs(webLogArray)
|
|
// 文件日志写入
|
|
global.GNOTIFY_LOG_FILE_WRITER.ProcessBatchLogs(webLogArray)
|
|
}
|
|
}
|
|
time.Sleep(100 * time.Millisecond)
|
|
global.GWAF_MEASURE_PROCESS_DEQUEENGINE.WriteData(time.Now().UnixNano() / 1e6)
|
|
}
|
|
}
|
|
}
|