mirror of
https://gitee.com/samwaf/SamWaf.git
synced 2026-09-01 15:32:55 +08:00
feat: nginx access
#IDNH2E
This commit is contained in:
@@ -52,6 +52,7 @@ type APIGroup struct {
|
||||
WafNotifyLogApi
|
||||
WafFirewallIPBlockApi
|
||||
WafPluginApi
|
||||
WafLogFileWriteApi
|
||||
}
|
||||
|
||||
var APIGroupAPP = new(APIGroup)
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"SamWaf/global"
|
||||
"SamWaf/model/common/response"
|
||||
"SamWaf/wafnotify/logfilewriter"
|
||||
"github.com/gin-gonic/gin"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type WafLogFileWriteApi struct {
|
||||
}
|
||||
|
||||
// GetPreviewApi 获取日志文件预览(最新N行)
|
||||
func (w *WafLogFileWriteApi) GetPreviewApi(c *gin.Context) {
|
||||
linesStr := c.DefaultQuery("lines", "100")
|
||||
lines, err := strconv.Atoi(linesStr)
|
||||
if err != nil || lines <= 0 {
|
||||
lines = 100
|
||||
}
|
||||
if lines > 500 {
|
||||
lines = 500
|
||||
}
|
||||
|
||||
if global.GNOTIFY_LOG_FILE_WRITER == nil {
|
||||
response.FailWithMessage("日志文件写入服务未初始化", c)
|
||||
return
|
||||
}
|
||||
|
||||
// 获取底层 notifier
|
||||
writer := getLogFileWriter()
|
||||
if writer == nil {
|
||||
response.FailWithMessage("日志文件写入服务未初始化", c)
|
||||
return
|
||||
}
|
||||
|
||||
preview, err := writer.GetLogPreview(lines)
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取日志预览失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithDetailed(preview, "获取成功", c)
|
||||
}
|
||||
|
||||
// GetCurrentFileInfoApi 获取当前日志文件信息
|
||||
func (w *WafLogFileWriteApi) GetCurrentFileInfoApi(c *gin.Context) {
|
||||
writer := getLogFileWriter()
|
||||
if writer == nil {
|
||||
response.FailWithMessage("日志文件写入服务未初始化", c)
|
||||
return
|
||||
}
|
||||
|
||||
fileInfo, err := writer.GetCurrentFileInfo()
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取文件信息失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithDetailed(fileInfo, "获取成功", c)
|
||||
}
|
||||
|
||||
// GetBackupFilesApi 获取备份文件列表
|
||||
func (w *WafLogFileWriteApi) GetBackupFilesApi(c *gin.Context) {
|
||||
writer := getLogFileWriter()
|
||||
if writer == nil {
|
||||
response.FailWithMessage("日志文件写入服务未初始化", c)
|
||||
return
|
||||
}
|
||||
|
||||
files, err := writer.GetBackupFiles()
|
||||
if err != nil {
|
||||
response.FailWithMessage("获取备份文件列表失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithDetailed(files, "获取成功", c)
|
||||
}
|
||||
|
||||
// ClearLogFileApi 清空当前日志文件
|
||||
func (w *WafLogFileWriteApi) ClearLogFileApi(c *gin.Context) {
|
||||
writer := getLogFileWriter()
|
||||
if writer == nil {
|
||||
response.FailWithMessage("日志文件写入服务未初始化", c)
|
||||
return
|
||||
}
|
||||
|
||||
err := writer.ClearLogFile()
|
||||
if err != nil {
|
||||
response.FailWithMessage("清空日志文件失败: "+err.Error(), c)
|
||||
return
|
||||
}
|
||||
|
||||
response.OkWithMessage("清空成功", c)
|
||||
}
|
||||
|
||||
// GetTemplateVariablesApi 获取可用的模板变量列表
|
||||
func (w *WafLogFileWriteApi) GetTemplateVariablesApi(c *gin.Context) {
|
||||
variables := logfilewriter.GetTemplateVariables()
|
||||
response.OkWithDetailed(variables, "获取成功", c)
|
||||
}
|
||||
|
||||
// getLogFileWriter 获取底层的 LogFileWriter
|
||||
func getLogFileWriter() *logfilewriter.LogFileWriter {
|
||||
if global.GNOTIFY_LOG_FILE_WRITER == nil {
|
||||
return nil
|
||||
}
|
||||
notifier := global.GNOTIFY_LOG_FILE_WRITER.GetNotifier()
|
||||
if notifier == nil {
|
||||
return nil
|
||||
}
|
||||
if writer, ok := notifier.(*logfilewriter.LogFileWriter); ok {
|
||||
return writer
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -279,6 +279,18 @@ func (m *wafSystenService) run() {
|
||||
waftask.TaskLoadSetting(true)
|
||||
//启动通知相关程序
|
||||
global.GNOTIFY_KAKFA_SERVICE = wafnotify.InitNotifyKafkaEngine(global.GCONFIG_RECORD_KAFKA_ENABLE, global.GCONFIG_RECORD_KAFKA_URL, global.GCONFIG_RECORD_KAFKA_TOPIC) //kafka
|
||||
// 日志文件写入
|
||||
compressFlag := global.GCONFIG_LOG_FILE_WRITE_COMPRESS == 1
|
||||
global.GNOTIFY_LOG_FILE_WRITER = wafnotify.InitLogFileWriterEngine(
|
||||
global.GCONFIG_LOG_FILE_WRITE_ENABLE,
|
||||
global.GCONFIG_LOG_FILE_WRITE_PATH,
|
||||
global.GCONFIG_LOG_FILE_WRITE_FORMAT,
|
||||
global.GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL,
|
||||
global.GCONFIG_LOG_FILE_WRITE_MAX_SIZE,
|
||||
int(global.GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS),
|
||||
int(global.GCONFIG_LOG_FILE_WRITE_MAX_DAYS),
|
||||
compressFlag,
|
||||
)
|
||||
//启动waf
|
||||
globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE = &wafenginecore.WafEngine{
|
||||
HostTarget: map[string]*wafenginmodel.HostSafe{},
|
||||
|
||||
@@ -70,4 +70,14 @@ var (
|
||||
GCONFIG_ZEROSSL_EAB_KID string = "" // zerossl eab_kid
|
||||
GCONFIG_ZEROSSL_EAB_HMAC_KEY string = "" // zerossl eab_hmac_key
|
||||
|
||||
// 日志文件写入配置 (额外输出,不影响SQLite存储)
|
||||
GCONFIG_LOG_FILE_WRITE_ENABLE int64 = 0 // 日志文件写入开关 (0关闭 1开启)
|
||||
GCONFIG_LOG_FILE_WRITE_PATH string = "logs/access.log" // 日志文件路径
|
||||
GCONFIG_LOG_FILE_WRITE_FORMAT string = "nginx" // 日志格式: nginx, apache, custom
|
||||
GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL string = "" // 自定义格式模板
|
||||
GCONFIG_LOG_FILE_WRITE_MAX_SIZE int64 = 100 // 单个日志文件最大大小 (MB)
|
||||
GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS int64 = 10 // 保留的历史文件数量
|
||||
GCONFIG_LOG_FILE_WRITE_MAX_DAYS int64 = 30 // 保留天数
|
||||
GCONFIG_LOG_FILE_WRITE_COMPRESS int64 = 0 // 是否压缩历史文件 (0关闭 1开启)
|
||||
|
||||
)
|
||||
|
||||
@@ -136,6 +136,7 @@ var (
|
||||
|
||||
/*******通知相关*************/
|
||||
GNOTIFY_KAKFA_SERVICE *wafnotify.WafNotifyService //通知服务
|
||||
GNOTIFY_LOG_FILE_WRITER *wafnotify.WafNotifyService //日志文件写入服务
|
||||
GNOTIFY_SEND_MAX_LIMIT_MINTUTES = time.Duration(5) * time.Minute // 规则相关信息最大发送抑止 默认5分钟
|
||||
|
||||
/*******日志记录相关*************/
|
||||
|
||||
@@ -50,6 +50,7 @@ type ApiGroup struct {
|
||||
NotifyLogRouter
|
||||
FirewallIPBlockRouter
|
||||
PluginRouter
|
||||
LogFileWriteRouter
|
||||
}
|
||||
type PublicApiGroup struct {
|
||||
LoginRouter
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"SamWaf/api"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type LogFileWriteRouter struct {
|
||||
}
|
||||
|
||||
func (receiver *LogFileWriteRouter) InitLogFileWriteRouter(group *gin.RouterGroup) {
|
||||
api := api.APIGroupAPP.WafLogFileWriteApi
|
||||
router := group.Group("")
|
||||
router.GET("/api/v1/logfilewrite/preview", api.GetPreviewApi)
|
||||
router.GET("/api/v1/logfilewrite/currentfile", api.GetCurrentFileInfoApi)
|
||||
router.GET("/api/v1/logfilewrite/backupfiles", api.GetBackupFilesApi)
|
||||
router.POST("/api/v1/logfilewrite/clear", api.ClearLogFileApi)
|
||||
router.GET("/api/v1/logfilewrite/variables", api.GetTemplateVariablesApi)
|
||||
}
|
||||
@@ -91,6 +91,7 @@ func (web *WafWebManager) initRouter(r *gin.Engine) {
|
||||
router.ApiGroupApp.InitNotifySubscriptionRouter(RouterGroup)
|
||||
router.ApiGroupApp.InitNotifyLogRouter(RouterGroup)
|
||||
router.ApiGroupApp.InitFirewallIPBlockRouter(RouterGroup)
|
||||
router.ApiGroupApp.InitLogFileWriteRouter(RouterGroup)
|
||||
}
|
||||
|
||||
if global.GWAF_RELEASE == "true" {
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package logfilewriter
|
||||
|
||||
import (
|
||||
"SamWaf/innerbean"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// 预定义格式模板
|
||||
const (
|
||||
// NginxCombinedFormat nginx combined 格式
|
||||
NginxCombinedFormat = `${src_ip} - - [${create_time}] "${method} ${url} HTTP/1.1" ${status_code} ${content_length} "${referer}" "${user_agent}"`
|
||||
|
||||
// ApacheCombinedFormat apache combined 格式
|
||||
ApacheCombinedFormat = `${src_ip} - - [${create_time}] "${method} ${url} HTTP/1.1" ${status_code} ${content_length} "${referer}" "${user_agent}"`
|
||||
)
|
||||
|
||||
// GetFormatTemplate 根据格式名称获取模板
|
||||
func GetFormatTemplate(format string) string {
|
||||
switch strings.ToLower(format) {
|
||||
case "nginx":
|
||||
return NginxCombinedFormat
|
||||
case "apache":
|
||||
return ApacheCombinedFormat
|
||||
case "custom":
|
||||
return "" // 自定义格式由用户提供
|
||||
default:
|
||||
return NginxCombinedFormat
|
||||
}
|
||||
}
|
||||
|
||||
// FormatLog 将 WebLog 格式化为指定模板的字符串
|
||||
func FormatLog(log *innerbean.WebLog, template string) string {
|
||||
if template == "" {
|
||||
template = NginxCombinedFormat
|
||||
}
|
||||
|
||||
result := template
|
||||
|
||||
// 替换所有模板变量
|
||||
result = strings.ReplaceAll(result, "${src_ip}", log.SRC_IP)
|
||||
result = strings.ReplaceAll(result, "${src_port}", log.SRC_PORT)
|
||||
result = strings.ReplaceAll(result, "${host}", log.HOST)
|
||||
result = strings.ReplaceAll(result, "${url}", log.URL)
|
||||
result = strings.ReplaceAll(result, "${raw_query}", log.RawQuery)
|
||||
result = strings.ReplaceAll(result, "${method}", log.METHOD)
|
||||
result = strings.ReplaceAll(result, "${scheme}", log.Scheme)
|
||||
result = strings.ReplaceAll(result, "${referer}", log.REFERER)
|
||||
result = strings.ReplaceAll(result, "${user_agent}", log.USER_AGENT)
|
||||
result = strings.ReplaceAll(result, "${status_code}", strconv.Itoa(log.STATUS_CODE))
|
||||
result = strings.ReplaceAll(result, "${content_length}", strconv.FormatInt(log.CONTENT_LENGTH, 10))
|
||||
result = strings.ReplaceAll(result, "${create_time}", log.CREATE_TIME)
|
||||
result = strings.ReplaceAll(result, "${time_spent}", strconv.FormatInt(log.TimeSpent, 10))
|
||||
result = strings.ReplaceAll(result, "${country}", log.COUNTRY)
|
||||
result = strings.ReplaceAll(result, "${province}", log.PROVINCE)
|
||||
result = strings.ReplaceAll(result, "${city}", log.CITY)
|
||||
result = strings.ReplaceAll(result, "${action}", log.ACTION)
|
||||
result = strings.ReplaceAll(result, "${rule}", log.RULE)
|
||||
result = strings.ReplaceAll(result, "${risk_level}", strconv.Itoa(log.RISK_LEVEL))
|
||||
result = strings.ReplaceAll(result, "${cookies}", log.COOKIES)
|
||||
result = strings.ReplaceAll(result, "${req_uuid}", log.REQ_UUID)
|
||||
result = strings.ReplaceAll(result, "${is_bot}", strconv.Itoa(log.IsBot))
|
||||
result = strings.ReplaceAll(result, "${guest_identification}", log.GUEST_IDENTIFICATION)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// TemplateVariables 返回所有可用的模板变量信息(供前端展示)
|
||||
type TemplateVariable struct {
|
||||
Name string `json:"name"` // 变量名
|
||||
Field string `json:"field"` // 对应字段
|
||||
Desc string `json:"desc"` // 说明
|
||||
Example string `json:"example"` // 示例值
|
||||
}
|
||||
|
||||
// GetTemplateVariables 返回所有可用的模板变量
|
||||
func GetTemplateVariables() []TemplateVariable {
|
||||
return []TemplateVariable{
|
||||
{Name: "${src_ip}", Field: "SRC_IP", Desc: "客户端IP", Example: "192.168.1.100"},
|
||||
{Name: "${src_port}", Field: "SRC_PORT", Desc: "客户端端口", Example: "52341"},
|
||||
{Name: "${host}", Field: "HOST", Desc: "请求主机名", Example: "www.example.com"},
|
||||
{Name: "${url}", Field: "URL", Desc: "请求URL", Example: "/api/user?id=1"},
|
||||
{Name: "${raw_query}", Field: "RawQuery", Desc: "URL查询参数", Example: "id=1&name=test"},
|
||||
{Name: "${method}", Field: "METHOD", Desc: "请求方法", Example: "GET"},
|
||||
{Name: "${scheme}", Field: "Scheme", Desc: "协议", Example: "https"},
|
||||
{Name: "${referer}", Field: "REFERER", Desc: "Referer头", Example: "https://example.com/"},
|
||||
{Name: "${user_agent}", Field: "USER_AGENT", Desc: "User-Agent", Example: "Mozilla/5.0..."},
|
||||
{Name: "${status_code}", Field: "STATUS_CODE", Desc: "HTTP状态码", Example: "200"},
|
||||
{Name: "${content_length}", Field: "CONTENT_LENGTH", Desc: "响应大小", Example: "2048"},
|
||||
{Name: "${create_time}", Field: "CREATE_TIME", Desc: "请求时间", Example: "2026-02-09 10:00:00"},
|
||||
{Name: "${time_spent}", Field: "TimeSpent", Desc: "耗时(ms)", Example: "125"},
|
||||
{Name: "${country}", Field: "COUNTRY", Desc: "国家", Example: "中国"},
|
||||
{Name: "${province}", Field: "PROVINCE", Desc: "省份", Example: "广东省"},
|
||||
{Name: "${city}", Field: "CITY", Desc: "城市", Example: "深圳市"},
|
||||
{Name: "${action}", Field: "ACTION", Desc: "防御动作", Example: "放行"},
|
||||
{Name: "${rule}", Field: "RULE", Desc: "触发规则", Example: "SQL注入检测"},
|
||||
{Name: "${risk_level}", Field: "RISK_LEVEL", Desc: "风险等级", Example: "0"},
|
||||
{Name: "${cookies}", Field: "COOKIES", Desc: "Cookies", Example: "session=abc123"},
|
||||
{Name: "${req_uuid}", Field: "REQ_UUID", Desc: "请求ID", Example: "a1b2c3d4-..."},
|
||||
{Name: "${is_bot}", Field: "IsBot", Desc: "是否机器人", Example: "0"},
|
||||
{Name: "${guest_identification}", Field: "GUEST_IDENTIFICATION", Desc: "访客标识", Example: "Googlebot"},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,539 @@
|
||||
package logfilewriter
|
||||
|
||||
import (
|
||||
"SamWaf/common/zlog"
|
||||
"SamWaf/innerbean"
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LogFileWriter 实现 WafNotify 接口,将日志写入文件
|
||||
type LogFileWriter struct {
|
||||
mu sync.Mutex
|
||||
filePath string // 日志文件路径
|
||||
format string // 日志格式: nginx, apache, custom
|
||||
customTpl string // 自定义格式模板
|
||||
maxSize int64 // 单文件最大大小 (MB)
|
||||
maxBackups int // 保留的历史文件数量
|
||||
maxDays int // 保留天数
|
||||
compress bool // 是否压缩历史文件
|
||||
file *os.File
|
||||
writer *bufio.Writer
|
||||
currentSize int64 // 当前文件大小
|
||||
template string // 解析后的模板
|
||||
}
|
||||
|
||||
// NewLogFileWriter 创建日志文件写入器
|
||||
func NewLogFileWriter(filePath, format, customTpl string, maxSize int64, maxBackups int, maxDays int, compress bool) (*LogFileWriter, error) {
|
||||
lw := &LogFileWriter{
|
||||
filePath: filePath,
|
||||
format: format,
|
||||
customTpl: customTpl,
|
||||
maxSize: maxSize,
|
||||
maxBackups: maxBackups,
|
||||
maxDays: maxDays,
|
||||
compress: compress,
|
||||
}
|
||||
|
||||
// 解析模板
|
||||
lw.resolveTemplate()
|
||||
|
||||
// 打开文件
|
||||
if err := lw.openFile(); err != nil {
|
||||
return lw, err
|
||||
}
|
||||
|
||||
return lw, nil
|
||||
}
|
||||
|
||||
// resolveTemplate 解析格式模板
|
||||
func (lw *LogFileWriter) resolveTemplate() {
|
||||
if lw.format == "custom" && lw.customTpl != "" {
|
||||
lw.template = lw.customTpl
|
||||
} else {
|
||||
lw.template = GetFormatTemplate(lw.format)
|
||||
}
|
||||
}
|
||||
|
||||
// openFile 打开或创建日志文件
|
||||
func (lw *LogFileWriter) openFile() error {
|
||||
// 确保目录存在
|
||||
dir := filepath.Dir(lw.filePath)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return fmt.Errorf("创建日志目录失败: %v", err)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(lw.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("打开日志文件失败: %v", err)
|
||||
}
|
||||
|
||||
// 获取当前文件大小
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("获取文件信息失败: %v", err)
|
||||
}
|
||||
|
||||
lw.file = f
|
||||
lw.writer = bufio.NewWriterSize(f, 64*1024) // 64KB缓冲区
|
||||
lw.currentSize = info.Size()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NotifySingle 实现 WafNotify 接口 - 写入单条日志
|
||||
func (lw *LogFileWriter) NotifySingle(log *innerbean.WebLog) error {
|
||||
lw.mu.Lock()
|
||||
defer lw.mu.Unlock()
|
||||
|
||||
if lw.file == nil {
|
||||
if err := lw.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
line := FormatLog(log, lw.template) + "\n"
|
||||
n, err := lw.writer.WriteString(line)
|
||||
if err != nil {
|
||||
return fmt.Errorf("写入日志失败: %v", err)
|
||||
}
|
||||
lw.currentSize += int64(n)
|
||||
|
||||
// 检查是否需要轮转
|
||||
if lw.needsRotation() {
|
||||
if err := lw.rotate(); err != nil {
|
||||
zlog.Error("日志文件轮转失败: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NotifyBatch 实现 WafNotify 接口 - 批量写入日志
|
||||
func (lw *LogFileWriter) NotifyBatch(logs []*innerbean.WebLog) error {
|
||||
lw.mu.Lock()
|
||||
defer lw.mu.Unlock()
|
||||
|
||||
if lw.file == nil {
|
||||
if err := lw.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, log := range logs {
|
||||
line := FormatLog(log, lw.template) + "\n"
|
||||
n, err := lw.writer.WriteString(line)
|
||||
if err != nil {
|
||||
return fmt.Errorf("写入日志失败: %v", err)
|
||||
}
|
||||
lw.currentSize += int64(n)
|
||||
}
|
||||
|
||||
// 刷新缓冲区
|
||||
if err := lw.writer.Flush(); err != nil {
|
||||
zlog.Error("刷新日志缓冲区失败: " + err.Error())
|
||||
}
|
||||
|
||||
// 检查是否需要轮转
|
||||
if lw.needsRotation() {
|
||||
if err := lw.rotate(); err != nil {
|
||||
zlog.Error("日志文件轮转失败: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// needsRotation 检查是否需要轮转
|
||||
func (lw *LogFileWriter) needsRotation() bool {
|
||||
maxBytes := lw.maxSize * 1024 * 1024
|
||||
return lw.currentSize >= maxBytes
|
||||
}
|
||||
|
||||
// rotate 执行日志文件轮转
|
||||
func (lw *LogFileWriter) rotate() error {
|
||||
// 刷新并关闭当前文件
|
||||
if lw.writer != nil {
|
||||
lw.writer.Flush()
|
||||
}
|
||||
if lw.file != nil {
|
||||
lw.file.Close()
|
||||
lw.file = nil
|
||||
lw.writer = nil
|
||||
}
|
||||
|
||||
// 轮转文件: access.log -> access.log.1.log -> access.log.2.log ...
|
||||
// 先移动已有的备份文件
|
||||
for i := lw.maxBackups; i >= 1; i-- {
|
||||
src := lw.backupName(i)
|
||||
dst := lw.backupName(i + 1)
|
||||
if _, err := os.Stat(src); err == nil {
|
||||
if i == lw.maxBackups {
|
||||
// 最大编号的文件直接删除
|
||||
os.Remove(src)
|
||||
} else {
|
||||
os.Rename(src, dst)
|
||||
}
|
||||
}
|
||||
// 同时处理压缩文件
|
||||
srcGz := src + ".gz"
|
||||
dstGz := dst + ".gz"
|
||||
if _, err := os.Stat(srcGz); err == nil {
|
||||
if i == lw.maxBackups {
|
||||
os.Remove(srcGz)
|
||||
} else {
|
||||
os.Rename(srcGz, dstGz)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 当前文件重命名为 .1.log
|
||||
backup1 := lw.backupName(1)
|
||||
if err := os.Rename(lw.filePath, backup1); err != nil {
|
||||
zlog.Error("重命名日志文件失败: " + err.Error())
|
||||
}
|
||||
|
||||
// 压缩刚轮转的文件
|
||||
if lw.compress {
|
||||
go lw.compressFile(backup1)
|
||||
}
|
||||
|
||||
// 清理过期文件
|
||||
go lw.cleanOldFiles()
|
||||
|
||||
// 打开新文件
|
||||
return lw.openFile()
|
||||
}
|
||||
|
||||
// backupName 生成备份文件名
|
||||
// 例如: logs/access.log -> logs/access.log.1.log
|
||||
func (lw *LogFileWriter) backupName(index int) string {
|
||||
return lw.filePath + "." + strconv.Itoa(index) + ".log"
|
||||
}
|
||||
|
||||
// compressFile 压缩文件
|
||||
func (lw *LogFileWriter) compressFile(srcPath string) {
|
||||
src, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
zlog.Error("打开待压缩文件失败: " + err.Error())
|
||||
return
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dstPath := srcPath + ".gz"
|
||||
dst, err := os.Create(dstPath)
|
||||
if err != nil {
|
||||
zlog.Error("创建压缩文件失败: " + err.Error())
|
||||
return
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
gz := gzip.NewWriter(dst)
|
||||
defer gz.Close()
|
||||
|
||||
if _, err := io.Copy(gz, src); err != nil {
|
||||
zlog.Error("压缩文件失败: " + err.Error())
|
||||
os.Remove(dstPath)
|
||||
return
|
||||
}
|
||||
|
||||
gz.Close()
|
||||
dst.Close()
|
||||
src.Close()
|
||||
|
||||
// 删除原始文件
|
||||
os.Remove(srcPath)
|
||||
}
|
||||
|
||||
// cleanOldFiles 清理超期文件
|
||||
func (lw *LogFileWriter) cleanOldFiles() {
|
||||
dir := filepath.Dir(lw.filePath)
|
||||
baseName := filepath.Base(lw.filePath)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
zlog.Error("读取日志目录失败: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
maxAge := time.Duration(lw.maxDays) * 24 * time.Hour
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
// 匹配备份文件: baseName.N.log 或 baseName.N.log.gz
|
||||
if !strings.HasPrefix(name, baseName+".") {
|
||||
continue
|
||||
}
|
||||
// 跳过当前日志文件本身
|
||||
if name == baseName {
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// 按天数清理
|
||||
if lw.maxDays > 0 && now.Sub(info.ModTime()) > maxAge {
|
||||
fullPath := filepath.Join(dir, name)
|
||||
os.Remove(fullPath)
|
||||
zlog.Info("清理过期日志文件: " + fullPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close 关闭日志文件
|
||||
func (lw *LogFileWriter) Close() {
|
||||
lw.mu.Lock()
|
||||
defer lw.mu.Unlock()
|
||||
|
||||
if lw.writer != nil {
|
||||
lw.writer.Flush()
|
||||
}
|
||||
if lw.file != nil {
|
||||
lw.file.Close()
|
||||
lw.file = nil
|
||||
lw.writer = nil
|
||||
}
|
||||
}
|
||||
|
||||
// GetLogPreview 获取日志文件最新N行(供前端预览)
|
||||
func (lw *LogFileWriter) GetLogPreview(lines int) ([]string, error) {
|
||||
lw.mu.Lock()
|
||||
// 先刷新缓冲区以确保读到最新内容
|
||||
if lw.writer != nil {
|
||||
lw.writer.Flush()
|
||||
}
|
||||
lw.mu.Unlock()
|
||||
|
||||
f, err := os.Open(lw.filePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []string{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
return readLastLines(f, lines)
|
||||
}
|
||||
|
||||
// readLastLines 读取文件最后N行
|
||||
func readLastLines(f *os.File, n int) ([]string, error) {
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fileSize := info.Size()
|
||||
if fileSize == 0 {
|
||||
return []string{}, nil
|
||||
}
|
||||
|
||||
// 从文件末尾向前读取
|
||||
bufSize := int64(8192)
|
||||
if bufSize > fileSize {
|
||||
bufSize = fileSize
|
||||
}
|
||||
|
||||
var allLines []string
|
||||
offset := fileSize
|
||||
remaining := ""
|
||||
|
||||
for offset > 0 {
|
||||
readSize := bufSize
|
||||
if readSize > offset {
|
||||
readSize = offset
|
||||
}
|
||||
offset -= readSize
|
||||
|
||||
buf := make([]byte, readSize)
|
||||
_, err := f.ReadAt(buf, offset)
|
||||
if err != nil && err != io.EOF {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
chunk := string(buf) + remaining
|
||||
lines := strings.Split(chunk, "\n")
|
||||
|
||||
// 第一段可能不完整,保留到下次
|
||||
remaining = lines[0]
|
||||
lines = lines[1:]
|
||||
|
||||
// 逆序添加
|
||||
for i := len(lines) - 1; i >= 0; i-- {
|
||||
line := strings.TrimRight(lines[i], "\r")
|
||||
if line != "" {
|
||||
allLines = append([]string{line}, allLines...)
|
||||
}
|
||||
}
|
||||
|
||||
if len(allLines) >= n {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 处理最后剩余的部分
|
||||
if remaining != "" && len(allLines) < n {
|
||||
remaining = strings.TrimRight(remaining, "\r")
|
||||
if remaining != "" {
|
||||
allLines = append([]string{remaining}, allLines...)
|
||||
}
|
||||
}
|
||||
|
||||
// 只返回最后N行
|
||||
if len(allLines) > n {
|
||||
allLines = allLines[len(allLines)-n:]
|
||||
}
|
||||
|
||||
return allLines, nil
|
||||
}
|
||||
|
||||
// BackupFileInfo 备份文件信息
|
||||
type BackupFileInfo struct {
|
||||
Name string `json:"name"` // 文件名
|
||||
Size int64 `json:"size"` // 文件大小 (bytes)
|
||||
ModTime string `json:"mod_time"` // 修改时间
|
||||
FullPath string `json:"full_path"` // 完整路径
|
||||
}
|
||||
|
||||
// GetBackupFiles 获取备份文件列表
|
||||
func (lw *LogFileWriter) GetBackupFiles() ([]BackupFileInfo, error) {
|
||||
dir := filepath.Dir(lw.filePath)
|
||||
baseName := filepath.Base(lw.filePath)
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []BackupFileInfo{}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var files []BackupFileInfo
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
// 匹配备份文件
|
||||
if !strings.HasPrefix(name, baseName+".") {
|
||||
continue
|
||||
}
|
||||
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
files = append(files, BackupFileInfo{
|
||||
Name: name,
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime().Format("2006-01-02 15:04:05"),
|
||||
FullPath: filepath.Join(dir, name),
|
||||
})
|
||||
}
|
||||
|
||||
// 按修改时间倒序排列
|
||||
sort.Slice(files, func(i, j int) bool {
|
||||
return files[i].ModTime > files[j].ModTime
|
||||
})
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// GetCurrentFileInfo 获取当前日志文件信息
|
||||
func (lw *LogFileWriter) GetCurrentFileInfo() (*BackupFileInfo, error) {
|
||||
info, err := os.Stat(lw.filePath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return &BackupFileInfo{
|
||||
Name: filepath.Base(lw.filePath),
|
||||
Size: 0,
|
||||
ModTime: "",
|
||||
FullPath: lw.filePath,
|
||||
}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &BackupFileInfo{
|
||||
Name: info.Name(),
|
||||
Size: info.Size(),
|
||||
ModTime: info.ModTime().Format("2006-01-02 15:04:05"),
|
||||
FullPath: lw.filePath,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ClearLogFile 清空当前日志文件
|
||||
func (lw *LogFileWriter) ClearLogFile() error {
|
||||
lw.mu.Lock()
|
||||
defer lw.mu.Unlock()
|
||||
|
||||
// 关闭当前文件
|
||||
if lw.writer != nil {
|
||||
lw.writer.Flush()
|
||||
}
|
||||
if lw.file != nil {
|
||||
lw.file.Close()
|
||||
lw.file = nil
|
||||
lw.writer = nil
|
||||
}
|
||||
|
||||
// 清空文件
|
||||
if err := os.Truncate(lw.filePath, 0); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 重新打开
|
||||
return lw.openFile()
|
||||
}
|
||||
|
||||
// UpdateConfig 更新配置(运行时动态修改)
|
||||
func (lw *LogFileWriter) UpdateConfig(filePath, format, customTpl string, maxSize int64, maxBackups int, maxDays int, compress bool) {
|
||||
lw.mu.Lock()
|
||||
defer lw.mu.Unlock()
|
||||
|
||||
pathChanged := lw.filePath != filePath
|
||||
|
||||
lw.format = format
|
||||
lw.customTpl = customTpl
|
||||
lw.maxSize = maxSize
|
||||
lw.maxBackups = maxBackups
|
||||
lw.maxDays = maxDays
|
||||
lw.compress = compress
|
||||
lw.resolveTemplate()
|
||||
|
||||
// 如果路径改变,重新打开文件
|
||||
if pathChanged {
|
||||
if lw.writer != nil {
|
||||
lw.writer.Flush()
|
||||
}
|
||||
if lw.file != nil {
|
||||
lw.file.Close()
|
||||
lw.file = nil
|
||||
lw.writer = nil
|
||||
}
|
||||
lw.filePath = filePath
|
||||
if err := lw.openFile(); err != nil {
|
||||
zlog.Error("更新配置后打开新日志文件失败: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package wafnotify
|
||||
|
||||
import (
|
||||
"SamWaf/wafnotify/kafka"
|
||||
"SamWaf/wafnotify/logfilewriter"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
@@ -17,3 +18,13 @@ func InitNotifyKafkaEngine(enable int64, url string, topic string) *WafNotifySer
|
||||
// 创建日志服务并注入 notifier
|
||||
return NewWafNotifyService(notifier, enable)
|
||||
}
|
||||
|
||||
// InitLogFileWriterEngine 初始化日志文件写入引擎
|
||||
func InitLogFileWriterEngine(enable int64, filePath, format, customTpl string, maxSize int64, maxBackups int, maxDays int, compress bool) *WafNotifyService {
|
||||
notifier, err := logfilewriter.NewLogFileWriter(filePath, format, customTpl, maxSize, maxBackups, maxDays, compress)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to create log file writer: %v\n", err)
|
||||
return NewWafNotifyService(notifier, enable)
|
||||
}
|
||||
return NewWafNotifyService(notifier, enable)
|
||||
}
|
||||
|
||||
@@ -26,6 +26,14 @@ func (ls *WafNotifyService) ChangeEnable(enable int64) {
|
||||
ls.enable = enable
|
||||
}
|
||||
|
||||
// GetNotifier 获取底层通知器
|
||||
func (ls *WafNotifyService) GetNotifier() wafinterface.WafNotify {
|
||||
if ls == nil {
|
||||
return nil
|
||||
}
|
||||
return ls.notifier
|
||||
}
|
||||
|
||||
// 处理并发送单条日志
|
||||
func (ls *WafNotifyService) ProcessSingleLog(log *innerbean.WebLog) error {
|
||||
if ls.enable == 0 {
|
||||
|
||||
@@ -83,6 +83,8 @@ func ProcessLogDequeEngine() {
|
||||
// 日志流做统计
|
||||
waftask.CollectStatsFromLogs(webLogArray)
|
||||
global.GNOTIFY_KAKFA_SERVICE.ProcessBatchLogs(webLogArray)
|
||||
// 文件日志写入 (额外输出)
|
||||
global.GNOTIFY_LOG_FILE_WRITER.ProcessBatchLogs(webLogArray)
|
||||
}
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
@@ -130,6 +130,19 @@ func setConfigIntValue(name string, value int64, change int) {
|
||||
global.GCONFIG_ENABLE_HTTP3 = value
|
||||
case "record_log_desensitize":
|
||||
global.GCONFIG_RECORD_LOG_DESENSITIZE = value
|
||||
case "log_file_write_enable":
|
||||
if global.GCONFIG_LOG_FILE_WRITE_ENABLE != value && global.GNOTIFY_LOG_FILE_WRITER != nil {
|
||||
global.GNOTIFY_LOG_FILE_WRITER.ChangeEnable(value)
|
||||
}
|
||||
global.GCONFIG_LOG_FILE_WRITE_ENABLE = value
|
||||
case "log_file_write_max_size":
|
||||
global.GCONFIG_LOG_FILE_WRITE_MAX_SIZE = value
|
||||
case "log_file_write_max_backups":
|
||||
global.GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS = value
|
||||
case "log_file_write_max_days":
|
||||
global.GCONFIG_LOG_FILE_WRITE_MAX_DAYS = value
|
||||
case "log_file_write_compress":
|
||||
global.GCONFIG_LOG_FILE_WRITE_COMPRESS = value
|
||||
default:
|
||||
zlog.Warn("Unknown config item:", name)
|
||||
}
|
||||
@@ -194,6 +207,12 @@ func setConfigStringValue(name string, value string, change int) {
|
||||
case "zerossl_eab_hmac_key":
|
||||
global.GCONFIG_ZEROSSL_EAB_HMAC_KEY = value
|
||||
break
|
||||
case "log_file_write_path":
|
||||
global.GCONFIG_LOG_FILE_WRITE_PATH = value
|
||||
case "log_file_write_format":
|
||||
global.GCONFIG_LOG_FILE_WRITE_FORMAT = value
|
||||
case "log_file_write_custom_tpl":
|
||||
global.GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL = value
|
||||
default:
|
||||
zlog.Warn("Unknown config item:", name)
|
||||
}
|
||||
@@ -330,4 +349,14 @@ func TaskLoadSetting(initLoad bool) {
|
||||
updateConfigStringItem(initLoad, "ssl", "zerossl_access_key", global.GCONFIG_ZEROSSL_ACCESS_KEY, "zerossl访问key", "string", "", configMap)
|
||||
updateConfigStringItem(initLoad, "ssl", "zerossl_eab_kid", global.GCONFIG_ZEROSSL_EAB_KID, "zerossl eab_kid", "string", "", configMap)
|
||||
updateConfigStringItem(initLoad, "ssl", "zerossl_eab_hmac_key", global.GCONFIG_ZEROSSL_EAB_HMAC_KEY, "zerossl eab_hmac_key", "string", "", configMap)
|
||||
|
||||
// 日志文件写入相关配置
|
||||
updateConfigIntItem(initLoad, "logfile", "log_file_write_enable", global.GCONFIG_LOG_FILE_WRITE_ENABLE, "日志文件写入开关(0关闭 1开启)", "options", "0|关闭,1|开启", configMap)
|
||||
updateConfigStringItem(initLoad, "logfile", "log_file_write_path", global.GCONFIG_LOG_FILE_WRITE_PATH, "日志文件输出路径", "string", "", configMap)
|
||||
updateConfigStringItem(initLoad, "logfile", "log_file_write_format", global.GCONFIG_LOG_FILE_WRITE_FORMAT, "日志格式(nginx/apache/custom)", "options", "nginx|Nginx Combined,apache|Apache Combined,custom|自定义格式", configMap)
|
||||
updateConfigStringItem(initLoad, "logfile", "log_file_write_custom_tpl", global.GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL, "自定义日志格式模板", "string", "", configMap)
|
||||
updateConfigIntItem(initLoad, "logfile", "log_file_write_max_size", global.GCONFIG_LOG_FILE_WRITE_MAX_SIZE, "单个日志文件最大大小(MB)", "int", "", configMap)
|
||||
updateConfigIntItem(initLoad, "logfile", "log_file_write_max_backups", global.GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS, "保留的历史文件数量", "int", "", configMap)
|
||||
updateConfigIntItem(initLoad, "logfile", "log_file_write_max_days", global.GCONFIG_LOG_FILE_WRITE_MAX_DAYS, "保留天数", "int", "", configMap)
|
||||
updateConfigIntItem(initLoad, "logfile", "log_file_write_compress", global.GCONFIG_LOG_FILE_WRITE_COMPRESS, "是否压缩历史文件(0关闭 1开启)", "options", "0|关闭,1|开启", configMap)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user