diff --git a/api/entrance.go b/api/entrance.go index 08cb983..fd03032 100644 --- a/api/entrance.go +++ b/api/entrance.go @@ -52,6 +52,7 @@ type APIGroup struct { WafNotifyLogApi WafFirewallIPBlockApi WafPluginApi + WafLogFileWriteApi } var APIGroupAPP = new(APIGroup) diff --git a/api/waf_logfilewrite.go b/api/waf_logfilewrite.go new file mode 100644 index 0000000..0751834 --- /dev/null +++ b/api/waf_logfilewrite.go @@ -0,0 +1,116 @@ +package api + +import ( + "SamWaf/global" + "SamWaf/model/common/response" + "SamWaf/wafnotify/logfilewriter" + "github.com/gin-gonic/gin" + "strconv" +) + +type WafLogFileWriteApi struct { +} + +// GetPreviewApi 获取日志文件预览（最新N行） +func (w *WafLogFileWriteApi) GetPreviewApi(c *gin.Context) { + linesStr := c.DefaultQuery("lines", "100") + lines, err := strconv.Atoi(linesStr) + if err != nil || lines <= 0 { + lines = 100 + } + if lines > 500 { + lines = 500 + } + + if global.GNOTIFY_LOG_FILE_WRITER == nil { + response.FailWithMessage("日志文件写入服务未初始化", c) + return + } + + // 获取底层 notifier + writer := getLogFileWriter() + if writer == nil { + response.FailWithMessage("日志文件写入服务未初始化", c) + return + } + + preview, err := writer.GetLogPreview(lines) + if err != nil { + response.FailWithMessage("获取日志预览失败: "+err.Error(), c) + return + } + + response.OkWithDetailed(preview, "获取成功", c) +} + +// GetCurrentFileInfoApi 获取当前日志文件信息 +func (w *WafLogFileWriteApi) GetCurrentFileInfoApi(c *gin.Context) { + writer := getLogFileWriter() + if writer == nil { + response.FailWithMessage("日志文件写入服务未初始化", c) + return + } + + fileInfo, err := writer.GetCurrentFileInfo() + if err != nil { + response.FailWithMessage("获取文件信息失败: "+err.Error(), c) + return + } + + response.OkWithDetailed(fileInfo, "获取成功", c) +} + +// GetBackupFilesApi 获取备份文件列表 +func (w *WafLogFileWriteApi) GetBackupFilesApi(c *gin.Context) { + writer := getLogFileWriter() + if writer == nil { + response.FailWithMessage("日志文件写入服务未初始化", c) + return + } + + files, err := writer.GetBackupFiles() + if err != nil { + response.FailWithMessage("获取备份文件列表失败: "+err.Error(), c) + return + } + + response.OkWithDetailed(files, "获取成功", c) +} + +// ClearLogFileApi 清空当前日志文件 +func (w *WafLogFileWriteApi) ClearLogFileApi(c *gin.Context) { + writer := getLogFileWriter() + if writer == nil { + response.FailWithMessage("日志文件写入服务未初始化", c) + return + } + + err := writer.ClearLogFile() + if err != nil { + response.FailWithMessage("清空日志文件失败: "+err.Error(), c) + return + } + + response.OkWithMessage("清空成功", c) +} + +// GetTemplateVariablesApi 获取可用的模板变量列表 +func (w *WafLogFileWriteApi) GetTemplateVariablesApi(c *gin.Context) { + variables := logfilewriter.GetTemplateVariables() + response.OkWithDetailed(variables, "获取成功", c) +} + +// getLogFileWriter 获取底层的 LogFileWriter +func getLogFileWriter() *logfilewriter.LogFileWriter { + if global.GNOTIFY_LOG_FILE_WRITER == nil { + return nil + } + notifier := global.GNOTIFY_LOG_FILE_WRITER.GetNotifier() + if notifier == nil { + return nil + } + if writer, ok := notifier.(*logfilewriter.LogFileWriter); ok { + return writer + } + return nil +} diff --git a/cmd/samwaf/main.go b/cmd/samwaf/main.go index 686ef63..cb522cf 100644 --- a/cmd/samwaf/main.go +++ b/cmd/samwaf/main.go @@ -279,6 +279,18 @@ func (m *wafSystenService) run() { waftask.TaskLoadSetting(true) //启动通知相关程序 global.GNOTIFY_KAKFA_SERVICE = wafnotify.InitNotifyKafkaEngine(global.GCONFIG_RECORD_KAFKA_ENABLE, global.GCONFIG_RECORD_KAFKA_URL, global.GCONFIG_RECORD_KAFKA_TOPIC) //kafka + // 日志文件写入 + compressFlag := global.GCONFIG_LOG_FILE_WRITE_COMPRESS == 1 + global.GNOTIFY_LOG_FILE_WRITER = wafnotify.InitLogFileWriterEngine( + global.GCONFIG_LOG_FILE_WRITE_ENABLE, + global.GCONFIG_LOG_FILE_WRITE_PATH, + global.GCONFIG_LOG_FILE_WRITE_FORMAT, + global.GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL, + global.GCONFIG_LOG_FILE_WRITE_MAX_SIZE, + int(global.GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS), + int(global.GCONFIG_LOG_FILE_WRITE_MAX_DAYS), + compressFlag, + ) //启动waf globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE = &wafenginecore.WafEngine{ HostTarget: map[string]*wafenginmodel.HostSafe{}, diff --git a/global/config.go b/global/config.go index b5aaa0f..9e5361b 100644 --- a/global/config.go +++ b/global/config.go @@ -70,4 +70,14 @@ var ( GCONFIG_ZEROSSL_EAB_KID string = "" // zerossl eab_kid GCONFIG_ZEROSSL_EAB_HMAC_KEY string = "" // zerossl eab_hmac_key + // 日志文件写入配置 (额外输出，不影响SQLite存储) + GCONFIG_LOG_FILE_WRITE_ENABLE int64 = 0 // 日志文件写入开关 (0关闭 1开启) + GCONFIG_LOG_FILE_WRITE_PATH string = "logs/access.log" // 日志文件路径 + GCONFIG_LOG_FILE_WRITE_FORMAT string = "nginx" // 日志格式: nginx, apache, custom + GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL string = "" // 自定义格式模板 + GCONFIG_LOG_FILE_WRITE_MAX_SIZE int64 = 100 // 单个日志文件最大大小 (MB) + GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS int64 = 10 // 保留的历史文件数量 + GCONFIG_LOG_FILE_WRITE_MAX_DAYS int64 = 30 // 保留天数 + GCONFIG_LOG_FILE_WRITE_COMPRESS int64 = 0 // 是否压缩历史文件 (0关闭 1开启) + ) diff --git a/global/global.go b/global/global.go index daea491..8a9fd99 100644 --- a/global/global.go +++ b/global/global.go @@ -136,6 +136,7 @@ var ( /*******通知相关*************/ GNOTIFY_KAKFA_SERVICE *wafnotify.WafNotifyService //通知服务 + GNOTIFY_LOG_FILE_WRITER *wafnotify.WafNotifyService //日志文件写入服务 GNOTIFY_SEND_MAX_LIMIT_MINTUTES = time.Duration(5) * time.Minute // 规则相关信息最大发送抑止 默认5分钟 /*******日志记录相关*************/ diff --git a/router/entrance.go b/router/entrance.go index 19767b0..23927c2 100644 --- a/router/entrance.go +++ b/router/entrance.go @@ -50,6 +50,7 @@ type ApiGroup struct { NotifyLogRouter FirewallIPBlockRouter PluginRouter + LogFileWriteRouter } type PublicApiGroup struct { LoginRouter diff --git a/router/waf_logfilewrite.go b/router/waf_logfilewrite.go new file mode 100644 index 0000000..274f861 --- /dev/null +++ b/router/waf_logfilewrite.go @@ -0,0 +1,19 @@ +package router + +import ( + "SamWaf/api" + "github.com/gin-gonic/gin" +) + +type LogFileWriteRouter struct { +} + +func (receiver *LogFileWriteRouter) InitLogFileWriteRouter(group *gin.RouterGroup) { + api := api.APIGroupAPP.WafLogFileWriteApi + router := group.Group("") + router.GET("/api/v1/logfilewrite/preview", api.GetPreviewApi) + router.GET("/api/v1/logfilewrite/currentfile", api.GetCurrentFileInfoApi) + router.GET("/api/v1/logfilewrite/backupfiles", api.GetBackupFilesApi) + router.POST("/api/v1/logfilewrite/clear", api.ClearLogFileApi) + router.GET("/api/v1/logfilewrite/variables", api.GetTemplateVariablesApi) +} diff --git a/wafmangeweb/localserver.go b/wafmangeweb/localserver.go index 4a84887..75725d4 100644 --- a/wafmangeweb/localserver.go +++ b/wafmangeweb/localserver.go @@ -91,6 +91,7 @@ func (web *WafWebManager) initRouter(r *gin.Engine) { router.ApiGroupApp.InitNotifySubscriptionRouter(RouterGroup) router.ApiGroupApp.InitNotifyLogRouter(RouterGroup) router.ApiGroupApp.InitFirewallIPBlockRouter(RouterGroup) + router.ApiGroupApp.InitLogFileWriteRouter(RouterGroup) } if global.GWAF_RELEASE == "true" { diff --git a/wafnotify/logfilewriter/formatter.go b/wafnotify/logfilewriter/formatter.go new file mode 100644 index 0000000..ec6d023 --- /dev/null +++ b/wafnotify/logfilewriter/formatter.go @@ -0,0 +1,103 @@ +package logfilewriter + +import ( + "SamWaf/innerbean" + "strconv" + "strings" +) + +// 预定义格式模板 +const ( + // NginxCombinedFormat nginx combined 格式 + NginxCombinedFormat = `${src_ip} - - [${create_time}] "${method} ${url} HTTP/1.1" ${status_code} ${content_length} "${referer}" "${user_agent}"` + + // ApacheCombinedFormat apache combined 格式 + ApacheCombinedFormat = `${src_ip} - - [${create_time}] "${method} ${url} HTTP/1.1" ${status_code} ${content_length} "${referer}" "${user_agent}"` +) + +// GetFormatTemplate 根据格式名称获取模板 +func GetFormatTemplate(format string) string { + switch strings.ToLower(format) { + case "nginx": + return NginxCombinedFormat + case "apache": + return ApacheCombinedFormat + case "custom": + return "" // 自定义格式由用户提供 + default: + return NginxCombinedFormat + } +} + +// FormatLog 将 WebLog 格式化为指定模板的字符串 +func FormatLog(log *innerbean.WebLog, template string) string { + if template == "" { + template = NginxCombinedFormat + } + + result := template + + // 替换所有模板变量 + result = strings.ReplaceAll(result, "${src_ip}", log.SRC_IP) + result = strings.ReplaceAll(result, "${src_port}", log.SRC_PORT) + result = strings.ReplaceAll(result, "${host}", log.HOST) + result = strings.ReplaceAll(result, "${url}", log.URL) + result = strings.ReplaceAll(result, "${raw_query}", log.RawQuery) + result = strings.ReplaceAll(result, "${method}", log.METHOD) + result = strings.ReplaceAll(result, "${scheme}", log.Scheme) + result = strings.ReplaceAll(result, "${referer}", log.REFERER) + result = strings.ReplaceAll(result, "${user_agent}", log.USER_AGENT) + result = strings.ReplaceAll(result, "${status_code}", strconv.Itoa(log.STATUS_CODE)) + result = strings.ReplaceAll(result, "${content_length}", strconv.FormatInt(log.CONTENT_LENGTH, 10)) + result = strings.ReplaceAll(result, "${create_time}", log.CREATE_TIME) + result = strings.ReplaceAll(result, "${time_spent}", strconv.FormatInt(log.TimeSpent, 10)) + result = strings.ReplaceAll(result, "${country}", log.COUNTRY) + result = strings.ReplaceAll(result, "${province}", log.PROVINCE) + result = strings.ReplaceAll(result, "${city}", log.CITY) + result = strings.ReplaceAll(result, "${action}", log.ACTION) + result = strings.ReplaceAll(result, "${rule}", log.RULE) + result = strings.ReplaceAll(result, "${risk_level}", strconv.Itoa(log.RISK_LEVEL)) + result = strings.ReplaceAll(result, "${cookies}", log.COOKIES) + result = strings.ReplaceAll(result, "${req_uuid}", log.REQ_UUID) + result = strings.ReplaceAll(result, "${is_bot}", strconv.Itoa(log.IsBot)) + result = strings.ReplaceAll(result, "${guest_identification}", log.GUEST_IDENTIFICATION) + + return result +} + +// TemplateVariables 返回所有可用的模板变量信息（供前端展示） +type TemplateVariable struct { + Name string `json:"name"` // 变量名 + Field string `json:"field"` // 对应字段 + Desc string `json:"desc"` // 说明 + Example string `json:"example"` // 示例值 +} + +// GetTemplateVariables 返回所有可用的模板变量 +func GetTemplateVariables() []TemplateVariable { + return []TemplateVariable{ + {Name: "${src_ip}", Field: "SRC_IP", Desc: "客户端IP", Example: "192.168.1.100"}, + {Name: "${src_port}", Field: "SRC_PORT", Desc: "客户端端口", Example: "52341"}, + {Name: "${host}", Field: "HOST", Desc: "请求主机名", Example: "www.example.com"}, + {Name: "${url}", Field: "URL", Desc: "请求URL", Example: "/api/user?id=1"}, + {Name: "${raw_query}", Field: "RawQuery", Desc: "URL查询参数", Example: "id=1&name=test"}, + {Name: "${method}", Field: "METHOD", Desc: "请求方法", Example: "GET"}, + {Name: "${scheme}", Field: "Scheme", Desc: "协议", Example: "https"}, + {Name: "${referer}", Field: "REFERER", Desc: "Referer头", Example: "https://example.com/"}, + {Name: "${user_agent}", Field: "USER_AGENT", Desc: "User-Agent", Example: "Mozilla/5.0..."}, + {Name: "${status_code}", Field: "STATUS_CODE", Desc: "HTTP状态码", Example: "200"}, + {Name: "${content_length}", Field: "CONTENT_LENGTH", Desc: "响应大小", Example: "2048"}, + {Name: "${create_time}", Field: "CREATE_TIME", Desc: "请求时间", Example: "2026-02-09 10:00:00"}, + {Name: "${time_spent}", Field: "TimeSpent", Desc: "耗时(ms)", Example: "125"}, + {Name: "${country}", Field: "COUNTRY", Desc: "国家", Example: "中国"}, + {Name: "${province}", Field: "PROVINCE", Desc: "省份", Example: "广东省"}, + {Name: "${city}", Field: "CITY", Desc: "城市", Example: "深圳市"}, + {Name: "${action}", Field: "ACTION", Desc: "防御动作", Example: "放行"}, + {Name: "${rule}", Field: "RULE", Desc: "触发规则", Example: "SQL注入检测"}, + {Name: "${risk_level}", Field: "RISK_LEVEL", Desc: "风险等级", Example: "0"}, + {Name: "${cookies}", Field: "COOKIES", Desc: "Cookies", Example: "session=abc123"}, + {Name: "${req_uuid}", Field: "REQ_UUID", Desc: "请求ID", Example: "a1b2c3d4-..."}, + {Name: "${is_bot}", Field: "IsBot", Desc: "是否机器人", Example: "0"}, + {Name: "${guest_identification}", Field: "GUEST_IDENTIFICATION", Desc: "访客标识", Example: "Googlebot"}, + } +} diff --git a/wafnotify/logfilewriter/logfilewriter.go b/wafnotify/logfilewriter/logfilewriter.go new file mode 100644 index 0000000..c3331be --- /dev/null +++ b/wafnotify/logfilewriter/logfilewriter.go @@ -0,0 +1,539 @@ +package logfilewriter + +import ( + "SamWaf/common/zlog" + "SamWaf/innerbean" + "bufio" + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// LogFileWriter 实现 WafNotify 接口，将日志写入文件 +type LogFileWriter struct { + mu sync.Mutex + filePath string // 日志文件路径 + format string // 日志格式: nginx, apache, custom + customTpl string // 自定义格式模板 + maxSize int64 // 单文件最大大小 (MB) + maxBackups int // 保留的历史文件数量 + maxDays int // 保留天数 + compress bool // 是否压缩历史文件 + file *os.File + writer *bufio.Writer + currentSize int64 // 当前文件大小 + template string // 解析后的模板 +} + +// NewLogFileWriter 创建日志文件写入器 +func NewLogFileWriter(filePath, format, customTpl string, maxSize int64, maxBackups int, maxDays int, compress bool) (*LogFileWriter, error) { + lw := &LogFileWriter{ + filePath: filePath, + format: format, + customTpl: customTpl, + maxSize: maxSize, + maxBackups: maxBackups, + maxDays: maxDays, + compress: compress, + } + + // 解析模板 + lw.resolveTemplate() + + // 打开文件 + if err := lw.openFile(); err != nil { + return lw, err + } + + return lw, nil +} + +// resolveTemplate 解析格式模板 +func (lw *LogFileWriter) resolveTemplate() { + if lw.format == "custom" && lw.customTpl != "" { + lw.template = lw.customTpl + } else { + lw.template = GetFormatTemplate(lw.format) + } +} + +// openFile 打开或创建日志文件 +func (lw *LogFileWriter) openFile() error { + // 确保目录存在 + dir := filepath.Dir(lw.filePath) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("创建日志目录失败: %v", err) + } + + f, err := os.OpenFile(lw.filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return fmt.Errorf("打开日志文件失败: %v", err) + } + + // 获取当前文件大小 + info, err := f.Stat() + if err != nil { + f.Close() + return fmt.Errorf("获取文件信息失败: %v", err) + } + + lw.file = f + lw.writer = bufio.NewWriterSize(f, 64*1024) // 64KB缓冲区 + lw.currentSize = info.Size() + + return nil +} + +// NotifySingle 实现 WafNotify 接口 - 写入单条日志 +func (lw *LogFileWriter) NotifySingle(log *innerbean.WebLog) error { + lw.mu.Lock() + defer lw.mu.Unlock() + + if lw.file == nil { + if err := lw.openFile(); err != nil { + return err + } + } + + line := FormatLog(log, lw.template) + "\n" + n, err := lw.writer.WriteString(line) + if err != nil { + return fmt.Errorf("写入日志失败: %v", err) + } + lw.currentSize += int64(n) + + // 检查是否需要轮转 + if lw.needsRotation() { + if err := lw.rotate(); err != nil { + zlog.Error("日志文件轮转失败: " + err.Error()) + } + } + + return nil +} + +// NotifyBatch 实现 WafNotify 接口 - 批量写入日志 +func (lw *LogFileWriter) NotifyBatch(logs []*innerbean.WebLog) error { + lw.mu.Lock() + defer lw.mu.Unlock() + + if lw.file == nil { + if err := lw.openFile(); err != nil { + return err + } + } + + for _, log := range logs { + line := FormatLog(log, lw.template) + "\n" + n, err := lw.writer.WriteString(line) + if err != nil { + return fmt.Errorf("写入日志失败: %v", err) + } + lw.currentSize += int64(n) + } + + // 刷新缓冲区 + if err := lw.writer.Flush(); err != nil { + zlog.Error("刷新日志缓冲区失败: " + err.Error()) + } + + // 检查是否需要轮转 + if lw.needsRotation() { + if err := lw.rotate(); err != nil { + zlog.Error("日志文件轮转失败: " + err.Error()) + } + } + + return nil +} + +// needsRotation 检查是否需要轮转 +func (lw *LogFileWriter) needsRotation() bool { + maxBytes := lw.maxSize * 1024 * 1024 + return lw.currentSize >= maxBytes +} + +// rotate 执行日志文件轮转 +func (lw *LogFileWriter) rotate() error { + // 刷新并关闭当前文件 + if lw.writer != nil { + lw.writer.Flush() + } + if lw.file != nil { + lw.file.Close() + lw.file = nil + lw.writer = nil + } + + // 轮转文件: access.log -> access.log.1.log -> access.log.2.log ... + // 先移动已有的备份文件 + for i := lw.maxBackups; i >= 1; i-- { + src := lw.backupName(i) + dst := lw.backupName(i + 1) + if _, err := os.Stat(src); err == nil { + if i == lw.maxBackups { + // 最大编号的文件直接删除 + os.Remove(src) + } else { + os.Rename(src, dst) + } + } + // 同时处理压缩文件 + srcGz := src + ".gz" + dstGz := dst + ".gz" + if _, err := os.Stat(srcGz); err == nil { + if i == lw.maxBackups { + os.Remove(srcGz) + } else { + os.Rename(srcGz, dstGz) + } + } + } + + // 当前文件重命名为 .1.log + backup1 := lw.backupName(1) + if err := os.Rename(lw.filePath, backup1); err != nil { + zlog.Error("重命名日志文件失败: " + err.Error()) + } + + // 压缩刚轮转的文件 + if lw.compress { + go lw.compressFile(backup1) + } + + // 清理过期文件 + go lw.cleanOldFiles() + + // 打开新文件 + return lw.openFile() +} + +// backupName 生成备份文件名 +// 例如: logs/access.log -> logs/access.log.1.log +func (lw *LogFileWriter) backupName(index int) string { + return lw.filePath + "." + strconv.Itoa(index) + ".log" +} + +// compressFile 压缩文件 +func (lw *LogFileWriter) compressFile(srcPath string) { + src, err := os.Open(srcPath) + if err != nil { + zlog.Error("打开待压缩文件失败: " + err.Error()) + return + } + defer src.Close() + + dstPath := srcPath + ".gz" + dst, err := os.Create(dstPath) + if err != nil { + zlog.Error("创建压缩文件失败: " + err.Error()) + return + } + defer dst.Close() + + gz := gzip.NewWriter(dst) + defer gz.Close() + + if _, err := io.Copy(gz, src); err != nil { + zlog.Error("压缩文件失败: " + err.Error()) + os.Remove(dstPath) + return + } + + gz.Close() + dst.Close() + src.Close() + + // 删除原始文件 + os.Remove(srcPath) +} + +// cleanOldFiles 清理超期文件 +func (lw *LogFileWriter) cleanOldFiles() { + dir := filepath.Dir(lw.filePath) + baseName := filepath.Base(lw.filePath) + + entries, err := os.ReadDir(dir) + if err != nil { + zlog.Error("读取日志目录失败: " + err.Error()) + return + } + + now := time.Now() + maxAge := time.Duration(lw.maxDays) * 24 * time.Hour + + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + // 匹配备份文件: baseName.N.log 或 baseName.N.log.gz + if !strings.HasPrefix(name, baseName+".") { + continue + } + // 跳过当前日志文件本身 + if name == baseName { + continue + } + + info, err := entry.Info() + if err != nil { + continue + } + + // 按天数清理 + if lw.maxDays > 0 && now.Sub(info.ModTime()) > maxAge { + fullPath := filepath.Join(dir, name) + os.Remove(fullPath) + zlog.Info("清理过期日志文件: " + fullPath) + } + } +} + +// Close 关闭日志文件 +func (lw *LogFileWriter) Close() { + lw.mu.Lock() + defer lw.mu.Unlock() + + if lw.writer != nil { + lw.writer.Flush() + } + if lw.file != nil { + lw.file.Close() + lw.file = nil + lw.writer = nil + } +} + +// GetLogPreview 获取日志文件最新N行（供前端预览） +func (lw *LogFileWriter) GetLogPreview(lines int) ([]string, error) { + lw.mu.Lock() + // 先刷新缓冲区以确保读到最新内容 + if lw.writer != nil { + lw.writer.Flush() + } + lw.mu.Unlock() + + f, err := os.Open(lw.filePath) + if err != nil { + if os.IsNotExist(err) { + return []string{}, nil + } + return nil, err + } + defer f.Close() + + return readLastLines(f, lines) +} + +// readLastLines 读取文件最后N行 +func readLastLines(f *os.File, n int) ([]string, error) { + info, err := f.Stat() + if err != nil { + return nil, err + } + + fileSize := info.Size() + if fileSize == 0 { + return []string{}, nil + } + + // 从文件末尾向前读取 + bufSize := int64(8192) + if bufSize > fileSize { + bufSize = fileSize + } + + var allLines []string + offset := fileSize + remaining := "" + + for offset > 0 { + readSize := bufSize + if readSize > offset { + readSize = offset + } + offset -= readSize + + buf := make([]byte, readSize) + _, err := f.ReadAt(buf, offset) + if err != nil && err != io.EOF { + return nil, err + } + + chunk := string(buf) + remaining + lines := strings.Split(chunk, "\n") + + // 第一段可能不完整，保留到下次 + remaining = lines[0] + lines = lines[1:] + + // 逆序添加 + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimRight(lines[i], "\r") + if line != "" { + allLines = append([]string{line}, allLines...) + } + } + + if len(allLines) >= n { + break + } + } + + // 处理最后剩余的部分 + if remaining != "" && len(allLines) < n { + remaining = strings.TrimRight(remaining, "\r") + if remaining != "" { + allLines = append([]string{remaining}, allLines...) + } + } + + // 只返回最后N行 + if len(allLines) > n { + allLines = allLines[len(allLines)-n:] + } + + return allLines, nil +} + +// BackupFileInfo 备份文件信息 +type BackupFileInfo struct { + Name string `json:"name"` // 文件名 + Size int64 `json:"size"` // 文件大小 (bytes) + ModTime string `json:"mod_time"` // 修改时间 + FullPath string `json:"full_path"` // 完整路径 +} + +// GetBackupFiles 获取备份文件列表 +func (lw *LogFileWriter) GetBackupFiles() ([]BackupFileInfo, error) { + dir := filepath.Dir(lw.filePath) + baseName := filepath.Base(lw.filePath) + + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return []BackupFileInfo{}, nil + } + return nil, err + } + + var files []BackupFileInfo + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + // 匹配备份文件 + if !strings.HasPrefix(name, baseName+".") { + continue + } + + info, err := entry.Info() + if err != nil { + continue + } + + files = append(files, BackupFileInfo{ + Name: name, + Size: info.Size(), + ModTime: info.ModTime().Format("2006-01-02 15:04:05"), + FullPath: filepath.Join(dir, name), + }) + } + + // 按修改时间倒序排列 + sort.Slice(files, func(i, j int) bool { + return files[i].ModTime > files[j].ModTime + }) + + return files, nil +} + +// GetCurrentFileInfo 获取当前日志文件信息 +func (lw *LogFileWriter) GetCurrentFileInfo() (*BackupFileInfo, error) { + info, err := os.Stat(lw.filePath) + if err != nil { + if os.IsNotExist(err) { + return &BackupFileInfo{ + Name: filepath.Base(lw.filePath), + Size: 0, + ModTime: "", + FullPath: lw.filePath, + }, nil + } + return nil, err + } + + return &BackupFileInfo{ + Name: info.Name(), + Size: info.Size(), + ModTime: info.ModTime().Format("2006-01-02 15:04:05"), + FullPath: lw.filePath, + }, nil +} + +// ClearLogFile 清空当前日志文件 +func (lw *LogFileWriter) ClearLogFile() error { + lw.mu.Lock() + defer lw.mu.Unlock() + + // 关闭当前文件 + if lw.writer != nil { + lw.writer.Flush() + } + if lw.file != nil { + lw.file.Close() + lw.file = nil + lw.writer = nil + } + + // 清空文件 + if err := os.Truncate(lw.filePath, 0); err != nil { + if !os.IsNotExist(err) { + return err + } + } + + // 重新打开 + return lw.openFile() +} + +// UpdateConfig 更新配置（运行时动态修改） +func (lw *LogFileWriter) UpdateConfig(filePath, format, customTpl string, maxSize int64, maxBackups int, maxDays int, compress bool) { + lw.mu.Lock() + defer lw.mu.Unlock() + + pathChanged := lw.filePath != filePath + + lw.format = format + lw.customTpl = customTpl + lw.maxSize = maxSize + lw.maxBackups = maxBackups + lw.maxDays = maxDays + lw.compress = compress + lw.resolveTemplate() + + // 如果路径改变，重新打开文件 + if pathChanged { + if lw.writer != nil { + lw.writer.Flush() + } + if lw.file != nil { + lw.file.Close() + lw.file = nil + lw.writer = nil + } + lw.filePath = filePath + if err := lw.openFile(); err != nil { + zlog.Error("更新配置后打开新日志文件失败: " + err.Error()) + } + } +} diff --git a/wafnotify/process.go b/wafnotify/process.go index 3cb5c8a..7a35d8d 100644 --- a/wafnotify/process.go +++ b/wafnotify/process.go @@ -2,6 +2,7 @@ package wafnotify import ( "SamWaf/wafnotify/kafka" + "SamWaf/wafnotify/logfilewriter" "fmt" "strings" ) @@ -17,3 +18,13 @@ func InitNotifyKafkaEngine(enable int64, url string, topic string) *WafNotifySer // 创建日志服务并注入 notifier return NewWafNotifyService(notifier, enable) } + +// InitLogFileWriterEngine 初始化日志文件写入引擎 +func InitLogFileWriterEngine(enable int64, filePath, format, customTpl string, maxSize int64, maxBackups int, maxDays int, compress bool) *WafNotifyService { + notifier, err := logfilewriter.NewLogFileWriter(filePath, format, customTpl, maxSize, maxBackups, maxDays, compress) + if err != nil { + fmt.Printf("Failed to create log file writer: %v\n", err) + return NewWafNotifyService(notifier, enable) + } + return NewWafNotifyService(notifier, enable) +} diff --git a/wafnotify/wafnotify.go b/wafnotify/wafnotify.go index ad2af8e..1d2ba04 100644 --- a/wafnotify/wafnotify.go +++ b/wafnotify/wafnotify.go @@ -26,6 +26,14 @@ func (ls *WafNotifyService) ChangeEnable(enable int64) { ls.enable = enable } +// GetNotifier 获取底层通知器 +func (ls *WafNotifyService) GetNotifier() wafinterface.WafNotify { + if ls == nil { + return nil + } + return ls.notifier +} + // 处理并发送单条日志 func (ls *WafNotifyService) ProcessSingleLog(log *innerbean.WebLog) error { if ls.enable == 0 { diff --git a/wafqueue/log_queue.go b/wafqueue/log_queue.go index 8fc5884..52cc743 100644 --- a/wafqueue/log_queue.go +++ b/wafqueue/log_queue.go @@ -83,6 +83,8 @@ func ProcessLogDequeEngine() { // 日志流做统计 waftask.CollectStatsFromLogs(webLogArray) global.GNOTIFY_KAKFA_SERVICE.ProcessBatchLogs(webLogArray) + // 文件日志写入 (额外输出) + global.GNOTIFY_LOG_FILE_WRITER.ProcessBatchLogs(webLogArray) } } time.Sleep(100 * time.Millisecond) diff --git a/waftask/task_config.go b/waftask/task_config.go index 4632886..9a60361 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -130,6 +130,19 @@ func setConfigIntValue(name string, value int64, change int) { global.GCONFIG_ENABLE_HTTP3 = value case "record_log_desensitize": global.GCONFIG_RECORD_LOG_DESENSITIZE = value + case "log_file_write_enable": + if global.GCONFIG_LOG_FILE_WRITE_ENABLE != value && global.GNOTIFY_LOG_FILE_WRITER != nil { + global.GNOTIFY_LOG_FILE_WRITER.ChangeEnable(value) + } + global.GCONFIG_LOG_FILE_WRITE_ENABLE = value + case "log_file_write_max_size": + global.GCONFIG_LOG_FILE_WRITE_MAX_SIZE = value + case "log_file_write_max_backups": + global.GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS = value + case "log_file_write_max_days": + global.GCONFIG_LOG_FILE_WRITE_MAX_DAYS = value + case "log_file_write_compress": + global.GCONFIG_LOG_FILE_WRITE_COMPRESS = value default: zlog.Warn("Unknown config item:", name) } @@ -194,6 +207,12 @@ func setConfigStringValue(name string, value string, change int) { case "zerossl_eab_hmac_key": global.GCONFIG_ZEROSSL_EAB_HMAC_KEY = value break + case "log_file_write_path": + global.GCONFIG_LOG_FILE_WRITE_PATH = value + case "log_file_write_format": + global.GCONFIG_LOG_FILE_WRITE_FORMAT = value + case "log_file_write_custom_tpl": + global.GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL = value default: zlog.Warn("Unknown config item:", name) } @@ -330,4 +349,14 @@ func TaskLoadSetting(initLoad bool) { updateConfigStringItem(initLoad, "ssl", "zerossl_access_key", global.GCONFIG_ZEROSSL_ACCESS_KEY, "zerossl访问key", "string", "", configMap) updateConfigStringItem(initLoad, "ssl", "zerossl_eab_kid", global.GCONFIG_ZEROSSL_EAB_KID, "zerossl eab_kid", "string", "", configMap) updateConfigStringItem(initLoad, "ssl", "zerossl_eab_hmac_key", global.GCONFIG_ZEROSSL_EAB_HMAC_KEY, "zerossl eab_hmac_key", "string", "", configMap) + + // 日志文件写入相关配置 + updateConfigIntItem(initLoad, "logfile", "log_file_write_enable", global.GCONFIG_LOG_FILE_WRITE_ENABLE, "日志文件写入开关（0关闭 1开启）", "options", "0|关闭,1|开启", configMap) + updateConfigStringItem(initLoad, "logfile", "log_file_write_path", global.GCONFIG_LOG_FILE_WRITE_PATH, "日志文件输出路径", "string", "", configMap) + updateConfigStringItem(initLoad, "logfile", "log_file_write_format", global.GCONFIG_LOG_FILE_WRITE_FORMAT, "日志格式（nginx/apache/custom）", "options", "nginx|Nginx Combined,apache|Apache Combined,custom|自定义格式", configMap) + updateConfigStringItem(initLoad, "logfile", "log_file_write_custom_tpl", global.GCONFIG_LOG_FILE_WRITE_CUSTOM_TPL, "自定义日志格式模板", "string", "", configMap) + updateConfigIntItem(initLoad, "logfile", "log_file_write_max_size", global.GCONFIG_LOG_FILE_WRITE_MAX_SIZE, "单个日志文件最大大小（MB）", "int", "", configMap) + updateConfigIntItem(initLoad, "logfile", "log_file_write_max_backups", global.GCONFIG_LOG_FILE_WRITE_MAX_BACKUPS, "保留的历史文件数量", "int", "", configMap) + updateConfigIntItem(initLoad, "logfile", "log_file_write_max_days", global.GCONFIG_LOG_FILE_WRITE_MAX_DAYS, "保留天数", "int", "", configMap) + updateConfigIntItem(initLoad, "logfile", "log_file_write_compress", global.GCONFIG_LOG_FILE_WRITE_COMPRESS, "是否压缩历史文件（0关闭 1开启）", "options", "0|关闭,1|开启", configMap) }