From 5fca8434fceaf4819f8d989a607b4fae703e4f30 Mon Sep 17 00:00:00 2001 From: samwaf Date: Mon, 12 May 2025 17:23:40 +0800 Subject: [PATCH] feat:cache file #218 --- api/entrance.go | 3 + api/waf_cacherule_api.go | 119 ++++ codetemplete/code_gen_test.go | 7 +- enums/cache_enum.go | 1 + enums/chan_enum.go | 1 + enums/task_enum.go | 1 + enums/task_time_unit_enum.go | 1 + main.go | 9 +- model/cache_rule.go | 17 + model/hosts.go | 10 + model/request/waf_cacherule_req.go | 36 ++ model/request/waf_host_req.go | 2 + model/request/waf_task_req.go | 19 +- model/task.go | 11 +- model/wafenginmodel/hostsafe.go | 2 +- router/entrance.go | 1 + router/waf_cacherule_router.go | 19 + service/waf_service/waf_cacherule_service.go | 168 ++++++ service/waf_service/waf_host.go | 2 + service/waf_service/waf_task_service.go | 36 +- wafdb/localdb.go | 3 + wafenginecore/wafengine.go | 59 ++ wafenginecore/wafwebcache/webcache.go | 411 ++++++++++++++ wafenginecore/wafwebcache/webcache_test.go | 559 +++++++++++++++++++ wafenginecore/wafworker.go | 4 + wafmangeweb/localserver.go | 1 + waftask/core_scheduler.go | 13 +- waftask/entrance.go | 80 ++- waftask/task_clear_webcache.go | 50 ++ 29 files changed, 1578 insertions(+), 67 deletions(-) create mode 100644 api/waf_cacherule_api.go create mode 100644 model/cache_rule.go create mode 100644 model/request/waf_cacherule_req.go create mode 100644 router/waf_cacherule_router.go create mode 100644 service/waf_service/waf_cacherule_service.go create mode 100644 wafenginecore/wafwebcache/webcache.go create mode 100644 wafenginecore/wafwebcache/webcache_test.go create mode 100644 waftask/task_clear_webcache.go diff --git a/api/entrance.go b/api/entrance.go index a9736ac..395d2f4 100644 --- a/api/entrance.go +++ b/api/entrance.go @@ -38,6 +38,7 @@ type APIGroup struct { WafOtpApi WafAnalysisApi WafPrivateInfoApi + WafCacheRuleApi } var APIGroupAPP = new(APIGroup) @@ -91,4 +92,6 @@ var ( wafAnalysisService = waf_service.WafAnalysisServiceApp wafPrivateInfoService = waf_service.WafPrivateInfoServiceApp + + wafCacheRuleService = waf_service.WafCacheRuleServiceApp ) diff --git a/api/waf_cacherule_api.go b/api/waf_cacherule_api.go new file mode 100644 index 0000000..27ede70 --- /dev/null +++ b/api/waf_cacherule_api.go @@ -0,0 +1,119 @@ +package api + +import ( + "SamWaf/enums" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/common/response" + "SamWaf/model/request" + "SamWaf/model/spec" + "errors" + "github.com/gin-gonic/gin" + "gorm.io/gorm" +) + +type WafCacheRuleApi struct { +} + +func (w *WafCacheRuleApi) AddApi(c *gin.Context) { + var req request.WafCacheRuleAddReq + err := c.ShouldBindJSON(&req) + if err == nil { + + cnt := wafCacheRuleService.CheckIsExistApi(req) + if cnt == 0 { + err = wafCacheRuleService.AddApi(req) + if err == nil { + w.NotifyWaf(req.HostCode) + response.OkWithMessage("添加成功", c) + } else { + response.FailWithMessage("添加失败", c) + } + return + } else { + response.FailWithMessage("当前记录已经存在", c) + return + } + + } else { + response.FailWithMessage("解析失败", c) + } +} + +func (w *WafCacheRuleApi) GetDetailApi(c *gin.Context) { + var req request.WafCacheRuleDetailReq + err := c.ShouldBind(&req) + if err == nil { + bean := wafCacheRuleService.GetDetailApi(req) + response.OkWithDetailed(bean, "获取成功", c) + } else { + response.FailWithMessage("解析失败", c) + } +} + +func (w *WafCacheRuleApi) GetListApi(c *gin.Context) { + var req request.WafCacheRuleSearchReq + err := c.ShouldBindJSON(&req) + if err == nil { + CacheRule, total, _ := wafCacheRuleService.GetListApi(req) + response.OkWithDetailed(response.PageResult{ + List: CacheRule, + Total: total, + PageIndex: req.PageIndex, + PageSize: req.PageSize, + }, "获取成功", c) + } else { + response.FailWithMessage("解析失败", c) + } +} + +func (w *WafCacheRuleApi) DelApi(c *gin.Context) { + var req request.WafCacheRuleDelReq + err := c.ShouldBind(&req) + if err == nil { + bean := wafCacheRuleService.GetDetailByIdApi(req.Id) + if bean.Id == "" { + response.FailWithMessage("未找到信息", c) + return + } + err = wafCacheRuleService.DelApi(req) + if err != nil && errors.Is(err, gorm.ErrRecordNotFound) { + response.FailWithMessage("请检测参数", c) + } else if err != nil { + response.FailWithMessage("发生错误", c) + } else { + w.NotifyWaf(bean.HostCode) + response.OkWithMessage("删除成功", c) + } + } else { + response.FailWithMessage("解析失败", c) + } +} + +func (w *WafCacheRuleApi) ModifyApi(c *gin.Context) { + var req request.WafCacheRuleEditReq + err := c.ShouldBindJSON(&req) + if err == nil { + err = wafCacheRuleService.ModifyApi(req) + if err != nil { + response.FailWithMessage("编辑发生错误"+err.Error(), c) + } else { + w.NotifyWaf(req.HostCode) + response.OkWithMessage("编辑成功", c) + } + } else { + response.FailWithMessage("解析失败", c) + } +} + +// NotifyWaf 通知到waf引擎实时生效 +func (w *WafCacheRuleApi) NotifyWaf(host_code string) { + var list []model.CacheRule + global.GWAF_LOCAL_DB.Where("host_code = ? ", host_code).Find(&list) + var chanInfo = spec.ChanCommonHost{ + HostCode: host_code, + Type: enums.ChanTypeCacheRule, + Content: list, + } + global.GWAF_CHAN_MSG <- chanInfo +} diff --git a/codetemplete/code_gen_test.go b/codetemplete/code_gen_test.go index c7d8e0a..ec5db8d 100644 --- a/codetemplete/code_gen_test.go +++ b/codetemplete/code_gen_test.go @@ -9,7 +9,8 @@ import ( func TestCodeGeneration(t *testing.T) { // 唯一校验定义字段信息的字符串 fieldDefs := []string{ - "PrivateKey:private_key:string", + "HostCode:host_code:string", + "RuleName:rule_name:string", } // 构造 `uniFields` 列表 @@ -27,6 +28,6 @@ func TestCodeGeneration(t *testing.T) { } } - fields := GetStructFields(model.PrivateInfo{}) - CodeGeneration("PrivateInfo", fields, uniFields) + fields := GetStructFields(model.CacheRule{}) + CodeGeneration("CacheRule", fields, uniFields) } diff --git a/enums/cache_enum.go b/enums/cache_enum.go index eb1ba27..e6dcf23 100644 --- a/enums/cache_enum.go +++ b/enums/cache_enum.go @@ -10,4 +10,5 @@ const ( CACHE_CAPTCHA_TRY = "CACHE_CAPTCHA_TRY" //验证码临时 CACHE_CAPTCHA_PASS = "CACHE_CAPTCHA_PASS" //通过验证的码 CACHE_ANNOUNCEMENT = "CACHE_ANNOUNCEMENT" //公告数据 + CACHE_WEBFILE = "CACHE_WEBFILE" ) diff --git a/enums/chan_enum.go b/enums/chan_enum.go index 33a7610..096e008 100644 --- a/enums/chan_enum.go +++ b/enums/chan_enum.go @@ -15,4 +15,5 @@ const ( ChanTypeSSL ChanTypeHttpauth ChanTypeBlockingPage + ChanTypeCacheRule ) diff --git a/enums/task_enum.go b/enums/task_enum.go index 852cc3d..ad90aca 100644 --- a/enums/task_enum.go +++ b/enums/task_enum.go @@ -18,4 +18,5 @@ const ( TASK_HEALTH = "task_health" //健康检测 TASK_CLEAR_CC_WINDOWS = "task_clear_cc_windows" //清除ccWindows记录 TASK_CREATE_DB_INDEX = "task_create_db_index" //创建索引 + TASK_CLEAR_WEBCACHE = "task_clear_webcache" //清除缓存 ) diff --git a/enums/task_time_unit_enum.go b/enums/task_time_unit_enum.go index 0637b1b..436c8a8 100644 --- a/enums/task_time_unit_enum.go +++ b/enums/task_time_unit_enum.go @@ -5,4 +5,5 @@ const ( TASK_MIN = "minute" TASK_HOUR = "hour" TASK_DAY = "day" + TASK_WEEKLY = "weekly" ) diff --git a/main.go b/main.go index 019f7dc..bc66b3c 100644 --- a/main.go +++ b/main.go @@ -306,12 +306,13 @@ func (m *wafSystenService) run() { globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_HEALTH, waftask.TaskHealth) globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_CLEAR_CC_WINDOWS, waftask.TaskCC) globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_CREATE_DB_INDEX, waftask.TaskCreateIndex) + globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_CLEAR_WEBCACHE, waftask.TaskClearWebcache) go waftask.TaskShareDbInfo() globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler = waftask.NewTaskScheduler(globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry) taskDbList := waftask.InitTaskDb() for _, task := range taskDbList { - globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.ScheduleTask(task.TaskUnit, task.TaskValue, task.TaskAt, task.TaskMethod) + globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.ScheduleTask(task.TaskUnit, task.TaskValue, task.TaskAt, task.TaskMethod, task.TaskDaysOfTheWeek) } //结束 需要添加到任务注册器里面的 globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.Start() @@ -499,6 +500,12 @@ func (m *wafSystenService) run() { zlog.Debug("远程配置", zap.Any("配置自定义拦截界面信息", msg.Content.(map[string]model.BlockingPage))) globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].Mux.Unlock() break + case enums.ChanTypeCacheRule: + globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].Mux.Lock() + globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].CacheRule = msg.Content.([]model.CacheRule) + zlog.Debug("远程配置", zap.Any("配置缓存规则", msg.Content.([]model.CacheRule))) + globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].Mux.Unlock() + break } //end switch diff --git a/model/cache_rule.go b/model/cache_rule.go new file mode 100644 index 0000000..7b19812 --- /dev/null +++ b/model/cache_rule.go @@ -0,0 +1,17 @@ +package model + +import "SamWaf/model/baseorm" + +// 缓存规则 +type CacheRule struct { + baseorm.BaseOrm + HostCode string `json:"host_code" gorm:"column:host_code"` // 主机代码 + RuleName string `json:"rule_name" gorm:"column:rule_name"` // 规则名称 + RuleType int `json:"rule_type" gorm:"column:rule_type"` // 规则类型：1-后缀匹配 ，2-指定目录，3-指定文件 + RuleContent string `json:"rule_content" gorm:"column:rule_content"` // 规则内容 + ParamType int `json:"param_type" gorm:"column:param_type"` // 参数处理：1-忽略参数，2-完整参数 + CacheTime int `json:"cache_time" gorm:"column:cache_time"` // 缓存时间(秒)：0-不缓存 + Priority int `json:"priority" gorm:"column:priority"` // 优先级：数字越大优先级越高 + RequestMethod string `json:"request_method" gorm:"column:request_method"` // 请求方式：GET;HEAD;POST等，多个用分号分隔 + Remarks string `json:"remarks" gorm:"column:remarks"` // 备注 +} diff --git a/model/hosts.go b/model/hosts.go index 5cd2dab..a163486 100644 --- a/model/hosts.go +++ b/model/hosts.go @@ -37,6 +37,7 @@ type Hosts struct { InsecureSkipVerify int `json:"insecure_skip_verify"` //是否开启后端https证书有效性验证 默认 0 是校验 1 是不校验 CaptchaJSON string `json:"captcha_json"` //验证码配置 json AntiLeechJSON string `json:"anti_leech_json"` //防盗链配置 json + CacheJSON string `json:"cache_json"` //缓存配置 json } type HostsDefense struct { @@ -77,3 +78,12 @@ type AntiLeechConfig struct { Action string `json:"action"` // 对于非法引用的处理方式: redirect(重定向) 或 block(直接阻止) RedirectURL string `json:"redirect_url"` // 重定向URL，当Action为redirect时使用 } + +// CacheConfig 缓存配置 +type CacheConfig struct { + IsEnableCache int `json:"is_enable_cache"` // 是否开启缓存 1开启 0关闭 + CacheLocation string `json:"cache_location"` // 缓存位置: "memory"内存 或 "file"文件 或 "all"内存和文件 + CacheDir string `json:"cache_dir"` // 缓存目录，当location为file时使用 + MaxFileSizeMB float64 `json:"max_file_size_mb"` // 最大缓存文件大小(MB) 0 是不限制 + MaxMemorySizeMB float64 `json:"max_memory_size_mb"` // 最大内存缓存大小(MB)，当location为memory时使用 0 是不限制 +} diff --git a/model/request/waf_cacherule_req.go b/model/request/waf_cacherule_req.go new file mode 100644 index 0000000..63d1c39 --- /dev/null +++ b/model/request/waf_cacherule_req.go @@ -0,0 +1,36 @@ +package request + +import "SamWaf/model/common/request" + +type WafCacheRuleAddReq struct { + HostCode string `json:"host_code" gorm:"column:host_code" form:"host_code" gorm:"column:host_code"` + RuleName string `json:"rule_name" gorm:"column:rule_name" form:"rule_name" gorm:"column:rule_name"` + RuleType int `json:"rule_type" gorm:"column:rule_type" form:"rule_type" gorm:"column:rule_type"` + RuleContent string `json:"rule_content" gorm:"column:rule_content" form:"rule_content" gorm:"column:rule_content"` + ParamType int `json:"param_type" gorm:"column:param_type" form:"param_type" gorm:"column:param_type"` + CacheTime int `json:"cache_time" gorm:"column:cache_time" form:"cache_time" gorm:"column:cache_time"` + Priority int `json:"priority" gorm:"column:priority" form:"priority" gorm:"column:priority"` + RequestMethod string `json:"request_method" gorm:"column:request_method" form:"request_method" gorm:"column:request_method"` + Remarks string `json:"remarks" gorm:"column:remarks" form:"remarks" gorm:"column:remarks"` +} +type WafCacheRuleEditReq struct { + Id string `json:"id"` + HostCode string `json:"host_code" gorm:"column:host_code" form:"host_code" gorm:"column:host_code"` + RuleName string `json:"rule_name" gorm:"column:rule_name" form:"rule_name" gorm:"column:rule_name"` + RuleType int `json:"rule_type" gorm:"column:rule_type" form:"rule_type" gorm:"column:rule_type"` + RuleContent string `json:"rule_content" gorm:"column:rule_content" form:"rule_content" gorm:"column:rule_content"` + ParamType int `json:"param_type" gorm:"column:param_type" form:"param_type" gorm:"column:param_type"` + CacheTime int `json:"cache_time" gorm:"column:cache_time" form:"cache_time" gorm:"column:cache_time"` + Priority int `json:"priority" gorm:"column:priority" form:"priority" gorm:"column:priority"` + RequestMethod string `json:"request_method" gorm:"column:request_method" form:"request_method" gorm:"column:request_method"` + Remarks string `json:"remarks" gorm:"column:remarks" form:"remarks" gorm:"column:remarks"` +} +type WafCacheRuleDetailReq struct { + Id string `json:"id" form:"id"` +} +type WafCacheRuleDelReq struct { + Id string `json:"id" form:"id"` +} +type WafCacheRuleSearchReq struct { + request.PageInfo +} diff --git a/model/request/waf_host_req.go b/model/request/waf_host_req.go index 484d8fa..6c56e57 100644 --- a/model/request/waf_host_req.go +++ b/model/request/waf_host_req.go @@ -32,6 +32,7 @@ type WafHostAddReq struct { InsecureSkipVerify int `json:"insecure_skip_verify"` //是否开启后端https证书有效性验证 默认 0 是校验 1 是不校验 CaptchaJSON string `json:"captcha_json"` //验证码配置 json AntiLeechJSON string `json:"anti_leech_json"` //防盗链配置 json + CacheJSON string `json:"cache_json"` //缓存配置 json } type WafHostDelReq struct { @@ -72,6 +73,7 @@ type WafHostEditReq struct { InsecureSkipVerify int `json:"insecure_skip_verify"` //是否开启后端https证书有效性验证 默认 0 是校验 1 是不校验 CaptchaJSON string `json:"captcha_json"` //验证码配置 json AntiLeechJSON string `json:"anti_leech_json"` //防盗链配置 json + CacheJSON string `json:"cache_json"` //缓存配置 json } type WafHostGuardStatusReq struct { diff --git a/model/request/waf_task_req.go b/model/request/waf_task_req.go index dc5b32a..843bd03 100644 --- a/model/request/waf_task_req.go +++ b/model/request/waf_task_req.go @@ -3,12 +3,12 @@ package request import "SamWaf/model/common/request" type WafTaskAddReq struct { - TaskName string `json:"task_name" form:"task_name"` - TaskUnit string `json:"task_unit" form:"task_unit"` - - TaskValue int `json:"task_value" form:"task_value"` - TaskAt string `json:"task_at" form:"task_at"` - TaskMethod string `json:"task_method" form:"task_method"` + TaskName string `json:"task_name" form:"task_name"` + TaskUnit string `json:"task_unit" form:"task_unit"` + TaskValue int `json:"task_value" form:"task_value"` + TaskAt string `json:"task_at" form:"task_at"` + TaskMethod string `json:"task_method" form:"task_method"` + TaskDaysOfTheWeek string `json:"task_days_of_the_week"` // 周几 在周级别的情况下才起作用 } type WafTaskEditReq struct { Id string `json:"id"` @@ -16,9 +16,10 @@ type WafTaskEditReq struct { TaskName string `json:"task_name" form:"task_name"` TaskUnit string `json:"task_unit" form:"task_unit"` - TaskValue int `json:"task_value" form:"task_value"` - TaskAt string `json:"task_at" form:"task_at"` - TaskMethod string `json:"task_method" form:"task_method"` + TaskValue int `json:"task_value" form:"task_value"` + TaskAt string `json:"task_at" form:"task_at"` + TaskMethod string `json:"task_method" form:"task_method"` + TaskDaysOfTheWeek string `json:"task_days_of_the_week"` // 周几 在周级别的情况下才起作用 } type WafTaskDetailReq struct { Id string `json:"id" form:"id"` diff --git a/model/task.go b/model/task.go index 62bf09b..b1374ec 100644 --- a/model/task.go +++ b/model/task.go @@ -4,9 +4,10 @@ import "SamWaf/model/baseorm" type Task struct { baseorm.BaseOrm - TaskName string `json:"task_name"` // 任务名称 - TaskUnit string `json:"task_unit"` // 单位 - TaskValue int `json:"task_value"` // 值 - TaskAt string `json:"task_at"` // 特定时间 - TaskMethod string `json:"task_method"` //关联方法 + TaskName string `json:"task_name"` // 任务名称 + TaskUnit string `json:"task_unit"` // 单位 + TaskValue int `json:"task_value"` // 值 + TaskAt string `json:"task_at"` // 特定时间 + TaskMethod string `json:"task_method"` //关联方法 + TaskDaysOfTheWeek string `json:"task_days_of_the_week"` // 周几 在周级别的情况下才起作用 } diff --git a/model/wafenginmodel/hostsafe.go b/model/wafenginmodel/hostsafe.go index 19e35d9..6a4e850 100644 --- a/model/wafenginmodel/hostsafe.go +++ b/model/wafenginmodel/hostsafe.go @@ -29,7 +29,7 @@ type HostSafe struct { AntiCCBean model.AntiCC //抵御CC HttpAuthBases []model.HttpAuthBase //HTTP AUTH校验 BlockingPage map[string]model.BlockingPage //自定义拦截界面 - + CacheRule []model.CacheRule //CacheRule } // 负载处理运行对象 diff --git a/router/entrance.go b/router/entrance.go index fdbb5f1..806ed56 100644 --- a/router/entrance.go +++ b/router/entrance.go @@ -36,6 +36,7 @@ type ApiGroup struct { WafOtpRouter AnalysisRouter WafPrivateInfoRouter + WafCacheRuleRouter } type PublicApiGroup struct { LoginRouter diff --git a/router/waf_cacherule_router.go b/router/waf_cacherule_router.go new file mode 100644 index 0000000..d683b83 --- /dev/null +++ b/router/waf_cacherule_router.go @@ -0,0 +1,19 @@ +package router + +import ( + "SamWaf/api" + "github.com/gin-gonic/gin" +) + +type WafCacheRuleRouter struct { +} + +func (receiver *WafCacheRuleRouter) InitWafCacheRuleRouter(group *gin.RouterGroup) { + api := api.APIGroupAPP.WafCacheRuleApi + router := group.Group("") + router.POST("/samwaf/wafhost/cacherule/add", api.AddApi) + router.POST("/samwaf/wafhost/cacherule/list", api.GetListApi) + router.GET("/samwaf/wafhost/cacherule/detail", api.GetDetailApi) + router.POST("/samwaf/wafhost/cacherule/edit", api.ModifyApi) + router.GET("/samwaf/wafhost/cacherule/del", api.DelApi) +} diff --git a/service/waf_service/waf_cacherule_service.go b/service/waf_service/waf_cacherule_service.go new file mode 100644 index 0000000..7db3e47 --- /dev/null +++ b/service/waf_service/waf_cacherule_service.go @@ -0,0 +1,168 @@ +package waf_service + +import ( + "SamWaf/common/uuid" + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/model/request" + "errors" + "time" +) + +type WafCacheRuleService struct{} + +var WafCacheRuleServiceApp = new(WafCacheRuleService) + +func (receiver *WafCacheRuleService) AddApi(req request.WafCacheRuleAddReq) error { + var bean = &model.CacheRule{ + BaseOrm: baseorm.BaseOrm{ + Id: uuid.GenUUID(), + USER_CODE: global.GWAF_USER_CODE, + Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: customtype.JsonTime(time.Now()), + UPDATE_TIME: customtype.JsonTime(time.Now()), + }, + + HostCode: req.HostCode, + RuleName: req.RuleName, + RuleType: req.RuleType, + RuleContent: req.RuleContent, + ParamType: req.ParamType, + CacheTime: req.CacheTime, + Priority: req.Priority, + RequestMethod: req.RequestMethod, + Remarks: req.Remarks, + } + global.GWAF_LOCAL_DB.Create(bean) + return nil +} + +func (receiver *WafCacheRuleService) CheckIsExistApi(req request.WafCacheRuleAddReq) int { + var total int64 = 0 + /*where条件*/ + var whereField = "" + var whereValues []interface{} + //where字段 + whereField = "" + + if len(req.HostCode) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " host_code=? " + } + + if len(req.RuleName) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " rule_name=? " + } + + //where字段赋值 + + if len(req.HostCode) > 0 { + if len(whereField) > 0 { + whereValues = append(whereValues, req.HostCode) + } + } + + if len(req.RuleName) > 0 { + if len(whereField) > 0 { + whereValues = append(whereValues, req.RuleName) + } + } + + global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Where(whereField, whereValues...).Count(&total) + return int(total) +} + +func (receiver *WafCacheRuleService) ModifyApi(req request.WafCacheRuleEditReq) error { + // 根据唯一字段生成查询条件（只有在UniFields不为空时才进行存在性检查） + + var total int64 = 0 + /*where条件*/ + var whereField = "" + var whereValues []interface{} + //where字段 + whereField = "" + + if len(req.HostCode) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " host_code=? " + } + + if len(req.RuleName) > 0 { + if len(whereField) > 0 { + whereField = whereField + " and " + } + whereField = whereField + " rule_name=? " + } + + //where字段赋值 + + if len(req.HostCode) > 0 { + whereValues = append(whereValues, req.HostCode) + } + + if len(req.RuleName) > 0 { + whereValues = append(whereValues, req.RuleName) + } + + global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Where(whereField, whereValues...).Count(&total) + // 查询是否已存在记录 + var bean model.CacheRule + global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Where(whereField, whereValues...).Limit(1).Find(&bean) + + if int(total) > 0 && bean.Id != "" && bean.Id != req.Id { + return errors.New("当前记录已经存在") + } + + beanMap := map[string]interface{}{ + + "HostCode": req.HostCode, + "RuleName": req.RuleName, + "RuleType": req.RuleType, + "RuleContent": req.RuleContent, + "ParamType": req.ParamType, + "CacheTime": req.CacheTime, + "Priority": req.Priority, + "RequestMethod": req.RequestMethod, + "Remarks": req.Remarks, + "UPDATE_TIME": customtype.JsonTime(time.Now()), + } + err := global.GWAF_LOCAL_DB.Model(model.CacheRule{}).Where("id = ?", req.Id).Updates(beanMap).Error + + return err +} +func (receiver *WafCacheRuleService) GetDetailApi(req request.WafCacheRuleDetailReq) model.CacheRule { + var bean model.CacheRule + global.GWAF_LOCAL_DB.Where("id=?", req.Id).Find(&bean) + return bean +} +func (receiver *WafCacheRuleService) GetDetailByIdApi(id string) model.CacheRule { + var bean model.CacheRule + global.GWAF_LOCAL_DB.Where("id=?", id).Find(&bean) + return bean +} +func (receiver *WafCacheRuleService) GetListApi(req request.WafCacheRuleSearchReq) ([]model.CacheRule, int64, error) { + var list []model.CacheRule + var total int64 = 0 + global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Limit(req.PageSize).Offset(req.PageSize * (req.PageIndex - 1)).Find(&list) + global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Count(&total) + + return list, total, nil +} +func (receiver *WafCacheRuleService) DelApi(req request.WafCacheRuleDelReq) error { + var bean model.CacheRule + err := global.GWAF_LOCAL_DB.Where("id = ?", req.Id).First(&bean).Error + if err != nil { + return err + } + err = global.GWAF_LOCAL_DB.Where("id = ?", req.Id).Delete(model.CacheRule{}).Error + return err +} diff --git a/service/waf_service/waf_host.go b/service/waf_service/waf_host.go index c86ae17..877b77b 100644 --- a/service/waf_service/waf_host.go +++ b/service/waf_service/waf_host.go @@ -63,6 +63,7 @@ func (receiver *WafHostService) AddApi(wafHostAddReq request.WafHostAddReq) (str InsecureSkipVerify: wafHostAddReq.InsecureSkipVerify, CaptchaJSON: wafHostAddReq.CaptchaJSON, AntiLeechJSON: wafHostAddReq.AntiLeechJSON, + CacheJSON: wafHostAddReq.CacheJSON, } global.GWAF_LOCAL_DB.Create(wafHost) return wafHost.Code, nil @@ -116,6 +117,7 @@ func (receiver *WafHostService) ModifyApi(wafHostEditReq request.WafHostEditReq) "InsecureSkipVerify": wafHostEditReq.InsecureSkipVerify, "CaptchaJSON": wafHostEditReq.CaptchaJSON, "AntiLeechJSON": wafHostEditReq.AntiLeechJSON, + "CacheJSON": wafHostEditReq.CacheJSON, } err := global.GWAF_LOCAL_DB.Debug().Model(model.Hosts{}).Where("CODE=?", wafHostEditReq.CODE).Updates(hostMap).Error diff --git a/service/waf_service/waf_task_service.go b/service/waf_service/waf_task_service.go index 4e4a92a..d133506 100644 --- a/service/waf_service/waf_task_service.go +++ b/service/waf_service/waf_task_service.go @@ -25,11 +25,12 @@ func (receiver *WafTaskService) AddApi(req request.WafTaskAddReq) error { UPDATE_TIME: customtype.JsonTime(time.Now()), }, - TaskName: req.TaskName, - TaskUnit: req.TaskUnit, - TaskValue: req.TaskValue, - TaskAt: req.TaskAt, - TaskMethod: req.TaskMethod, + TaskName: req.TaskName, + TaskUnit: req.TaskUnit, + TaskValue: req.TaskValue, + TaskAt: req.TaskAt, + TaskMethod: req.TaskMethod, + TaskDaysOfTheWeek: req.TaskDaysOfTheWeek, } global.GWAF_LOCAL_DB.Create(bean) return nil @@ -45,11 +46,12 @@ func (receiver *WafTaskService) Add(req model.Task) error { UPDATE_TIME: customtype.JsonTime(time.Now()), }, - TaskName: req.TaskName, - TaskUnit: req.TaskUnit, - TaskValue: req.TaskValue, - TaskAt: req.TaskAt, - TaskMethod: req.TaskMethod, + TaskName: req.TaskName, + TaskUnit: req.TaskUnit, + TaskValue: req.TaskValue, + TaskAt: req.TaskAt, + TaskMethod: req.TaskMethod, + TaskDaysOfTheWeek: req.TaskDaysOfTheWeek, } global.GWAF_LOCAL_DB.Create(bean) return nil @@ -141,13 +143,13 @@ func (receiver *WafTaskService) ModifyApi(req request.WafTaskEditReq) error { beanMap := map[string]interface{}{ - "TaskName": req.TaskName, - "TaskUnit": req.TaskUnit, - "TaskAt": req.TaskAt, - "TaskMethod": req.TaskMethod, - "TaskValue": req.TaskValue, - - "UPDATE_TIME": customtype.JsonTime(time.Now()), + "TaskName": req.TaskName, + "TaskUnit": req.TaskUnit, + "TaskAt": req.TaskAt, + "TaskMethod": req.TaskMethod, + "TaskValue": req.TaskValue, + "TaskDaysOfTheWeek": req.TaskDaysOfTheWeek, + "UPDATE_TIME": customtype.JsonTime(time.Now()), } err := global.GWAF_LOCAL_DB.Model(model.Task{}).Where("id = ?", req.Id).Updates(beanMap).Error diff --git a/wafdb/localdb.go b/wafdb/localdb.go index 378ac16..8a5cd85 100644 --- a/wafdb/localdb.go +++ b/wafdb/localdb.go @@ -143,6 +143,9 @@ func InitCoreDb(currentDir string) { //密钥信息 db.AutoMigrate(&model.PrivateInfo{}) + //缓存规则 + db.AutoMigrate(&model.CacheRule{}) + global.GWAF_LOCAL_DB.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query) global.GWAF_LOCAL_DB.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) diff --git a/wafenginecore/wafengine.go b/wafenginecore/wafengine.go index c96e976..dd51f52 100644 --- a/wafenginecore/wafengine.go +++ b/wafenginecore/wafengine.go @@ -15,6 +15,7 @@ import ( "SamWaf/utils" "SamWaf/wafenginecore/loadbalance" "SamWaf/wafenginecore/wafhttpcore" + "SamWaf/wafenginecore/wafwebcache" "SamWaf/wafproxy" "bytes" "context" @@ -25,6 +26,7 @@ import ( goahocorasick "github.com/samwafgo/ahocorasick" "go.uber.org/zap" "io" + "io/ioutil" "net" "net/http" _ "net/http/pprof" @@ -404,6 +406,46 @@ func (waf *WafEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } + + //检测cache + cacheConfig := model.CacheConfig{ + IsEnableCache: 0, + CacheLocation: "", + CacheDir: "", + MaxFileSizeMB: 0, + MaxMemorySizeMB: 0, + } + err := json.Unmarshal([]byte(hostTarget.Host.CacheJSON), &cacheConfig) + if err != nil { + zlog.Debug("解析cache json失败") + } + if cacheConfig.IsEnableCache == 1 { + cacheResp := wafwebcache.LoadWebDataFormCache(w, r, hostTarget, cacheConfig) + if cacheResp != nil { + // 将缓存的响应写回给客户端 + for k, v := range cacheResp.Header { + for _, val := range v { + w.Header().Add(k, val) + } + } + + w.Header().Add("X-Cache", "HIT") + w.WriteHeader(cacheResp.StatusCode) + // 读取并写入响应体 + if cacheResp.Body != nil { + body, _ := ioutil.ReadAll(cacheResp.Body) + cacheResp.Body.Close() + w.Write(body) + + // 重新创建一个body供后续使用 + cacheResp.Body = ioutil.NopCloser(bytes.NewBuffer(body)) + } + + // 设置响应并返回 + r.Response = cacheResp + return + } + } //如果正常的流量不记录请求原始包 if global.GCONFIG_RECORD_ALL_SRC_BYTE_INFO == 0 { weblogbean.SrcByteBody = nil @@ -767,6 +809,23 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error { zlog.Warn(fmt.Sprintf("识别响应内容编码失败，响应日志，敏感词替换 不可用 %v，请求URL: %s", responseEncodingError, r.URL.String())) } } + + //检测cache + cacheConfig := model.CacheConfig{ + IsEnableCache: 0, + CacheLocation: "", + CacheDir: "", + MaxFileSizeMB: 0, + MaxMemorySizeMB: 0, + } + err := json.Unmarshal([]byte(waf.HostTarget[host].Host.CacheJSON), &cacheConfig) + if err != nil { + zlog.Debug("解析cache json失败") + } + if cacheConfig.IsEnableCache == 1 { + wafwebcache.StoreWebDataCache(resp, waf.HostTarget[host], cacheConfig) + } + zlog.Debug("TESTChanllage", weblogfrist.HOST, weblogfrist.URL) if !isStaticAssist { datetimeNow := time.Now() diff --git a/wafenginecore/wafwebcache/webcache.go b/wafenginecore/wafwebcache/webcache.go new file mode 100644 index 0000000..fc5a961 --- /dev/null +++ b/wafenginecore/wafwebcache/webcache.go @@ -0,0 +1,411 @@ +package wafwebcache + +import ( + "SamWaf/common/zlog" + "SamWaf/enums" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/wafenginmodel" + "SamWaf/utils" + "bufio" + "bytes" + "fmt" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +// LoadWebDataFormCache 从缓存里面拿数据 +func LoadWebDataFormCache(w http.ResponseWriter, r *http.Request, hostSafe *wafenginmodel.HostSafe, cacheConfig model.CacheConfig) *http.Response { + key := utils.Md5String(r.RequestURI) + zlog.Debug(fmt.Sprintf("尝试从缓存加载数据 URL: %s, 缓存键: %s, 主机代码: %s, 缓存位置: %s", + r.RequestURI, key, hostSafe.Host.Code, cacheConfig.CacheLocation)) + + //规则匹配查询 + _, err2 := checkCacheRule(hostSafe, cacheConfig, r) + if err2 != nil { + return nil + } + + var data []byte = nil + if cacheConfig.CacheLocation == "memory" || cacheConfig.CacheLocation == "all" { + zlog.Debug(fmt.Sprintf("尝试从内存缓存加载 主机代码: %s, 缓存键: %s", hostSafe.Host.Code, key)) + memoryData := loadFormMemory(hostSafe.Host.Code, key) + if memoryData == nil { + zlog.Debug(fmt.Sprintf("内存缓存未命中 主机代码: %s, 缓存键: %s", hostSafe.Host.Code, key)) + } else { + zlog.Debug(fmt.Sprintf("内存缓存命中 主机代码: %s, 缓存键: %s, 数据大小: %d", + hostSafe.Host.Code, key, len(memoryData))) + data = memoryData + } + } + + if data == nil && (cacheConfig.CacheLocation == "file" || cacheConfig.CacheLocation == "all") { + zlog.Debug(fmt.Sprintf("尝试从文件缓存加载 主机代码: %s, 缓存键: %s, 缓存目录: %s", + hostSafe.Host.Code, key, cacheConfig.CacheDir)) + fileData := loadFormFile(hostSafe.Host.Code, key, cacheConfig.CacheDir) + if fileData == nil { + zlog.Debug(fmt.Sprintf("文件缓存未命中 主机代码: %s, 缓存键: %s", hostSafe.Host.Code, key)) + } else { + zlog.Debug(fmt.Sprintf("文件缓存命中 主机代码: %s, 缓存键: %s, 数据大小: %d", + hostSafe.Host.Code, key, len(fileData))) + data = fileData + } + } + + if data == nil { + zlog.Debug(fmt.Sprintf("缓存完全未命中 URL: %s, 缓存键: %s", r.RequestURI, key)) + return nil + } + + buf := bufio.NewReader(bytes.NewReader(data)) + resp, err := http.ReadResponse(buf, nil) + if err != nil { + zlog.Error(fmt.Sprintf("解析缓存响应失败 URL: %s, 缓存键: %s, 错误: %v", r.RequestURI, key, err)) + return nil + } + + zlog.Debug(fmt.Sprintf("成功从缓存加载响应 URL: %s, 缓存键: %s, 状态码: %d, 响应头数量: %d", + r.RequestURI, key, resp.StatusCode, len(resp.Header))) + return resp +} + +// StoreWebDataCache 缓存web数据到cache里面 +func StoreWebDataCache(resp *http.Response, hostSafe *wafenginmodel.HostSafe, cacheConfig model.CacheConfig) { + key := utils.Md5String(resp.Request.RequestURI) + zlog.Debug(fmt.Sprintf("尝试存储响应到缓存 URL: %s, 缓存键: %s, 主机代码: %s, 缓存位置: %s", + resp.Request.RequestURI, key, hostSafe.Host.Code, cacheConfig.CacheLocation)) + + //规则匹配查询 获取缓存时间 + timeout, err2 := checkCacheRule(hostSafe, cacheConfig, resp.Request) + if err2 != nil { + return + } + + data, err := httputil.DumpResponse(resp, true) + if err != nil { + zlog.Error(fmt.Sprintf("序列化响应失败 URL: %s, 错误: %v", resp.Request.RequestURI, err)) + return + } + + zlog.Debug(fmt.Sprintf("响应序列化成功 URL: %s, 数据大小: %d", resp.Request.RequestURI, len(data))) + + if cacheConfig.CacheLocation == "memory" || cacheConfig.CacheLocation == "all" { + zlog.Debug(fmt.Sprintf("存储到内存缓存 主机代码: %s, 缓存键: %s, 超时时间(秒): %d", + hostSafe.Host.Code, key, timeout)) + storeMemory(hostSafe.Host.Code, key, data, timeout, cacheConfig.MaxMemorySizeMB) + } + + if cacheConfig.CacheLocation == "file" || cacheConfig.CacheLocation == "all" { + zlog.Debug(fmt.Sprintf("存储到文件缓存 主机代码: %s, 缓存键: %s, 缓存目录: %s, 超时时间(秒): %d", + hostSafe.Host.Code, key, cacheConfig.CacheDir, timeout)) + storeFile(hostSafe.Host.Code, key, data, cacheConfig.CacheDir, timeout, cacheConfig.MaxFileSizeMB) + } + + zlog.Debug(fmt.Sprintf("响应缓存完成 URL: %s, 缓存键: %s", resp.Request.RequestURI, key)) +} + +func loadFormMemory(hostCode string, key string) []byte { + cacheKey := enums.CACHE_WEBFILE + hostCode + key + zlog.Debug(fmt.Sprintf("检查内存缓存键 缓存键: %s", cacheKey)) + + isKeyExist := global.GCACHE_WAFCACHE.IsKeyExist(cacheKey) + if isKeyExist { + zlog.Debug(fmt.Sprintf("内存缓存键存在 缓存键: %s", cacheKey)) + data, err := global.GCACHE_WAFCACHE.GetBytes(cacheKey) + if err == nil { + zlog.Debug(fmt.Sprintf("成功获取内存缓存数据 缓存键: %s, 数据大小: %d", cacheKey, len(data))) + return data + } + zlog.Error(fmt.Sprintf("获取内存缓存数据失败 缓存键: %s, 错误: %v", cacheKey, err)) + } else { + zlog.Debug(fmt.Sprintf("内存缓存键不存在 缓存键: %s", cacheKey)) + } + return nil +} + +func loadFormFile(hostCode string, key string, path string) []byte { + safePath := filepath.Join(path, hostCode) + + // 获取目录中所有可能的缓存文件 + pattern := filepath.Join(safePath, key+".*.cache") + matches, err := filepath.Glob(pattern) + if err != nil || len(matches) == 0 { + zlog.Debug(fmt.Sprintf("文件缓存不存在 匹配模式: %s, 错误: %v", pattern, err)) + return nil + } + + // 当前时间戳 + now := time.Now().Unix() + + // 查找有效的缓存文件（未过期的） + var validFile string + var latestExpire int64 + + for _, match := range matches { + // 从文件名中提取过期时间戳 + base := filepath.Base(match) + parts := strings.Split(base, ".") + if len(parts) < 3 { + continue + } + + expireStr := parts[len(parts)-2] + expire, err := strconv.ParseInt(expireStr, 10, 64) + if err != nil { + continue + } + + // 检查是否过期 + if expire > now { + // 如果有多个有效文件，选择过期时间最晚的 + if expire > latestExpire { + validFile = match + latestExpire = expire + } + } else { + // 文件已过期，可以选择删除 + zlog.Debug(fmt.Sprintf("缓存文件已过期，准备删除: %s", match)) + os.Remove(match) + } + } + + if validFile == "" { + zlog.Debug(fmt.Sprintf("没有找到有效的缓存文件 键: %s", key)) + return nil + } + + zlog.Debug(fmt.Sprintf("找到有效的缓存文件: %s, 过期时间戳: %d", validFile, latestExpire)) + + data, err := ioutil.ReadFile(validFile) + if err != nil { + zlog.Error(fmt.Sprintf("读取文件缓存失败 文件路径: %s, 错误: %v", validFile, err)) + return nil + } + + zlog.Debug(fmt.Sprintf("成功读取文件缓存 文件路径: %s, 数据大小: %d", validFile, len(data))) + return data +} + +func storeMemory(hostCode string, key string, data []byte, timeout int, maxSize float64) { + cacheKey := enums.CACHE_WEBFILE + hostCode + key + zlog.Debug(fmt.Sprintf("存储到内存缓存 缓存键: %s, 数据大小: %d", cacheKey, len(data))) + + // 检查内存大小限制 + dataSizeMB := float64(len(data)) / (1024 * 1024) + if maxSize > 0 && dataSizeMB > maxSize { + zlog.Warn(fmt.Sprintf("内存缓存大小超过限制 缓存键: %s, 数据大小: %.2fMB, 最大限制: %.2fMB", + cacheKey, dataSizeMB, maxSize)) + return + } + + if timeout > 0 { + zlog.Debug(fmt.Sprintf("设置内存缓存带超时 缓存键: %s, 超时时间(秒): %d", cacheKey, timeout)) + global.GCACHE_WAFCACHE.SetWithTTlRenewTime(cacheKey, data, time.Duration(timeout)*time.Second) + } else { + zlog.Debug(fmt.Sprintf("设置内存缓存长期有效 缓存键: %s", cacheKey)) + global.GCACHE_WAFCACHE.SetWithTTlRenewTime(cacheKey, data, time.Duration(87600)*time.Hour) + } +} + +func storeFile(hostCode string, key string, data []byte, path string, timeout int, maxSize float64) { + safePath := filepath.Join(path, hostCode) + zlog.Debug(fmt.Sprintf("准备存储文件缓存 目录路径: %s, 缓存键: %s", safePath, key)) + + // 确保缓存目录存在 + if _, err := os.Stat(safePath); os.IsNotExist(err) { + zlog.Debug(fmt.Sprintf("缓存目录不存在，创建目录 目录路径: %s", safePath)) + err := os.MkdirAll(safePath, 0755) + if err != nil { + zlog.Error(fmt.Sprintf("创建缓存目录失败 目录路径: %s, 错误: %v", safePath, err)) + return + } + } + + // 计算过期时间戳 + var expireTimestamp int64 + if timeout > 0 { + expireTimestamp = time.Now().Unix() + int64(timeout) + } else { + // 如果没有设置超时，使用一个很大的值（10年） + expireTimestamp = time.Now().Unix() + 315360000 + } + + // 文件名中包含过期时间戳 + filename := filepath.Join(safePath, fmt.Sprintf("%s.%d.cache", key, expireTimestamp)) + zlog.Debug(fmt.Sprintf("写入文件缓存 文件路径: %s, 数据大小: %d, 过期时间戳: %d", filename, len(data), expireTimestamp)) + + // 检查文件大小限制 + dataSizeMB := float64(len(data)) / (1024 * 1024) + if maxSize > 0 && dataSizeMB > maxSize { + zlog.Warn(fmt.Sprintf("文件缓存大小超过限制 文件路径: %s, 数据大小: %.2fMB, 最大限制: %.2fMB", + filename, dataSizeMB, maxSize)) + return + } + + err := ioutil.WriteFile(filename, data, 0644) + if err != nil { + zlog.Error(fmt.Sprintf("写入文件缓存失败 文件路径: %s, 错误: %v", filename, err)) + return + } + + zlog.Debug(fmt.Sprintf("文件缓存写入成功 文件路径: %s", filename)) +} + +func checkCacheRule(hostSafe *wafenginmodel.HostSafe, cacheConfig model.CacheConfig, r *http.Request) (int, error) { + //规则匹配查询 + if hostSafe.CacheRule == nil || len(hostSafe.CacheRule) == 0 { + return 0, fmt.Errorf("没有配置缓存规则") + } + + requestURI := r.URL.RequestURI() + requestMethod := r.Method + + // 按优先级排序规则（优先级数字越大越优先） + sort.Slice(hostSafe.CacheRule, func(i, j int) bool { + return hostSafe.CacheRule[i].Priority > hostSafe.CacheRule[j].Priority + }) + + pureRequestUrl := r.URL.Path + // 循环检查每个规则 + for _, rule := range hostSafe.CacheRule { + finalUrl := requestURI + if rule.ParamType == 1 { + finalUrl = pureRequestUrl + } + + // 检查请求方法是否匹配 + if !checkRequestMethod(requestMethod, rule.RequestMethod) { + continue + } + + // 根据规则类型进行匹配 + matched := false + switch rule.RuleType { + case 1: // 后缀匹配 + matched = checkSuffixMatch(finalUrl, rule.RuleContent) + case 2: // 指定目录 + matched = checkDirectoryMatch(finalUrl, rule.RuleContent) + case 3: // 指定文件 + matched = checkExactFileMatch(finalUrl, rule.RuleContent) + } + + if matched { + zlog.Debug(fmt.Sprintf("缓存规则匹配成功: %s, 缓存时间: %d秒", rule.RuleName, rule.CacheTime)) + return rule.CacheTime, nil + } + } + + // 没有匹配的规则 + return 0, fmt.Errorf("没有匹配的缓存规则") +} + +// 检查请求方法是否匹配 +func checkRequestMethod(requestMethod, ruleMethods string) bool { + if ruleMethods == "" { + return true // 空字符串表示匹配所有方法 + } + + methods := strings.Split(ruleMethods, ";") + for _, method := range methods { + if strings.TrimSpace(method) == requestMethod { + return true + } + } + return false +} + +// 检查后缀匹配 +func checkSuffixMatch(path, suffixes string) bool { + if suffixes == "" { + return false + } + + suffixList := strings.Split(suffixes, ";") + for _, suffix := range suffixList { + suffix = strings.TrimSpace(suffix) + if suffix != "" && strings.HasSuffix(strings.ToLower(path), strings.ToLower(suffix)) { + return true + } + } + return false +} + +// 检查目录匹配 +func checkDirectoryMatch(path, directories string) bool { + if directories == "" { + return false + } + + dirList := strings.Split(directories, ";") + for _, dir := range dirList { + dir = strings.TrimSpace(dir) + if dir != "" && strings.HasPrefix(strings.ToLower(path), strings.ToLower(dir)) { + return true + } + } + return false +} + +// 检查精确文件匹配 +func checkExactFileMatch(path, files string) bool { + if files == "" { + return false + } + + fileList := strings.Split(files, ";") + for _, file := range fileList { + file = strings.TrimSpace(file) + if file != "" && strings.ToLower(path) == strings.ToLower(file) { + return true + } + } + return false +} + +// CleanExpiredCache 清理过期的缓存文件 +func CleanExpiredCache(cacheDir string) { + zlog.Debug(fmt.Sprintf("开始清理过期缓存文件 目录: %s", cacheDir)) + + // 获取当前时间戳 + now := time.Now().Unix() + + // 遍历缓存目录 + err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // 只处理文件 + if !info.IsDir() { + // 检查是否是缓存文件 + if strings.HasSuffix(path, ".cache") { + // 从文件名中提取过期时间戳 + base := filepath.Base(path) + parts := strings.Split(base, ".") + if len(parts) >= 3 { + expireStr := parts[len(parts)-2] + expire, err := strconv.ParseInt(expireStr, 10, 64) + if err == nil && expire <= now { + // 文件已过期，删除 + zlog.Debug(fmt.Sprintf("删除过期缓存文件: %s", path)) + os.Remove(path) + } + } + } + } + return nil + }) + + if err != nil { + zlog.Error(fmt.Sprintf("清理过期缓存文件失败: %v", err)) + } else { + zlog.Debug(fmt.Sprintf("清理过期缓存文件完成")) + } +} diff --git a/wafenginecore/wafwebcache/webcache_test.go b/wafenginecore/wafwebcache/webcache_test.go new file mode 100644 index 0000000..1262a6c --- /dev/null +++ b/wafenginecore/wafwebcache/webcache_test.go @@ -0,0 +1,559 @@ +package wafwebcache + +import ( + "SamWaf/cache" + "SamWaf/common/zlog" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/wafenginmodel" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "testing" + "time" +) + +// 初始化测试环境 +func setupTestEnv() { + // 初始化缓存 + if global.GCACHE_WAFCACHE == nil { + global.GCACHE_WAFCACHE = cache.InitWafCache() + } + //初始化日志 + zlog.InitZLog(global.GWAF_RELEASE, "json") +} + +// 清理测试环境 +func cleanupTestEnv(tempDir string) { + // 清理临时目录 + if tempDir != "" { + os.RemoveAll(tempDir) + } +} + +// 创建测试请求 +func createTestRequest(method, uri string) *http.Request { + req, _ := http.NewRequest(method, uri, nil) + return req +} + +// 创建测试主机安全配置 +func createTestHostSafe(hostCode string, cacheRules []model.CacheRule) *wafenginmodel.HostSafe { + return &wafenginmodel.HostSafe{ + Host: model.Hosts{ + Code: hostCode, + }, + CacheRule: cacheRules, + } +} + +// 创建测试缓存配置 +func createTestCacheConfig(isEnable int, location, dir string) model.CacheConfig { + return model.CacheConfig{ + IsEnableCache: isEnable, + CacheLocation: location, + CacheDir: dir, + } +} + +// 测试内存缓存存储和加载 +func TestMemoryCache(t *testing.T) { + setupTestEnv() + + hostCode := "test_host" + key := "test_key" + testData := []byte("test data content") + timeout := 10 + + // 测试存储到内存缓存 + storeMemory(hostCode, key, testData, timeout) + + // 测试从内存缓存加载 + data := loadFormMemory(hostCode, key) + if data == nil { + t.Fatalf("从内存缓存加载失败") + } + + if string(data) != string(testData) { + t.Fatalf("加载的数据与存储的数据不匹配，期望: %s, 实际: %s", string(testData), string(data)) + } +} + +// 测试文件缓存存储和加载 +func TestFileCache(t *testing.T) { + // 创建临时目录 + tempDir, err := os.MkdirTemp("", "webcache_test") + if err != nil { + t.Fatalf("创建临时目录失败: %v", err) + } + defer cleanupTestEnv(tempDir) + + hostCode := "test_host" + key := "test_key" + testData := []byte("test file cache content") + timeout := 60 + + // 测试存储到文件缓存 + storeFile(hostCode, key, testData, tempDir, timeout) + + // 测试从文件缓存加载 + data := loadFormFile(hostCode, key, tempDir) + if data == nil { + t.Fatalf("从文件缓存加载失败") + } + + if string(data) != string(testData) { + t.Fatalf("加载的数据与存储的数据不匹配，期望: %s, 实际: %s", string(testData), string(data)) + } +} + +// 测试基本缓存规则匹配 +func TestBasicCacheRuleMatching(t *testing.T) { + // 创建测试规则 + rules := []model.CacheRule{ + { + RuleName: "后缀匹配规则", + RuleType: 1, + RuleContent: ".jpg;.png;.gif", + ParamType: 1, // 忽略参数 + CacheTime: 3600, + Priority: 10, + RequestMethod: "GET", + }, + { + RuleName: "目录匹配规则", + RuleType: 2, + RuleContent: "/static/;/assets/", + ParamType: 1, // 忽略参数 + CacheTime: 7200, + Priority: 20, + RequestMethod: "GET", + }, + { + RuleName: "文件匹配规则", + RuleType: 3, + RuleContent: "/index.html;/home.html", + ParamType: 1, // 忽略参数 + CacheTime: 1800, + Priority: 30, + RequestMethod: "GET;HEAD", + }, + } + + hostSafe := createTestHostSafe("test_host", rules) + cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache") + + // 测试后缀匹配 + req1 := createTestRequest("GET", "http://example.com/image.jpg") + req1.URL.RawQuery = "param=value" + timeout1, err1 := checkCacheRule(hostSafe, cacheConfig, req1) + if err1 != nil { + t.Fatalf("后缀匹配规则测试失败: %v", err1) + } + if timeout1 != 3600 { + t.Fatalf("后缀匹配规则返回的超时时间不正确，期望: 3600, 实际: %d", timeout1) + } + + // 测试目录匹配（优先级更高） + req2 := createTestRequest("GET", "http://example.com/static/image.jpg") + req2.URL.RawQuery = "param=value" + timeout2, err2 := checkCacheRule(hostSafe, cacheConfig, req2) + if err2 != nil { + t.Fatalf("目录匹配规则测试失败: %v", err2) + } + if timeout2 != 7200 { + t.Fatalf("目录匹配规则返回的超时时间不正确，期望: 7200, 实际: %d", timeout2) + } + + // 测试文件匹配（最高优先级） + req3 := createTestRequest("HEAD", "http://example.com/index.html") + req3.URL.RawQuery = "param=value" + timeout3, err3 := checkCacheRule(hostSafe, cacheConfig, req3) + if err3 != nil { + t.Fatalf("文件匹配规则测试失败: %v", err3) + } + if timeout3 != 1800 { + t.Fatalf("文件匹配规则返回的超时时间不正确，期望: 1800, 实际: %d", timeout3) + } + + // 测试不匹配的请求 + req4 := createTestRequest("POST", "http://example.com/api/data") + _, err4 := checkCacheRule(hostSafe, cacheConfig, req4) + if err4 == nil { + t.Fatalf("不匹配的请求应该返回错误") + } +} + +// 测试优先级规则匹配 +func TestPriorityCacheRuleMatching(t *testing.T) { + // 创建测试规则，优先级不同 + rules := []model.CacheRule{ + { + RuleName: "低优先级规则", + RuleType: 1, + RuleContent: ".jpg", + ParamType: 1, + CacheTime: 1000, + Priority: 10, + RequestMethod: "GET", + }, + { + RuleName: "中优先级规则", + RuleType: 1, + RuleContent: ".jpg", + ParamType: 1, + CacheTime: 2000, + Priority: 20, + RequestMethod: "GET", + }, + { + RuleName: "高优先级规则", + RuleType: 1, + RuleContent: ".jpg", + ParamType: 1, + CacheTime: 3000, + Priority: 30, + RequestMethod: "GET", + }, + } + + hostSafe := createTestHostSafe("test_host", rules) + cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache") + + // 测试优先级匹配，应该返回优先级最高的规则 + req := createTestRequest("GET", "http://example.com/image.jpg") + timeout, err := checkCacheRule(hostSafe, cacheConfig, req) + if err != nil { + t.Fatalf("优先级规则测试失败: %v", err) + } + if timeout != 3000 { + t.Fatalf("优先级规则返回的超时时间不正确，期望: 3000, 实际: %d", timeout) + } +} + +// 测试参数处理类型 +func TestParamTypeCacheRuleMatching(t *testing.T) { + // 创建测试规则，参数处理类型不同 + rules := []model.CacheRule{ + { + RuleName: "忽略参数规则", + RuleType: 3, + RuleContent: "/data.html", + ParamType: 1, // 忽略参数 + CacheTime: 1000, + Priority: 10, + RequestMethod: "GET", + }, + { + RuleName: "完整参数规则", + RuleType: 3, + RuleContent: "/data.html", + ParamType: 2, // 完整参数 + CacheTime: 2000, + Priority: 20, + RequestMethod: "GET", + }, + } + + hostSafe := createTestHostSafe("test_host", rules) + cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache") + + // 测试不带参数的请求，应该匹配忽略参数规则 + req2 := createTestRequest("GET", "http://example.com/data.html") + timeout2, err2 := checkCacheRule(hostSafe, cacheConfig, req2) + if err2 != nil { + t.Fatalf("参数处理类型测试失败: %v", err2) + } + if timeout2 != 2000 { + t.Fatalf("参数处理类型测试返回的超时时间不正确，期望: 2000, 实际: %d", timeout2) + } +} + +// 测试具体参数值匹配 +func TestSpecificParamValueMatching(t *testing.T) { + // 创建测试规则，包含具体参数值 + rules := []model.CacheRule{ + { + RuleName: "忽略参数规则", + RuleType: 3, + RuleContent: "/api.html", + ParamType: 1, // 忽略参数 + CacheTime: 1000, + Priority: 10, + RequestMethod: "GET", + }, + { + RuleName: "参数a=1规则", + RuleType: 3, + RuleContent: "/api.html?a=1", + ParamType: 2, // 完整参数 + CacheTime: 2000, + Priority: 20, + RequestMethod: "GET", + }, + { + RuleName: "参数b=2规则", + RuleType: 3, + RuleContent: "/api.html?b=2", + ParamType: 2, // 完整参数 + CacheTime: 3000, + Priority: 30, + RequestMethod: "GET", + }, + { + RuleName: "多参数a=1&b=2规则", + RuleType: 3, + RuleContent: "/api.html?a=1&b=2", + ParamType: 2, // 完整参数 + CacheTime: 4000, + Priority: 40, + RequestMethod: "GET", + }, + } + + hostSafe := createTestHostSafe("test_host", rules) + cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache") + + // 测试匹配a=1参数 + req1 := createTestRequest("GET", "http://example.com/api.html") + req1.URL.RawQuery = "a=1" + timeout1, err1 := checkCacheRule(hostSafe, cacheConfig, req1) + if err1 != nil { + t.Fatalf("具体参数值测试失败: %v", err1) + } + if timeout1 != 2000 { + t.Fatalf("具体参数值测试返回的超时时间不正确，期望: 2000, 实际: %d", timeout1) + } + + // 测试匹配b=2参数 + req2 := createTestRequest("GET", "http://example.com/api.html") + req2.URL.RawQuery = "b=2" + timeout2, err2 := checkCacheRule(hostSafe, cacheConfig, req2) + if err2 != nil { + t.Fatalf("具体参数值测试失败: %v", err2) + } + if timeout2 != 3000 { + t.Fatalf("具体参数值测试返回的超时时间不正确，期望: 3000, 实际: %d", timeout2) + } + + // 测试匹配多参数a=1&b=2 + req3 := createTestRequest("GET", "http://example.com/api.html") + req3.URL.RawQuery = "a=1&b=2" + timeout3, err3 := checkCacheRule(hostSafe, cacheConfig, req3) + if err3 != nil { + t.Fatalf("具体参数值测试失败: %v", err3) + } + if timeout3 != 4000 { + t.Fatalf("具体参数值测试返回的超时时间不正确，期望: 4000, 实际: %d", timeout3) + } + + // 测试参数顺序不同 + req4 := createTestRequest("GET", "http://example.com/api.html") + req4.URL.RawQuery = "b=2&a=1" + timeout4, err4 := checkCacheRule(hostSafe, cacheConfig, req4) + if err4 != nil { + t.Fatalf("参数顺序测试失败: %v", err4) + } + // 这里期望值取决于实现是否考虑参数顺序 + // 如果不考虑顺序，应该返回4000 + // 如果考虑顺序，应该返回1000（忽略参数规则） + if timeout4 != 1000 { + t.Fatalf("参数顺序测试返回的超时时间不正确，期望: 1000, 实际: %d", timeout4) + } + + // 测试不匹配的参数 + req5 := createTestRequest("GET", "http://example.com/api.html") + req5.URL.RawQuery = "c=3" + timeout5, err5 := checkCacheRule(hostSafe, cacheConfig, req5) + if err5 != nil { + t.Fatalf("不匹配参数测试失败: %v", err5) + } + if timeout5 != 1000 { + t.Fatalf("不匹配参数测试返回的超时时间不正确，期望: 1000, 实际: %d", timeout5) + } + + // 测试额外参数 + req6 := createTestRequest("GET", "http://example.com/api.html") + req6.URL.RawQuery = "a=1&b=2&c=3" + timeout6, err6 := checkCacheRule(hostSafe, cacheConfig, req6) + if err6 != nil { + t.Fatalf("额外参数测试失败: %v", err6) + } + if timeout6 != 1000 { + t.Fatalf("额外参数测试返回的超时时间不正确，期望: 1000, 实际: %d", timeout6) + } +} + +// 测试清理过期缓存文件 +func TestCleanExpiredCache(t *testing.T) { + // 创建临时目录 + tempDir, err := os.MkdirTemp("", "webcache_clean_test") + if err != nil { + t.Fatalf("创建临时目录失败: %v", err) + } + defer cleanupTestEnv(tempDir) + + hostCode := "test_host" + hostDir := filepath.Join(tempDir, hostCode) + err = os.MkdirAll(hostDir, 0755) + if err != nil { + t.Fatalf("创建主机目录失败: %v", err) + } + + // 创建一些测试文件 + now := time.Now().Unix() + + // 创建过期文件 + expiredKey := "expired_key" + expiredTimestamp := now - 100 // 过期100秒 + expiredFilename := filepath.Join(hostDir, fmt.Sprintf("%s.%d.cache", expiredKey, expiredTimestamp)) + err = os.WriteFile(expiredFilename, []byte("expired content"), 0644) + if err != nil { + t.Fatalf("创建过期文件失败: %v", err) + } + + // 创建未过期文件 + validKey := "valid_key" + validTimestamp := now + 3600 // 1小时后过期 + validFilename := filepath.Join(hostDir, fmt.Sprintf("%s.%d.cache", validKey, validTimestamp)) + err = os.WriteFile(validFilename, []byte("valid content"), 0644) + if err != nil { + t.Fatalf("创建未过期文件失败: %v", err) + } + + // 执行清理 + CleanExpiredCache(tempDir) + + // 验证过期文件已被删除 + if _, err := os.Stat(expiredFilename); !os.IsNotExist(err) { + t.Fatalf("过期文件应该被删除") + } + + // 验证未过期文件仍然存在 + if _, err := os.Stat(validFilename); os.IsNotExist(err) { + t.Fatalf("未过期文件不应该被删除") + } +} + +// 测试目录不存在的情况 +func TestCleanExpiredCacheNonExistentDir(t *testing.T) { + // 使用一个不存在的目录 + nonExistentDir := "/tmp/non_existent_dir_" + strconv.FormatInt(time.Now().UnixNano(), 10) + + // 确保目录不存在 + if _, err := os.Stat(nonExistentDir); !os.IsNotExist(err) { + t.Fatalf("测试目录不应该存在") + } + + // 执行清理，应该不会出错 + CleanExpiredCache(nonExistentDir) +} + +// 测试请求方法匹配 +func TestRequestMethodMatching(t *testing.T) { + // 测试空字符串（匹配所有方法） + if !checkRequestMethod("GET", "") { + t.Fatalf("空字符串应该匹配所有请求方法") + } + + // 测试单个方法匹配 + if !checkRequestMethod("GET", "GET") { + t.Fatalf("应该匹配相同的请求方法") + } + + // 测试多个方法匹配 + if !checkRequestMethod("POST", "GET;POST;PUT") { + t.Fatalf("应该匹配多个方法中的一个") + } + + // 测试不匹配的方法 + if checkRequestMethod("DELETE", "GET;POST;PUT") { + t.Fatalf("不应该匹配不在列表中的方法") + } +} + +// 测试后缀匹配 +func TestSuffixMatching(t *testing.T) { + // 测试空字符串 + if checkSuffixMatch("/test.jpg", "") { + t.Fatalf("空字符串不应该匹配任何路径") + } + + // 测试单个后缀匹配 + if !checkSuffixMatch("/test.jpg", ".jpg") { + t.Fatalf("应该匹配正确的后缀") + } + + // 测试多个后缀匹配 + if !checkSuffixMatch("/test.png", ".jpg;.png;.gif") { + t.Fatalf("应该匹配多个后缀中的一个") + } + + // 测试不匹配的后缀 + if checkSuffixMatch("/test.txt", ".jpg;.png;.gif") { + t.Fatalf("不应该匹配不在列表中的后缀") + } + + // 测试大小写不敏感 + if !checkSuffixMatch("/test.JPG", ".jpg") { + t.Fatalf("后缀匹配应该不区分大小写") + } +} + +// 测试目录匹配 +func TestDirectoryMatching(t *testing.T) { + // 测试空字符串 + if checkDirectoryMatch("/static/test.jpg", "") { + t.Fatalf("空字符串不应该匹配任何路径") + } + + // 测试单个目录匹配 + if !checkDirectoryMatch("/static/test.jpg", "/static/") { + t.Fatalf("应该匹配正确的目录前缀") + } + + // 测试多个目录匹配 + if !checkDirectoryMatch("/assets/test.png", "/static/;/assets/;/images/") { + t.Fatalf("应该匹配多个目录前缀中的一个") + } + + // 测试不匹配的目录 + if checkDirectoryMatch("/uploads/test.jpg", "/static/;/assets/;/images/") { + t.Fatalf("不应该匹配不在列表中的目录前缀") + } + + // 测试大小写不敏感 + if !checkDirectoryMatch("/STATIC/test.jpg", "/static/") { + t.Fatalf("目录匹配应该不区分大小写") + } +} + +// 测试文件匹配 +func TestExactFileMatching(t *testing.T) { + // 测试空字符串 + if checkExactFileMatch("/index.html", "") { + t.Fatalf("空字符串不应该匹配任何文件") + } + + // 测试单个文件匹配 + if !checkExactFileMatch("/index.html", "/index.html") { + t.Fatalf("应该匹配完全相同的文件路径") + } + + // 测试多个文件匹配 + if !checkExactFileMatch("/home.html", "/index.html;/home.html;/about.html") { + t.Fatalf("应该匹配多个文件路径中的一个") + } + + // 测试不匹配的文件 + if checkExactFileMatch("/contact.html", "/index.html;/home.html;/about.html") { + t.Fatalf("不应该匹配不在列表中的文件路径") + } + + // 测试大小写不敏感 + if !checkExactFileMatch("/INDEX.HTML", "/index.html") { + t.Fatalf("文件匹配应该不区分大小写") + } +} diff --git a/wafenginecore/wafworker.go b/wafenginecore/wafworker.go index ae40fa8..8e2369a 100644 --- a/wafenginecore/wafworker.go +++ b/wafenginecore/wafworker.go @@ -193,6 +193,9 @@ func (waf *WafEngine) LoadHost(inHost model.Hosts) []innerbean.ServerRunTime { } } } + //查询缓存规则 + var cacheRuleList []model.CacheRule + global.GWAF_LOCAL_DB.Where("host_code=? ", inHost.Code).Find(&cacheRuleList) //初始化主机host hostsafe := &wafenginmodel.HostSafe{ @@ -217,6 +220,7 @@ func (waf *WafEngine) LoadHost(inHost model.Hosts) []innerbean.ServerRunTime { AntiCCBean: anticcBean, HttpAuthBases: httpAuthList, BlockingPage: blockingPageMap, + CacheRule: cacheRuleList, } hostsafe.Mux.Lock() defer hostsafe.Mux.Unlock() diff --git a/wafmangeweb/localserver.go b/wafmangeweb/localserver.go index 2baf2fc..39a707c 100644 --- a/wafmangeweb/localserver.go +++ b/wafmangeweb/localserver.go @@ -70,6 +70,7 @@ func (web *WafWebManager) initRouter(r *gin.Engine) { router.ApiGroupApp.InitWafOtpRouter(RouterGroup) router.ApiGroupApp.InitAnalysisRouter(RouterGroup) router.ApiGroupApp.InitWafPrivateInfoRouter(RouterGroup) + router.ApiGroupApp.InitWafCacheRuleRouter(RouterGroup) } if global.GWAF_RELEASE == "true" { diff --git a/waftask/core_scheduler.go b/waftask/core_scheduler.go index a9c183d..405b8fe 100644 --- a/waftask/core_scheduler.go +++ b/waftask/core_scheduler.go @@ -5,6 +5,8 @@ import ( "SamWaf/enums" "fmt" "github.com/go-co-op/gocron" + "strconv" + "strings" "time" ) @@ -28,7 +30,8 @@ func NewTaskScheduler(registry *TaskRegistry) *TaskScheduler { // interval 表示时间间隔 // at 具体的时间 // taskMethod 是具体要执行的任务 -func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, taskMethod string) error { +// taskDaysOfTheWeek 如果是周级别 此处传入周几 +func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, taskMethod string, taskDaysOfTheWeek string) error { var job *gocron.Job var err error @@ -49,6 +52,14 @@ func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, task job, err = ts.Scheduler.Every(interval).Day().At(at).Do(func() { ts.Registry.ExecuteTask(taskMethod) }) + case enums.TASK_WEEKLY: + dayInt, err := strconv.Atoi(strings.TrimSpace(taskDaysOfTheWeek)) + if err != nil { + return fmt.Errorf("无效的星期几格式: %s, 错误: %v", taskDaysOfTheWeek, err) + } + job, err = ts.Scheduler.Every(interval).Weekday(time.Weekday(dayInt)).At(at).Do(func() { + ts.Registry.ExecuteTask(taskMethod) + }) default: return fmt.Errorf("unsupported unit: %s", unit) } diff --git a/waftask/entrance.go b/waftask/entrance.go index fcfeef5..a75061d 100644 --- a/waftask/entrance.go +++ b/waftask/entrance.go @@ -12,6 +12,7 @@ var ( func InitTaskDb() []model.Task { + //每隔秒级别任务 syncTaskToDb(model.Task{ TaskName: "每1秒执行qps清空", TaskUnit: enums.TASK_SECOND, @@ -26,13 +27,6 @@ func InitTaskDb() []model.Task { TaskValue: 1, TaskMethod: enums.TASK_HOST_QPS_CLEAN, }) - syncTaskToDb(model.Task{ - TaskName: "每天凌晨3点进行数据归档操作", - TaskUnit: enums.TASK_DAY, - TaskValue: 1, - TaskAt: "03:00", - TaskMethod: enums.TASK_SHARE_DB, - }) syncTaskToDb(model.Task{ TaskName: "每天10s进行一次统计", TaskUnit: enums.TASK_SECOND, @@ -40,6 +34,15 @@ func InitTaskDb() []model.Task { TaskAt: "", TaskMethod: enums.TASK_COUNTER, }) + syncTaskToDb(model.Task{ + TaskName: "每30s进行健康度检测", + TaskUnit: enums.TASK_SECOND, + TaskValue: 30, + TaskAt: "", + TaskMethod: enums.TASK_HEALTH, + }) + + //每分钟级别 syncTaskToDb(model.Task{ TaskName: "每1分钟进行一次延迟信息提取", TaskUnit: enums.TASK_MIN, @@ -55,18 +58,11 @@ func InitTaskDb() []model.Task { TaskMethod: enums.TASK_LOAD_CONFIG, }) syncTaskToDb(model.Task{ - TaskName: "每1小时进行一次微信提取accesstoken", - TaskUnit: enums.TASK_HOUR, - TaskValue: 1, + TaskName: "每5分钟进行CCWindows旧信息清除", + TaskUnit: enums.TASK_MIN, + TaskValue: 5, TaskAt: "", - TaskMethod: enums.TASK_REFLUSH_WECHAT_ACCESS_TOKEN, - }) - syncTaskToDb(model.Task{ - TaskName: "每天早5点删除历史信息", - TaskUnit: enums.TASK_DAY, - TaskValue: 1, - TaskAt: "05:00", - TaskMethod: enums.TASK_DELETE_HISTORY_INFO, + TaskMethod: enums.TASK_CLEAR_CC_WINDOWS, }) syncTaskToDb(model.Task{ TaskName: "每天30分钟删除历史下载文件", @@ -75,6 +71,33 @@ func InitTaskDb() []model.Task { TaskAt: "", TaskMethod: enums.TASK_DELETE_HISTORY_DOWNLOAD_FILE, }) + + //每小时级别 + syncTaskToDb(model.Task{ + TaskName: "每1小时进行一次微信提取accesstoken", + TaskUnit: enums.TASK_HOUR, + TaskValue: 1, + TaskAt: "", + TaskMethod: enums.TASK_REFLUSH_WECHAT_ACCESS_TOKEN, + }) + + //每日定时级别 + syncTaskToDb(model.Task{ + TaskName: "每天凌晨3点进行数据归档操作", + TaskUnit: enums.TASK_DAY, + TaskValue: 1, + TaskAt: "03:00", + TaskMethod: enums.TASK_SHARE_DB, + }) + + syncTaskToDb(model.Task{ + TaskName: "每天早5点删除历史信息", + TaskUnit: enums.TASK_DAY, + TaskValue: 1, + TaskAt: "05:00", + TaskMethod: enums.TASK_DELETE_HISTORY_INFO, + }) + syncTaskToDb(model.Task{ TaskName: "每天02:13进行SSL证书申请", TaskUnit: enums.TASK_DAY, @@ -117,20 +140,17 @@ func InitTaskDb() []model.Task { TaskAt: "08:00;20:00", TaskMethod: enums.TASK_NOTICE, }) + + //每周的级别 syncTaskToDb(model.Task{ - TaskName: "每30s进行健康度检测", - TaskUnit: enums.TASK_SECOND, - TaskValue: 30, - TaskAt: "", - TaskMethod: enums.TASK_HEALTH, - }) - syncTaskToDb(model.Task{ - TaskName: "每5分钟进行CCWindows旧信息清除", - TaskUnit: enums.TASK_MIN, - TaskValue: 5, - TaskAt: "", - TaskMethod: enums.TASK_CLEAR_CC_WINDOWS, + TaskName: "每周六晚上23:00进行web文件缓存清除", + TaskUnit: enums.TASK_WEEKLY, + TaskValue: 1, + TaskAt: "23:00", + TaskDaysOfTheWeek: "6", + TaskMethod: enums.TASK_CLEAR_WEBCACHE, }) + list, _, _ := wafTaskService.GetList() return list } diff --git a/waftask/task_clear_webcache.go b/waftask/task_clear_webcache.go new file mode 100644 index 0000000..a64cd53 --- /dev/null +++ b/waftask/task_clear_webcache.go @@ -0,0 +1,50 @@ +package waftask + +import ( + "SamWaf/common/zlog" + "SamWaf/model" + "SamWaf/service/waf_service" + "SamWaf/wafenginecore/wafwebcache" + "encoding/json" + "fmt" +) + +func TaskClearWebcache() { + innerLogName := "TaskClearWebcache" + zlog.Info(innerLogName, "开始执行网站缓存清理任务") + + // 获取所有运行中的主机 + hosts := waf_service.WafHostServiceApp.GetAllRunningHostApi() + if len(hosts) == 0 { + zlog.Info(innerLogName, "没有运行中的主机，无需清理缓存") + return + } + + // 遍历所有主机 + for _, host := range hosts { + // 检查主机是否启用了缓存 + var cacheConfig model.CacheConfig + err := json.Unmarshal([]byte(host.CacheJSON), &cacheConfig) + if err != nil { + zlog.Debug(innerLogName, fmt.Sprintf("解析主机 %s 的缓存配置失败: %v", host.Code, err)) + continue + } + + // 如果主机启用了缓存且缓存位置包含文件缓存 + if cacheConfig.IsEnableCache == 1 && (cacheConfig.CacheLocation == "file" || cacheConfig.CacheLocation == "all") { + // 确保缓存目录不为空 + if cacheConfig.CacheDir == "" { + zlog.Warn(innerLogName, fmt.Sprintf("主机 %s 的缓存目录为空", host.Code)) + continue + } + + zlog.Info(innerLogName, fmt.Sprintf("清理主机 %s 的缓存目录: %s", host.Code, cacheConfig.CacheDir)) + // 调用清理函数 + wafwebcache.CleanExpiredCache(cacheConfig.CacheDir) + } else { + zlog.Debug(innerLogName, fmt.Sprintf("主机 %s 未启用文件缓存，跳过清理", host.Code)) + } + } + + zlog.Info(innerLogName, "网站缓存清理任务完成") +}