Merge pull request #296 from samwafgo/feat_#218_cache

feat:cache file
This commit is contained in:
samwafgo
2025-05-12 17:25:28 +08:00
committed by GitHub
29 changed files with 1578 additions and 67 deletions
+3
View File
@@ -38,6 +38,7 @@ type APIGroup struct {
WafOtpApi
WafAnalysisApi
WafPrivateInfoApi
WafCacheRuleApi
}
var APIGroupAPP = new(APIGroup)
@@ -91,4 +92,6 @@ var (
wafAnalysisService = waf_service.WafAnalysisServiceApp
wafPrivateInfoService = waf_service.WafPrivateInfoServiceApp
wafCacheRuleService = waf_service.WafCacheRuleServiceApp
)
+119
View File
@@ -0,0 +1,119 @@
package api
import (
"SamWaf/enums"
"SamWaf/global"
"SamWaf/model"
"SamWaf/model/common/response"
"SamWaf/model/request"
"SamWaf/model/spec"
"errors"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
)
type WafCacheRuleApi struct {
}
func (w *WafCacheRuleApi) AddApi(c *gin.Context) {
var req request.WafCacheRuleAddReq
err := c.ShouldBindJSON(&req)
if err == nil {
cnt := wafCacheRuleService.CheckIsExistApi(req)
if cnt == 0 {
err = wafCacheRuleService.AddApi(req)
if err == nil {
w.NotifyWaf(req.HostCode)
response.OkWithMessage("添加成功", c)
} else {
response.FailWithMessage("添加失败", c)
}
return
} else {
response.FailWithMessage("当前记录已经存在", c)
return
}
} else {
response.FailWithMessage("解析失败", c)
}
}
func (w *WafCacheRuleApi) GetDetailApi(c *gin.Context) {
var req request.WafCacheRuleDetailReq
err := c.ShouldBind(&req)
if err == nil {
bean := wafCacheRuleService.GetDetailApi(req)
response.OkWithDetailed(bean, "获取成功", c)
} else {
response.FailWithMessage("解析失败", c)
}
}
func (w *WafCacheRuleApi) GetListApi(c *gin.Context) {
var req request.WafCacheRuleSearchReq
err := c.ShouldBindJSON(&req)
if err == nil {
CacheRule, total, _ := wafCacheRuleService.GetListApi(req)
response.OkWithDetailed(response.PageResult{
List: CacheRule,
Total: total,
PageIndex: req.PageIndex,
PageSize: req.PageSize,
}, "获取成功", c)
} else {
response.FailWithMessage("解析失败", c)
}
}
func (w *WafCacheRuleApi) DelApi(c *gin.Context) {
var req request.WafCacheRuleDelReq
err := c.ShouldBind(&req)
if err == nil {
bean := wafCacheRuleService.GetDetailByIdApi(req.Id)
if bean.Id == "" {
response.FailWithMessage("未找到信息", c)
return
}
err = wafCacheRuleService.DelApi(req)
if err != nil && errors.Is(err, gorm.ErrRecordNotFound) {
response.FailWithMessage("请检测参数", c)
} else if err != nil {
response.FailWithMessage("发生错误", c)
} else {
w.NotifyWaf(bean.HostCode)
response.OkWithMessage("删除成功", c)
}
} else {
response.FailWithMessage("解析失败", c)
}
}
func (w *WafCacheRuleApi) ModifyApi(c *gin.Context) {
var req request.WafCacheRuleEditReq
err := c.ShouldBindJSON(&req)
if err == nil {
err = wafCacheRuleService.ModifyApi(req)
if err != nil {
response.FailWithMessage("编辑发生错误"+err.Error(), c)
} else {
w.NotifyWaf(req.HostCode)
response.OkWithMessage("编辑成功", c)
}
} else {
response.FailWithMessage("解析失败", c)
}
}
// NotifyWaf 通知到waf引擎实时生效
func (w *WafCacheRuleApi) NotifyWaf(host_code string) {
var list []model.CacheRule
global.GWAF_LOCAL_DB.Where("host_code = ? ", host_code).Find(&list)
var chanInfo = spec.ChanCommonHost{
HostCode: host_code,
Type: enums.ChanTypeCacheRule,
Content: list,
}
global.GWAF_CHAN_MSG <- chanInfo
}
+4 -3
View File
@@ -9,7 +9,8 @@ import (
func TestCodeGeneration(t *testing.T) {
// 唯一校验定义字段信息的字符串
fieldDefs := []string{
"PrivateKey:private_key:string",
"HostCode:host_code:string",
"RuleName:rule_name:string",
}
// 构造 `uniFields` 列表
@@ -27,6 +28,6 @@ func TestCodeGeneration(t *testing.T) {
}
}
fields := GetStructFields(model.PrivateInfo{})
CodeGeneration("PrivateInfo", fields, uniFields)
fields := GetStructFields(model.CacheRule{})
CodeGeneration("CacheRule", fields, uniFields)
}
+1
View File
@@ -10,4 +10,5 @@ const (
CACHE_CAPTCHA_TRY = "CACHE_CAPTCHA_TRY" //验证码临时
CACHE_CAPTCHA_PASS = "CACHE_CAPTCHA_PASS" //通过验证的码
CACHE_ANNOUNCEMENT = "CACHE_ANNOUNCEMENT" //公告数据
CACHE_WEBFILE = "CACHE_WEBFILE"
)
+1
View File
@@ -15,4 +15,5 @@ const (
ChanTypeSSL
ChanTypeHttpauth
ChanTypeBlockingPage
ChanTypeCacheRule
)
+1
View File
@@ -18,4 +18,5 @@ const (
TASK_HEALTH = "task_health" //健康检测
TASK_CLEAR_CC_WINDOWS = "task_clear_cc_windows" //清除ccWindows记录
TASK_CREATE_DB_INDEX = "task_create_db_index" //创建索引
TASK_CLEAR_WEBCACHE = "task_clear_webcache" //清除缓存
)
+1
View File
@@ -5,4 +5,5 @@ const (
TASK_MIN = "minute"
TASK_HOUR = "hour"
TASK_DAY = "day"
TASK_WEEKLY = "weekly"
)
+8 -1
View File
@@ -306,12 +306,13 @@ func (m *wafSystenService) run() {
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_HEALTH, waftask.TaskHealth)
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_CLEAR_CC_WINDOWS, waftask.TaskCC)
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_CREATE_DB_INDEX, waftask.TaskCreateIndex)
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry.RegisterTask(enums.TASK_CLEAR_WEBCACHE, waftask.TaskClearWebcache)
go waftask.TaskShareDbInfo()
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler = waftask.NewTaskScheduler(globalobj.GWAF_RUNTIME_OBJ_WAF_TaskRegistry)
taskDbList := waftask.InitTaskDb()
for _, task := range taskDbList {
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.ScheduleTask(task.TaskUnit, task.TaskValue, task.TaskAt, task.TaskMethod)
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.ScheduleTask(task.TaskUnit, task.TaskValue, task.TaskAt, task.TaskMethod, task.TaskDaysOfTheWeek)
}
//结束 需要添加到任务注册器里面的
globalobj.GWAF_RUNTIME_OBJ_WAF_TaskScheduler.Start()
@@ -499,6 +500,12 @@ func (m *wafSystenService) run() {
zlog.Debug("远程配置", zap.Any("配置自定义拦截界面信息", msg.Content.(map[string]model.BlockingPage)))
globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].Mux.Unlock()
break
case enums.ChanTypeCacheRule:
globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].Mux.Lock()
globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].CacheRule = msg.Content.([]model.CacheRule)
zlog.Debug("远程配置", zap.Any("配置缓存规则", msg.Content.([]model.CacheRule)))
globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostTarget[globalobj.GWAF_RUNTIME_OBJ_WAF_ENGINE.HostCode[msg.HostCode]].Mux.Unlock()
break
}
//end switch
+17
View File
@@ -0,0 +1,17 @@
package model
import "SamWaf/model/baseorm"
// 缓存规则
type CacheRule struct {
baseorm.BaseOrm
HostCode string `json:"host_code" gorm:"column:host_code"` // 主机代码
RuleName string `json:"rule_name" gorm:"column:rule_name"` // 规则名称
RuleType int `json:"rule_type" gorm:"column:rule_type"` // 规则类型:1-后缀匹配 ,2-指定目录,3-指定文件
RuleContent string `json:"rule_content" gorm:"column:rule_content"` // 规则内容
ParamType int `json:"param_type" gorm:"column:param_type"` // 参数处理:1-忽略参数,2-完整参数
CacheTime int `json:"cache_time" gorm:"column:cache_time"` // 缓存时间(秒):0-不缓存
Priority int `json:"priority" gorm:"column:priority"` // 优先级:数字越大优先级越高
RequestMethod string `json:"request_method" gorm:"column:request_method"` // 请求方式:GET;HEAD;POST等,多个用分号分隔
Remarks string `json:"remarks" gorm:"column:remarks"` // 备注
}
+10
View File
@@ -37,6 +37,7 @@ type Hosts struct {
InsecureSkipVerify int `json:"insecure_skip_verify"` //是否开启后端https证书有效性验证 默认 0 是校验 1 是不校验
CaptchaJSON string `json:"captcha_json"` //验证码配置 json
AntiLeechJSON string `json:"anti_leech_json"` //防盗链配置 json
CacheJSON string `json:"cache_json"` //缓存配置 json
}
type HostsDefense struct {
@@ -77,3 +78,12 @@ type AntiLeechConfig struct {
Action string `json:"action"` // 对于非法引用的处理方式: redirect(重定向) 或 block(直接阻止)
RedirectURL string `json:"redirect_url"` // 重定向URL,当Action为redirect时使用
}
// CacheConfig 缓存配置
type CacheConfig struct {
IsEnableCache int `json:"is_enable_cache"` // 是否开启缓存 1开启 0关闭
CacheLocation string `json:"cache_location"` // 缓存位置: "memory"内存 或 "file"文件 或 "all"内存和文件
CacheDir string `json:"cache_dir"` // 缓存目录,当location为file时使用
MaxFileSizeMB float64 `json:"max_file_size_mb"` // 最大缓存文件大小(MB) 0 是不限制
MaxMemorySizeMB float64 `json:"max_memory_size_mb"` // 最大内存缓存大小(MB),当location为memory时使用 0 是不限制
}
+36
View File
@@ -0,0 +1,36 @@
package request
import "SamWaf/model/common/request"
type WafCacheRuleAddReq struct {
HostCode string `json:"host_code" gorm:"column:host_code" form:"host_code" gorm:"column:host_code"`
RuleName string `json:"rule_name" gorm:"column:rule_name" form:"rule_name" gorm:"column:rule_name"`
RuleType int `json:"rule_type" gorm:"column:rule_type" form:"rule_type" gorm:"column:rule_type"`
RuleContent string `json:"rule_content" gorm:"column:rule_content" form:"rule_content" gorm:"column:rule_content"`
ParamType int `json:"param_type" gorm:"column:param_type" form:"param_type" gorm:"column:param_type"`
CacheTime int `json:"cache_time" gorm:"column:cache_time" form:"cache_time" gorm:"column:cache_time"`
Priority int `json:"priority" gorm:"column:priority" form:"priority" gorm:"column:priority"`
RequestMethod string `json:"request_method" gorm:"column:request_method" form:"request_method" gorm:"column:request_method"`
Remarks string `json:"remarks" gorm:"column:remarks" form:"remarks" gorm:"column:remarks"`
}
type WafCacheRuleEditReq struct {
Id string `json:"id"`
HostCode string `json:"host_code" gorm:"column:host_code" form:"host_code" gorm:"column:host_code"`
RuleName string `json:"rule_name" gorm:"column:rule_name" form:"rule_name" gorm:"column:rule_name"`
RuleType int `json:"rule_type" gorm:"column:rule_type" form:"rule_type" gorm:"column:rule_type"`
RuleContent string `json:"rule_content" gorm:"column:rule_content" form:"rule_content" gorm:"column:rule_content"`
ParamType int `json:"param_type" gorm:"column:param_type" form:"param_type" gorm:"column:param_type"`
CacheTime int `json:"cache_time" gorm:"column:cache_time" form:"cache_time" gorm:"column:cache_time"`
Priority int `json:"priority" gorm:"column:priority" form:"priority" gorm:"column:priority"`
RequestMethod string `json:"request_method" gorm:"column:request_method" form:"request_method" gorm:"column:request_method"`
Remarks string `json:"remarks" gorm:"column:remarks" form:"remarks" gorm:"column:remarks"`
}
type WafCacheRuleDetailReq struct {
Id string `json:"id" form:"id"`
}
type WafCacheRuleDelReq struct {
Id string `json:"id" form:"id"`
}
type WafCacheRuleSearchReq struct {
request.PageInfo
}
+2
View File
@@ -32,6 +32,7 @@ type WafHostAddReq struct {
InsecureSkipVerify int `json:"insecure_skip_verify"` //是否开启后端https证书有效性验证 默认 0 是校验 1 是不校验
CaptchaJSON string `json:"captcha_json"` //验证码配置 json
AntiLeechJSON string `json:"anti_leech_json"` //防盗链配置 json
CacheJSON string `json:"cache_json"` //缓存配置 json
}
type WafHostDelReq struct {
@@ -72,6 +73,7 @@ type WafHostEditReq struct {
InsecureSkipVerify int `json:"insecure_skip_verify"` //是否开启后端https证书有效性验证 默认 0 是校验 1 是不校验
CaptchaJSON string `json:"captcha_json"` //验证码配置 json
AntiLeechJSON string `json:"anti_leech_json"` //防盗链配置 json
CacheJSON string `json:"cache_json"` //缓存配置 json
}
type WafHostGuardStatusReq struct {
+10 -9
View File
@@ -3,12 +3,12 @@ package request
import "SamWaf/model/common/request"
type WafTaskAddReq struct {
TaskName string `json:"task_name" form:"task_name"`
TaskUnit string `json:"task_unit" form:"task_unit"`
TaskValue int `json:"task_value" form:"task_value"`
TaskAt string `json:"task_at" form:"task_at"`
TaskMethod string `json:"task_method" form:"task_method"`
TaskName string `json:"task_name" form:"task_name"`
TaskUnit string `json:"task_unit" form:"task_unit"`
TaskValue int `json:"task_value" form:"task_value"`
TaskAt string `json:"task_at" form:"task_at"`
TaskMethod string `json:"task_method" form:"task_method"`
TaskDaysOfTheWeek string `json:"task_days_of_the_week"` // 周几 在周级别的情况下才起作用
}
type WafTaskEditReq struct {
Id string `json:"id"`
@@ -16,9 +16,10 @@ type WafTaskEditReq struct {
TaskName string `json:"task_name" form:"task_name"`
TaskUnit string `json:"task_unit" form:"task_unit"`
TaskValue int `json:"task_value" form:"task_value"`
TaskAt string `json:"task_at" form:"task_at"`
TaskMethod string `json:"task_method" form:"task_method"`
TaskValue int `json:"task_value" form:"task_value"`
TaskAt string `json:"task_at" form:"task_at"`
TaskMethod string `json:"task_method" form:"task_method"`
TaskDaysOfTheWeek string `json:"task_days_of_the_week"` // 周几 在周级别的情况下才起作用
}
type WafTaskDetailReq struct {
Id string `json:"id" form:"id"`
+6 -5
View File
@@ -4,9 +4,10 @@ import "SamWaf/model/baseorm"
type Task struct {
baseorm.BaseOrm
TaskName string `json:"task_name"` // 任务名称
TaskUnit string `json:"task_unit"` // 单位
TaskValue int `json:"task_value"` // 值
TaskAt string `json:"task_at"` // 特定时间
TaskMethod string `json:"task_method"` //关联方法
TaskName string `json:"task_name"` // 任务名称
TaskUnit string `json:"task_unit"` // 单位
TaskValue int `json:"task_value"` // 值
TaskAt string `json:"task_at"` // 特定时间
TaskMethod string `json:"task_method"` //关联方法
TaskDaysOfTheWeek string `json:"task_days_of_the_week"` // 周几 在周级别的情况下才起作用
}
+1 -1
View File
@@ -29,7 +29,7 @@ type HostSafe struct {
AntiCCBean model.AntiCC //抵御CC
HttpAuthBases []model.HttpAuthBase //HTTP AUTH校验
BlockingPage map[string]model.BlockingPage //自定义拦截界面
CacheRule []model.CacheRule //CacheRule
}
// 负载处理运行对象
+1
View File
@@ -36,6 +36,7 @@ type ApiGroup struct {
WafOtpRouter
AnalysisRouter
WafPrivateInfoRouter
WafCacheRuleRouter
}
type PublicApiGroup struct {
LoginRouter
+19
View File
@@ -0,0 +1,19 @@
package router
import (
"SamWaf/api"
"github.com/gin-gonic/gin"
)
type WafCacheRuleRouter struct {
}
func (receiver *WafCacheRuleRouter) InitWafCacheRuleRouter(group *gin.RouterGroup) {
api := api.APIGroupAPP.WafCacheRuleApi
router := group.Group("")
router.POST("/samwaf/wafhost/cacherule/add", api.AddApi)
router.POST("/samwaf/wafhost/cacherule/list", api.GetListApi)
router.GET("/samwaf/wafhost/cacherule/detail", api.GetDetailApi)
router.POST("/samwaf/wafhost/cacherule/edit", api.ModifyApi)
router.GET("/samwaf/wafhost/cacherule/del", api.DelApi)
}
@@ -0,0 +1,168 @@
package waf_service
import (
"SamWaf/common/uuid"
"SamWaf/customtype"
"SamWaf/global"
"SamWaf/model"
"SamWaf/model/baseorm"
"SamWaf/model/request"
"errors"
"time"
)
type WafCacheRuleService struct{}
var WafCacheRuleServiceApp = new(WafCacheRuleService)
func (receiver *WafCacheRuleService) AddApi(req request.WafCacheRuleAddReq) error {
var bean = &model.CacheRule{
BaseOrm: baseorm.BaseOrm{
Id: uuid.GenUUID(),
USER_CODE: global.GWAF_USER_CODE,
Tenant_ID: global.GWAF_TENANT_ID,
CREATE_TIME: customtype.JsonTime(time.Now()),
UPDATE_TIME: customtype.JsonTime(time.Now()),
},
HostCode: req.HostCode,
RuleName: req.RuleName,
RuleType: req.RuleType,
RuleContent: req.RuleContent,
ParamType: req.ParamType,
CacheTime: req.CacheTime,
Priority: req.Priority,
RequestMethod: req.RequestMethod,
Remarks: req.Remarks,
}
global.GWAF_LOCAL_DB.Create(bean)
return nil
}
func (receiver *WafCacheRuleService) CheckIsExistApi(req request.WafCacheRuleAddReq) int {
var total int64 = 0
/*where条件*/
var whereField = ""
var whereValues []interface{}
//where字段
whereField = ""
if len(req.HostCode) > 0 {
if len(whereField) > 0 {
whereField = whereField + " and "
}
whereField = whereField + " host_code=? "
}
if len(req.RuleName) > 0 {
if len(whereField) > 0 {
whereField = whereField + " and "
}
whereField = whereField + " rule_name=? "
}
//where字段赋值
if len(req.HostCode) > 0 {
if len(whereField) > 0 {
whereValues = append(whereValues, req.HostCode)
}
}
if len(req.RuleName) > 0 {
if len(whereField) > 0 {
whereValues = append(whereValues, req.RuleName)
}
}
global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Where(whereField, whereValues...).Count(&total)
return int(total)
}
func (receiver *WafCacheRuleService) ModifyApi(req request.WafCacheRuleEditReq) error {
// 根据唯一字段生成查询条件(只有在UniFields不为空时才进行存在性检查)
var total int64 = 0
/*where条件*/
var whereField = ""
var whereValues []interface{}
//where字段
whereField = ""
if len(req.HostCode) > 0 {
if len(whereField) > 0 {
whereField = whereField + " and "
}
whereField = whereField + " host_code=? "
}
if len(req.RuleName) > 0 {
if len(whereField) > 0 {
whereField = whereField + " and "
}
whereField = whereField + " rule_name=? "
}
//where字段赋值
if len(req.HostCode) > 0 {
whereValues = append(whereValues, req.HostCode)
}
if len(req.RuleName) > 0 {
whereValues = append(whereValues, req.RuleName)
}
global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Where(whereField, whereValues...).Count(&total)
// 查询是否已存在记录
var bean model.CacheRule
global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Where(whereField, whereValues...).Limit(1).Find(&bean)
if int(total) > 0 && bean.Id != "" && bean.Id != req.Id {
return errors.New("当前记录已经存在")
}
beanMap := map[string]interface{}{
"HostCode": req.HostCode,
"RuleName": req.RuleName,
"RuleType": req.RuleType,
"RuleContent": req.RuleContent,
"ParamType": req.ParamType,
"CacheTime": req.CacheTime,
"Priority": req.Priority,
"RequestMethod": req.RequestMethod,
"Remarks": req.Remarks,
"UPDATE_TIME": customtype.JsonTime(time.Now()),
}
err := global.GWAF_LOCAL_DB.Model(model.CacheRule{}).Where("id = ?", req.Id).Updates(beanMap).Error
return err
}
func (receiver *WafCacheRuleService) GetDetailApi(req request.WafCacheRuleDetailReq) model.CacheRule {
var bean model.CacheRule
global.GWAF_LOCAL_DB.Where("id=?", req.Id).Find(&bean)
return bean
}
func (receiver *WafCacheRuleService) GetDetailByIdApi(id string) model.CacheRule {
var bean model.CacheRule
global.GWAF_LOCAL_DB.Where("id=?", id).Find(&bean)
return bean
}
func (receiver *WafCacheRuleService) GetListApi(req request.WafCacheRuleSearchReq) ([]model.CacheRule, int64, error) {
var list []model.CacheRule
var total int64 = 0
global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Limit(req.PageSize).Offset(req.PageSize * (req.PageIndex - 1)).Find(&list)
global.GWAF_LOCAL_DB.Model(&model.CacheRule{}).Count(&total)
return list, total, nil
}
func (receiver *WafCacheRuleService) DelApi(req request.WafCacheRuleDelReq) error {
var bean model.CacheRule
err := global.GWAF_LOCAL_DB.Where("id = ?", req.Id).First(&bean).Error
if err != nil {
return err
}
err = global.GWAF_LOCAL_DB.Where("id = ?", req.Id).Delete(model.CacheRule{}).Error
return err
}
+2
View File
@@ -63,6 +63,7 @@ func (receiver *WafHostService) AddApi(wafHostAddReq request.WafHostAddReq) (str
InsecureSkipVerify: wafHostAddReq.InsecureSkipVerify,
CaptchaJSON: wafHostAddReq.CaptchaJSON,
AntiLeechJSON: wafHostAddReq.AntiLeechJSON,
CacheJSON: wafHostAddReq.CacheJSON,
}
global.GWAF_LOCAL_DB.Create(wafHost)
return wafHost.Code, nil
@@ -116,6 +117,7 @@ func (receiver *WafHostService) ModifyApi(wafHostEditReq request.WafHostEditReq)
"InsecureSkipVerify": wafHostEditReq.InsecureSkipVerify,
"CaptchaJSON": wafHostEditReq.CaptchaJSON,
"AntiLeechJSON": wafHostEditReq.AntiLeechJSON,
"CacheJSON": wafHostEditReq.CacheJSON,
}
err := global.GWAF_LOCAL_DB.Debug().Model(model.Hosts{}).Where("CODE=?", wafHostEditReq.CODE).Updates(hostMap).Error
+19 -17
View File
@@ -25,11 +25,12 @@ func (receiver *WafTaskService) AddApi(req request.WafTaskAddReq) error {
UPDATE_TIME: customtype.JsonTime(time.Now()),
},
TaskName: req.TaskName,
TaskUnit: req.TaskUnit,
TaskValue: req.TaskValue,
TaskAt: req.TaskAt,
TaskMethod: req.TaskMethod,
TaskName: req.TaskName,
TaskUnit: req.TaskUnit,
TaskValue: req.TaskValue,
TaskAt: req.TaskAt,
TaskMethod: req.TaskMethod,
TaskDaysOfTheWeek: req.TaskDaysOfTheWeek,
}
global.GWAF_LOCAL_DB.Create(bean)
return nil
@@ -45,11 +46,12 @@ func (receiver *WafTaskService) Add(req model.Task) error {
UPDATE_TIME: customtype.JsonTime(time.Now()),
},
TaskName: req.TaskName,
TaskUnit: req.TaskUnit,
TaskValue: req.TaskValue,
TaskAt: req.TaskAt,
TaskMethod: req.TaskMethod,
TaskName: req.TaskName,
TaskUnit: req.TaskUnit,
TaskValue: req.TaskValue,
TaskAt: req.TaskAt,
TaskMethod: req.TaskMethod,
TaskDaysOfTheWeek: req.TaskDaysOfTheWeek,
}
global.GWAF_LOCAL_DB.Create(bean)
return nil
@@ -141,13 +143,13 @@ func (receiver *WafTaskService) ModifyApi(req request.WafTaskEditReq) error {
beanMap := map[string]interface{}{
"TaskName": req.TaskName,
"TaskUnit": req.TaskUnit,
"TaskAt": req.TaskAt,
"TaskMethod": req.TaskMethod,
"TaskValue": req.TaskValue,
"UPDATE_TIME": customtype.JsonTime(time.Now()),
"TaskName": req.TaskName,
"TaskUnit": req.TaskUnit,
"TaskAt": req.TaskAt,
"TaskMethod": req.TaskMethod,
"TaskValue": req.TaskValue,
"TaskDaysOfTheWeek": req.TaskDaysOfTheWeek,
"UPDATE_TIME": customtype.JsonTime(time.Now()),
}
err := global.GWAF_LOCAL_DB.Model(model.Task{}).Where("id = ?", req.Id).Updates(beanMap).Error
+3
View File
@@ -143,6 +143,9 @@ func InitCoreDb(currentDir string) {
//密钥信息
db.AutoMigrate(&model.PrivateInfo{})
//缓存规则
db.AutoMigrate(&model.CacheRule{})
global.GWAF_LOCAL_DB.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query)
global.GWAF_LOCAL_DB.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update)
+59
View File
@@ -15,6 +15,7 @@ import (
"SamWaf/utils"
"SamWaf/wafenginecore/loadbalance"
"SamWaf/wafenginecore/wafhttpcore"
"SamWaf/wafenginecore/wafwebcache"
"SamWaf/wafproxy"
"bytes"
"context"
@@ -25,6 +26,7 @@ import (
goahocorasick "github.com/samwafgo/ahocorasick"
"go.uber.org/zap"
"io"
"io/ioutil"
"net"
"net/http"
_ "net/http/pprof"
@@ -404,6 +406,46 @@ func (waf *WafEngine) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
//检测cache
cacheConfig := model.CacheConfig{
IsEnableCache: 0,
CacheLocation: "",
CacheDir: "",
MaxFileSizeMB: 0,
MaxMemorySizeMB: 0,
}
err := json.Unmarshal([]byte(hostTarget.Host.CacheJSON), &cacheConfig)
if err != nil {
zlog.Debug("解析cache json失败")
}
if cacheConfig.IsEnableCache == 1 {
cacheResp := wafwebcache.LoadWebDataFormCache(w, r, hostTarget, cacheConfig)
if cacheResp != nil {
// 将缓存的响应写回给客户端
for k, v := range cacheResp.Header {
for _, val := range v {
w.Header().Add(k, val)
}
}
w.Header().Add("X-Cache", "HIT")
w.WriteHeader(cacheResp.StatusCode)
// 读取并写入响应体
if cacheResp.Body != nil {
body, _ := ioutil.ReadAll(cacheResp.Body)
cacheResp.Body.Close()
w.Write(body)
// 重新创建一个body供后续使用
cacheResp.Body = ioutil.NopCloser(bytes.NewBuffer(body))
}
// 设置响应并返回
r.Response = cacheResp
return
}
}
//如果正常的流量不记录请求原始包
if global.GCONFIG_RECORD_ALL_SRC_BYTE_INFO == 0 {
weblogbean.SrcByteBody = nil
@@ -767,6 +809,23 @@ func (waf *WafEngine) modifyResponse() func(*http.Response) error {
zlog.Warn(fmt.Sprintf("识别响应内容编码失败,响应日志,敏感词替换 不可用 %v,请求URL: %s", responseEncodingError, r.URL.String()))
}
}
//检测cache
cacheConfig := model.CacheConfig{
IsEnableCache: 0,
CacheLocation: "",
CacheDir: "",
MaxFileSizeMB: 0,
MaxMemorySizeMB: 0,
}
err := json.Unmarshal([]byte(waf.HostTarget[host].Host.CacheJSON), &cacheConfig)
if err != nil {
zlog.Debug("解析cache json失败")
}
if cacheConfig.IsEnableCache == 1 {
wafwebcache.StoreWebDataCache(resp, waf.HostTarget[host], cacheConfig)
}
zlog.Debug("TESTChanllage", weblogfrist.HOST, weblogfrist.URL)
if !isStaticAssist {
datetimeNow := time.Now()
+411
View File
@@ -0,0 +1,411 @@
package wafwebcache
import (
"SamWaf/common/zlog"
"SamWaf/enums"
"SamWaf/global"
"SamWaf/model"
"SamWaf/model/wafenginmodel"
"SamWaf/utils"
"bufio"
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
// LoadWebDataFormCache 从缓存里面拿数据
func LoadWebDataFormCache(w http.ResponseWriter, r *http.Request, hostSafe *wafenginmodel.HostSafe, cacheConfig model.CacheConfig) *http.Response {
key := utils.Md5String(r.RequestURI)
zlog.Debug(fmt.Sprintf("尝试从缓存加载数据 URL: %s, 缓存键: %s, 主机代码: %s, 缓存位置: %s",
r.RequestURI, key, hostSafe.Host.Code, cacheConfig.CacheLocation))
//规则匹配查询
_, err2 := checkCacheRule(hostSafe, cacheConfig, r)
if err2 != nil {
return nil
}
var data []byte = nil
if cacheConfig.CacheLocation == "memory" || cacheConfig.CacheLocation == "all" {
zlog.Debug(fmt.Sprintf("尝试从内存缓存加载 主机代码: %s, 缓存键: %s", hostSafe.Host.Code, key))
memoryData := loadFormMemory(hostSafe.Host.Code, key)
if memoryData == nil {
zlog.Debug(fmt.Sprintf("内存缓存未命中 主机代码: %s, 缓存键: %s", hostSafe.Host.Code, key))
} else {
zlog.Debug(fmt.Sprintf("内存缓存命中 主机代码: %s, 缓存键: %s, 数据大小: %d",
hostSafe.Host.Code, key, len(memoryData)))
data = memoryData
}
}
if data == nil && (cacheConfig.CacheLocation == "file" || cacheConfig.CacheLocation == "all") {
zlog.Debug(fmt.Sprintf("尝试从文件缓存加载 主机代码: %s, 缓存键: %s, 缓存目录: %s",
hostSafe.Host.Code, key, cacheConfig.CacheDir))
fileData := loadFormFile(hostSafe.Host.Code, key, cacheConfig.CacheDir)
if fileData == nil {
zlog.Debug(fmt.Sprintf("文件缓存未命中 主机代码: %s, 缓存键: %s", hostSafe.Host.Code, key))
} else {
zlog.Debug(fmt.Sprintf("文件缓存命中 主机代码: %s, 缓存键: %s, 数据大小: %d",
hostSafe.Host.Code, key, len(fileData)))
data = fileData
}
}
if data == nil {
zlog.Debug(fmt.Sprintf("缓存完全未命中 URL: %s, 缓存键: %s", r.RequestURI, key))
return nil
}
buf := bufio.NewReader(bytes.NewReader(data))
resp, err := http.ReadResponse(buf, nil)
if err != nil {
zlog.Error(fmt.Sprintf("解析缓存响应失败 URL: %s, 缓存键: %s, 错误: %v", r.RequestURI, key, err))
return nil
}
zlog.Debug(fmt.Sprintf("成功从缓存加载响应 URL: %s, 缓存键: %s, 状态码: %d, 响应头数量: %d",
r.RequestURI, key, resp.StatusCode, len(resp.Header)))
return resp
}
// StoreWebDataCache 缓存web数据到cache里面
func StoreWebDataCache(resp *http.Response, hostSafe *wafenginmodel.HostSafe, cacheConfig model.CacheConfig) {
key := utils.Md5String(resp.Request.RequestURI)
zlog.Debug(fmt.Sprintf("尝试存储响应到缓存 URL: %s, 缓存键: %s, 主机代码: %s, 缓存位置: %s",
resp.Request.RequestURI, key, hostSafe.Host.Code, cacheConfig.CacheLocation))
//规则匹配查询 获取缓存时间
timeout, err2 := checkCacheRule(hostSafe, cacheConfig, resp.Request)
if err2 != nil {
return
}
data, err := httputil.DumpResponse(resp, true)
if err != nil {
zlog.Error(fmt.Sprintf("序列化响应失败 URL: %s, 错误: %v", resp.Request.RequestURI, err))
return
}
zlog.Debug(fmt.Sprintf("响应序列化成功 URL: %s, 数据大小: %d", resp.Request.RequestURI, len(data)))
if cacheConfig.CacheLocation == "memory" || cacheConfig.CacheLocation == "all" {
zlog.Debug(fmt.Sprintf("存储到内存缓存 主机代码: %s, 缓存键: %s, 超时时间(秒): %d",
hostSafe.Host.Code, key, timeout))
storeMemory(hostSafe.Host.Code, key, data, timeout, cacheConfig.MaxMemorySizeMB)
}
if cacheConfig.CacheLocation == "file" || cacheConfig.CacheLocation == "all" {
zlog.Debug(fmt.Sprintf("存储到文件缓存 主机代码: %s, 缓存键: %s, 缓存目录: %s, 超时时间(秒): %d",
hostSafe.Host.Code, key, cacheConfig.CacheDir, timeout))
storeFile(hostSafe.Host.Code, key, data, cacheConfig.CacheDir, timeout, cacheConfig.MaxFileSizeMB)
}
zlog.Debug(fmt.Sprintf("响应缓存完成 URL: %s, 缓存键: %s", resp.Request.RequestURI, key))
}
func loadFormMemory(hostCode string, key string) []byte {
cacheKey := enums.CACHE_WEBFILE + hostCode + key
zlog.Debug(fmt.Sprintf("检查内存缓存键 缓存键: %s", cacheKey))
isKeyExist := global.GCACHE_WAFCACHE.IsKeyExist(cacheKey)
if isKeyExist {
zlog.Debug(fmt.Sprintf("内存缓存键存在 缓存键: %s", cacheKey))
data, err := global.GCACHE_WAFCACHE.GetBytes(cacheKey)
if err == nil {
zlog.Debug(fmt.Sprintf("成功获取内存缓存数据 缓存键: %s, 数据大小: %d", cacheKey, len(data)))
return data
}
zlog.Error(fmt.Sprintf("获取内存缓存数据失败 缓存键: %s, 错误: %v", cacheKey, err))
} else {
zlog.Debug(fmt.Sprintf("内存缓存键不存在 缓存键: %s", cacheKey))
}
return nil
}
func loadFormFile(hostCode string, key string, path string) []byte {
safePath := filepath.Join(path, hostCode)
// 获取目录中所有可能的缓存文件
pattern := filepath.Join(safePath, key+".*.cache")
matches, err := filepath.Glob(pattern)
if err != nil || len(matches) == 0 {
zlog.Debug(fmt.Sprintf("文件缓存不存在 匹配模式: %s, 错误: %v", pattern, err))
return nil
}
// 当前时间戳
now := time.Now().Unix()
// 查找有效的缓存文件(未过期的)
var validFile string
var latestExpire int64
for _, match := range matches {
// 从文件名中提取过期时间戳
base := filepath.Base(match)
parts := strings.Split(base, ".")
if len(parts) < 3 {
continue
}
expireStr := parts[len(parts)-2]
expire, err := strconv.ParseInt(expireStr, 10, 64)
if err != nil {
continue
}
// 检查是否过期
if expire > now {
// 如果有多个有效文件,选择过期时间最晚的
if expire > latestExpire {
validFile = match
latestExpire = expire
}
} else {
// 文件已过期,可以选择删除
zlog.Debug(fmt.Sprintf("缓存文件已过期,准备删除: %s", match))
os.Remove(match)
}
}
if validFile == "" {
zlog.Debug(fmt.Sprintf("没有找到有效的缓存文件 键: %s", key))
return nil
}
zlog.Debug(fmt.Sprintf("找到有效的缓存文件: %s, 过期时间戳: %d", validFile, latestExpire))
data, err := ioutil.ReadFile(validFile)
if err != nil {
zlog.Error(fmt.Sprintf("读取文件缓存失败 文件路径: %s, 错误: %v", validFile, err))
return nil
}
zlog.Debug(fmt.Sprintf("成功读取文件缓存 文件路径: %s, 数据大小: %d", validFile, len(data)))
return data
}
func storeMemory(hostCode string, key string, data []byte, timeout int, maxSize float64) {
cacheKey := enums.CACHE_WEBFILE + hostCode + key
zlog.Debug(fmt.Sprintf("存储到内存缓存 缓存键: %s, 数据大小: %d", cacheKey, len(data)))
// 检查内存大小限制
dataSizeMB := float64(len(data)) / (1024 * 1024)
if maxSize > 0 && dataSizeMB > maxSize {
zlog.Warn(fmt.Sprintf("内存缓存大小超过限制 缓存键: %s, 数据大小: %.2fMB, 最大限制: %.2fMB",
cacheKey, dataSizeMB, maxSize))
return
}
if timeout > 0 {
zlog.Debug(fmt.Sprintf("设置内存缓存带超时 缓存键: %s, 超时时间(秒): %d", cacheKey, timeout))
global.GCACHE_WAFCACHE.SetWithTTlRenewTime(cacheKey, data, time.Duration(timeout)*time.Second)
} else {
zlog.Debug(fmt.Sprintf("设置内存缓存长期有效 缓存键: %s", cacheKey))
global.GCACHE_WAFCACHE.SetWithTTlRenewTime(cacheKey, data, time.Duration(87600)*time.Hour)
}
}
func storeFile(hostCode string, key string, data []byte, path string, timeout int, maxSize float64) {
safePath := filepath.Join(path, hostCode)
zlog.Debug(fmt.Sprintf("准备存储文件缓存 目录路径: %s, 缓存键: %s", safePath, key))
// 确保缓存目录存在
if _, err := os.Stat(safePath); os.IsNotExist(err) {
zlog.Debug(fmt.Sprintf("缓存目录不存在,创建目录 目录路径: %s", safePath))
err := os.MkdirAll(safePath, 0755)
if err != nil {
zlog.Error(fmt.Sprintf("创建缓存目录失败 目录路径: %s, 错误: %v", safePath, err))
return
}
}
// 计算过期时间戳
var expireTimestamp int64
if timeout > 0 {
expireTimestamp = time.Now().Unix() + int64(timeout)
} else {
// 如果没有设置超时,使用一个很大的值(10年)
expireTimestamp = time.Now().Unix() + 315360000
}
// 文件名中包含过期时间戳
filename := filepath.Join(safePath, fmt.Sprintf("%s.%d.cache", key, expireTimestamp))
zlog.Debug(fmt.Sprintf("写入文件缓存 文件路径: %s, 数据大小: %d, 过期时间戳: %d", filename, len(data), expireTimestamp))
// 检查文件大小限制
dataSizeMB := float64(len(data)) / (1024 * 1024)
if maxSize > 0 && dataSizeMB > maxSize {
zlog.Warn(fmt.Sprintf("文件缓存大小超过限制 文件路径: %s, 数据大小: %.2fMB, 最大限制: %.2fMB",
filename, dataSizeMB, maxSize))
return
}
err := ioutil.WriteFile(filename, data, 0644)
if err != nil {
zlog.Error(fmt.Sprintf("写入文件缓存失败 文件路径: %s, 错误: %v", filename, err))
return
}
zlog.Debug(fmt.Sprintf("文件缓存写入成功 文件路径: %s", filename))
}
func checkCacheRule(hostSafe *wafenginmodel.HostSafe, cacheConfig model.CacheConfig, r *http.Request) (int, error) {
//规则匹配查询
if hostSafe.CacheRule == nil || len(hostSafe.CacheRule) == 0 {
return 0, fmt.Errorf("没有配置缓存规则")
}
requestURI := r.URL.RequestURI()
requestMethod := r.Method
// 按优先级排序规则(优先级数字越大越优先)
sort.Slice(hostSafe.CacheRule, func(i, j int) bool {
return hostSafe.CacheRule[i].Priority > hostSafe.CacheRule[j].Priority
})
pureRequestUrl := r.URL.Path
// 循环检查每个规则
for _, rule := range hostSafe.CacheRule {
finalUrl := requestURI
if rule.ParamType == 1 {
finalUrl = pureRequestUrl
}
// 检查请求方法是否匹配
if !checkRequestMethod(requestMethod, rule.RequestMethod) {
continue
}
// 根据规则类型进行匹配
matched := false
switch rule.RuleType {
case 1: // 后缀匹配
matched = checkSuffixMatch(finalUrl, rule.RuleContent)
case 2: // 指定目录
matched = checkDirectoryMatch(finalUrl, rule.RuleContent)
case 3: // 指定文件
matched = checkExactFileMatch(finalUrl, rule.RuleContent)
}
if matched {
zlog.Debug(fmt.Sprintf("缓存规则匹配成功: %s, 缓存时间: %d秒", rule.RuleName, rule.CacheTime))
return rule.CacheTime, nil
}
}
// 没有匹配的规则
return 0, fmt.Errorf("没有匹配的缓存规则")
}
// 检查请求方法是否匹配
func checkRequestMethod(requestMethod, ruleMethods string) bool {
if ruleMethods == "" {
return true // 空字符串表示匹配所有方法
}
methods := strings.Split(ruleMethods, ";")
for _, method := range methods {
if strings.TrimSpace(method) == requestMethod {
return true
}
}
return false
}
// 检查后缀匹配
func checkSuffixMatch(path, suffixes string) bool {
if suffixes == "" {
return false
}
suffixList := strings.Split(suffixes, ";")
for _, suffix := range suffixList {
suffix = strings.TrimSpace(suffix)
if suffix != "" && strings.HasSuffix(strings.ToLower(path), strings.ToLower(suffix)) {
return true
}
}
return false
}
// 检查目录匹配
func checkDirectoryMatch(path, directories string) bool {
if directories == "" {
return false
}
dirList := strings.Split(directories, ";")
for _, dir := range dirList {
dir = strings.TrimSpace(dir)
if dir != "" && strings.HasPrefix(strings.ToLower(path), strings.ToLower(dir)) {
return true
}
}
return false
}
// 检查精确文件匹配
func checkExactFileMatch(path, files string) bool {
if files == "" {
return false
}
fileList := strings.Split(files, ";")
for _, file := range fileList {
file = strings.TrimSpace(file)
if file != "" && strings.ToLower(path) == strings.ToLower(file) {
return true
}
}
return false
}
// CleanExpiredCache 清理过期的缓存文件
func CleanExpiredCache(cacheDir string) {
zlog.Debug(fmt.Sprintf("开始清理过期缓存文件 目录: %s", cacheDir))
// 获取当前时间戳
now := time.Now().Unix()
// 遍历缓存目录
err := filepath.Walk(cacheDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// 只处理文件
if !info.IsDir() {
// 检查是否是缓存文件
if strings.HasSuffix(path, ".cache") {
// 从文件名中提取过期时间戳
base := filepath.Base(path)
parts := strings.Split(base, ".")
if len(parts) >= 3 {
expireStr := parts[len(parts)-2]
expire, err := strconv.ParseInt(expireStr, 10, 64)
if err == nil && expire <= now {
// 文件已过期,删除
zlog.Debug(fmt.Sprintf("删除过期缓存文件: %s", path))
os.Remove(path)
}
}
}
}
return nil
})
if err != nil {
zlog.Error(fmt.Sprintf("清理过期缓存文件失败: %v", err))
} else {
zlog.Debug(fmt.Sprintf("清理过期缓存文件完成"))
}
}
+559
View File
@@ -0,0 +1,559 @@
package wafwebcache
import (
"SamWaf/cache"
"SamWaf/common/zlog"
"SamWaf/global"
"SamWaf/model"
"SamWaf/model/wafenginmodel"
"fmt"
"net/http"
"os"
"path/filepath"
"strconv"
"testing"
"time"
)
// 初始化测试环境
func setupTestEnv() {
// 初始化缓存
if global.GCACHE_WAFCACHE == nil {
global.GCACHE_WAFCACHE = cache.InitWafCache()
}
//初始化日志
zlog.InitZLog(global.GWAF_RELEASE, "json")
}
// 清理测试环境
func cleanupTestEnv(tempDir string) {
// 清理临时目录
if tempDir != "" {
os.RemoveAll(tempDir)
}
}
// 创建测试请求
func createTestRequest(method, uri string) *http.Request {
req, _ := http.NewRequest(method, uri, nil)
return req
}
// 创建测试主机安全配置
func createTestHostSafe(hostCode string, cacheRules []model.CacheRule) *wafenginmodel.HostSafe {
return &wafenginmodel.HostSafe{
Host: model.Hosts{
Code: hostCode,
},
CacheRule: cacheRules,
}
}
// 创建测试缓存配置
func createTestCacheConfig(isEnable int, location, dir string) model.CacheConfig {
return model.CacheConfig{
IsEnableCache: isEnable,
CacheLocation: location,
CacheDir: dir,
}
}
// 测试内存缓存存储和加载
func TestMemoryCache(t *testing.T) {
setupTestEnv()
hostCode := "test_host"
key := "test_key"
testData := []byte("test data content")
timeout := 10
// 测试存储到内存缓存
storeMemory(hostCode, key, testData, timeout)
// 测试从内存缓存加载
data := loadFormMemory(hostCode, key)
if data == nil {
t.Fatalf("从内存缓存加载失败")
}
if string(data) != string(testData) {
t.Fatalf("加载的数据与存储的数据不匹配,期望: %s, 实际: %s", string(testData), string(data))
}
}
// 测试文件缓存存储和加载
func TestFileCache(t *testing.T) {
// 创建临时目录
tempDir, err := os.MkdirTemp("", "webcache_test")
if err != nil {
t.Fatalf("创建临时目录失败: %v", err)
}
defer cleanupTestEnv(tempDir)
hostCode := "test_host"
key := "test_key"
testData := []byte("test file cache content")
timeout := 60
// 测试存储到文件缓存
storeFile(hostCode, key, testData, tempDir, timeout)
// 测试从文件缓存加载
data := loadFormFile(hostCode, key, tempDir)
if data == nil {
t.Fatalf("从文件缓存加载失败")
}
if string(data) != string(testData) {
t.Fatalf("加载的数据与存储的数据不匹配,期望: %s, 实际: %s", string(testData), string(data))
}
}
// 测试基本缓存规则匹配
func TestBasicCacheRuleMatching(t *testing.T) {
// 创建测试规则
rules := []model.CacheRule{
{
RuleName: "后缀匹配规则",
RuleType: 1,
RuleContent: ".jpg;.png;.gif",
ParamType: 1, // 忽略参数
CacheTime: 3600,
Priority: 10,
RequestMethod: "GET",
},
{
RuleName: "目录匹配规则",
RuleType: 2,
RuleContent: "/static/;/assets/",
ParamType: 1, // 忽略参数
CacheTime: 7200,
Priority: 20,
RequestMethod: "GET",
},
{
RuleName: "文件匹配规则",
RuleType: 3,
RuleContent: "/index.html;/home.html",
ParamType: 1, // 忽略参数
CacheTime: 1800,
Priority: 30,
RequestMethod: "GET;HEAD",
},
}
hostSafe := createTestHostSafe("test_host", rules)
cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache")
// 测试后缀匹配
req1 := createTestRequest("GET", "http://example.com/image.jpg")
req1.URL.RawQuery = "param=value"
timeout1, err1 := checkCacheRule(hostSafe, cacheConfig, req1)
if err1 != nil {
t.Fatalf("后缀匹配规则测试失败: %v", err1)
}
if timeout1 != 3600 {
t.Fatalf("后缀匹配规则返回的超时时间不正确,期望: 3600, 实际: %d", timeout1)
}
// 测试目录匹配(优先级更高)
req2 := createTestRequest("GET", "http://example.com/static/image.jpg")
req2.URL.RawQuery = "param=value"
timeout2, err2 := checkCacheRule(hostSafe, cacheConfig, req2)
if err2 != nil {
t.Fatalf("目录匹配规则测试失败: %v", err2)
}
if timeout2 != 7200 {
t.Fatalf("目录匹配规则返回的超时时间不正确,期望: 7200, 实际: %d", timeout2)
}
// 测试文件匹配(最高优先级)
req3 := createTestRequest("HEAD", "http://example.com/index.html")
req3.URL.RawQuery = "param=value"
timeout3, err3 := checkCacheRule(hostSafe, cacheConfig, req3)
if err3 != nil {
t.Fatalf("文件匹配规则测试失败: %v", err3)
}
if timeout3 != 1800 {
t.Fatalf("文件匹配规则返回的超时时间不正确,期望: 1800, 实际: %d", timeout3)
}
// 测试不匹配的请求
req4 := createTestRequest("POST", "http://example.com/api/data")
_, err4 := checkCacheRule(hostSafe, cacheConfig, req4)
if err4 == nil {
t.Fatalf("不匹配的请求应该返回错误")
}
}
// 测试优先级规则匹配
func TestPriorityCacheRuleMatching(t *testing.T) {
// 创建测试规则,优先级不同
rules := []model.CacheRule{
{
RuleName: "低优先级规则",
RuleType: 1,
RuleContent: ".jpg",
ParamType: 1,
CacheTime: 1000,
Priority: 10,
RequestMethod: "GET",
},
{
RuleName: "中优先级规则",
RuleType: 1,
RuleContent: ".jpg",
ParamType: 1,
CacheTime: 2000,
Priority: 20,
RequestMethod: "GET",
},
{
RuleName: "高优先级规则",
RuleType: 1,
RuleContent: ".jpg",
ParamType: 1,
CacheTime: 3000,
Priority: 30,
RequestMethod: "GET",
},
}
hostSafe := createTestHostSafe("test_host", rules)
cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache")
// 测试优先级匹配,应该返回优先级最高的规则
req := createTestRequest("GET", "http://example.com/image.jpg")
timeout, err := checkCacheRule(hostSafe, cacheConfig, req)
if err != nil {
t.Fatalf("优先级规则测试失败: %v", err)
}
if timeout != 3000 {
t.Fatalf("优先级规则返回的超时时间不正确,期望: 3000, 实际: %d", timeout)
}
}
// 测试参数处理类型
func TestParamTypeCacheRuleMatching(t *testing.T) {
// 创建测试规则,参数处理类型不同
rules := []model.CacheRule{
{
RuleName: "忽略参数规则",
RuleType: 3,
RuleContent: "/data.html",
ParamType: 1, // 忽略参数
CacheTime: 1000,
Priority: 10,
RequestMethod: "GET",
},
{
RuleName: "完整参数规则",
RuleType: 3,
RuleContent: "/data.html",
ParamType: 2, // 完整参数
CacheTime: 2000,
Priority: 20,
RequestMethod: "GET",
},
}
hostSafe := createTestHostSafe("test_host", rules)
cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache")
// 测试不带参数的请求,应该匹配忽略参数规则
req2 := createTestRequest("GET", "http://example.com/data.html")
timeout2, err2 := checkCacheRule(hostSafe, cacheConfig, req2)
if err2 != nil {
t.Fatalf("参数处理类型测试失败: %v", err2)
}
if timeout2 != 2000 {
t.Fatalf("参数处理类型测试返回的超时时间不正确,期望: 2000, 实际: %d", timeout2)
}
}
// 测试具体参数值匹配
func TestSpecificParamValueMatching(t *testing.T) {
// 创建测试规则,包含具体参数值
rules := []model.CacheRule{
{
RuleName: "忽略参数规则",
RuleType: 3,
RuleContent: "/api.html",
ParamType: 1, // 忽略参数
CacheTime: 1000,
Priority: 10,
RequestMethod: "GET",
},
{
RuleName: "参数a=1规则",
RuleType: 3,
RuleContent: "/api.html?a=1",
ParamType: 2, // 完整参数
CacheTime: 2000,
Priority: 20,
RequestMethod: "GET",
},
{
RuleName: "参数b=2规则",
RuleType: 3,
RuleContent: "/api.html?b=2",
ParamType: 2, // 完整参数
CacheTime: 3000,
Priority: 30,
RequestMethod: "GET",
},
{
RuleName: "多参数a=1&b=2规则",
RuleType: 3,
RuleContent: "/api.html?a=1&b=2",
ParamType: 2, // 完整参数
CacheTime: 4000,
Priority: 40,
RequestMethod: "GET",
},
}
hostSafe := createTestHostSafe("test_host", rules)
cacheConfig := createTestCacheConfig(1, "memory", "/tmp/cache")
// 测试匹配a=1参数
req1 := createTestRequest("GET", "http://example.com/api.html")
req1.URL.RawQuery = "a=1"
timeout1, err1 := checkCacheRule(hostSafe, cacheConfig, req1)
if err1 != nil {
t.Fatalf("具体参数值测试失败: %v", err1)
}
if timeout1 != 2000 {
t.Fatalf("具体参数值测试返回的超时时间不正确,期望: 2000, 实际: %d", timeout1)
}
// 测试匹配b=2参数
req2 := createTestRequest("GET", "http://example.com/api.html")
req2.URL.RawQuery = "b=2"
timeout2, err2 := checkCacheRule(hostSafe, cacheConfig, req2)
if err2 != nil {
t.Fatalf("具体参数值测试失败: %v", err2)
}
if timeout2 != 3000 {
t.Fatalf("具体参数值测试返回的超时时间不正确,期望: 3000, 实际: %d", timeout2)
}
// 测试匹配多参数a=1&b=2
req3 := createTestRequest("GET", "http://example.com/api.html")
req3.URL.RawQuery = "a=1&b=2"
timeout3, err3 := checkCacheRule(hostSafe, cacheConfig, req3)
if err3 != nil {
t.Fatalf("具体参数值测试失败: %v", err3)
}
if timeout3 != 4000 {
t.Fatalf("具体参数值测试返回的超时时间不正确,期望: 4000, 实际: %d", timeout3)
}
// 测试参数顺序不同
req4 := createTestRequest("GET", "http://example.com/api.html")
req4.URL.RawQuery = "b=2&a=1"
timeout4, err4 := checkCacheRule(hostSafe, cacheConfig, req4)
if err4 != nil {
t.Fatalf("参数顺序测试失败: %v", err4)
}
// 这里期望值取决于实现是否考虑参数顺序
// 如果不考虑顺序,应该返回4000
// 如果考虑顺序,应该返回1000(忽略参数规则)
if timeout4 != 1000 {
t.Fatalf("参数顺序测试返回的超时时间不正确,期望: 1000, 实际: %d", timeout4)
}
// 测试不匹配的参数
req5 := createTestRequest("GET", "http://example.com/api.html")
req5.URL.RawQuery = "c=3"
timeout5, err5 := checkCacheRule(hostSafe, cacheConfig, req5)
if err5 != nil {
t.Fatalf("不匹配参数测试失败: %v", err5)
}
if timeout5 != 1000 {
t.Fatalf("不匹配参数测试返回的超时时间不正确,期望: 1000, 实际: %d", timeout5)
}
// 测试额外参数
req6 := createTestRequest("GET", "http://example.com/api.html")
req6.URL.RawQuery = "a=1&b=2&c=3"
timeout6, err6 := checkCacheRule(hostSafe, cacheConfig, req6)
if err6 != nil {
t.Fatalf("额外参数测试失败: %v", err6)
}
if timeout6 != 1000 {
t.Fatalf("额外参数测试返回的超时时间不正确,期望: 1000, 实际: %d", timeout6)
}
}
// 测试清理过期缓存文件
func TestCleanExpiredCache(t *testing.T) {
// 创建临时目录
tempDir, err := os.MkdirTemp("", "webcache_clean_test")
if err != nil {
t.Fatalf("创建临时目录失败: %v", err)
}
defer cleanupTestEnv(tempDir)
hostCode := "test_host"
hostDir := filepath.Join(tempDir, hostCode)
err = os.MkdirAll(hostDir, 0755)
if err != nil {
t.Fatalf("创建主机目录失败: %v", err)
}
// 创建一些测试文件
now := time.Now().Unix()
// 创建过期文件
expiredKey := "expired_key"
expiredTimestamp := now - 100 // 过期100秒
expiredFilename := filepath.Join(hostDir, fmt.Sprintf("%s.%d.cache", expiredKey, expiredTimestamp))
err = os.WriteFile(expiredFilename, []byte("expired content"), 0644)
if err != nil {
t.Fatalf("创建过期文件失败: %v", err)
}
// 创建未过期文件
validKey := "valid_key"
validTimestamp := now + 3600 // 1小时后过期
validFilename := filepath.Join(hostDir, fmt.Sprintf("%s.%d.cache", validKey, validTimestamp))
err = os.WriteFile(validFilename, []byte("valid content"), 0644)
if err != nil {
t.Fatalf("创建未过期文件失败: %v", err)
}
// 执行清理
CleanExpiredCache(tempDir)
// 验证过期文件已被删除
if _, err := os.Stat(expiredFilename); !os.IsNotExist(err) {
t.Fatalf("过期文件应该被删除")
}
// 验证未过期文件仍然存在
if _, err := os.Stat(validFilename); os.IsNotExist(err) {
t.Fatalf("未过期文件不应该被删除")
}
}
// 测试目录不存在的情况
func TestCleanExpiredCacheNonExistentDir(t *testing.T) {
// 使用一个不存在的目录
nonExistentDir := "/tmp/non_existent_dir_" + strconv.FormatInt(time.Now().UnixNano(), 10)
// 确保目录不存在
if _, err := os.Stat(nonExistentDir); !os.IsNotExist(err) {
t.Fatalf("测试目录不应该存在")
}
// 执行清理,应该不会出错
CleanExpiredCache(nonExistentDir)
}
// 测试请求方法匹配
func TestRequestMethodMatching(t *testing.T) {
// 测试空字符串(匹配所有方法)
if !checkRequestMethod("GET", "") {
t.Fatalf("空字符串应该匹配所有请求方法")
}
// 测试单个方法匹配
if !checkRequestMethod("GET", "GET") {
t.Fatalf("应该匹配相同的请求方法")
}
// 测试多个方法匹配
if !checkRequestMethod("POST", "GET;POST;PUT") {
t.Fatalf("应该匹配多个方法中的一个")
}
// 测试不匹配的方法
if checkRequestMethod("DELETE", "GET;POST;PUT") {
t.Fatalf("不应该匹配不在列表中的方法")
}
}
// 测试后缀匹配
func TestSuffixMatching(t *testing.T) {
// 测试空字符串
if checkSuffixMatch("/test.jpg", "") {
t.Fatalf("空字符串不应该匹配任何路径")
}
// 测试单个后缀匹配
if !checkSuffixMatch("/test.jpg", ".jpg") {
t.Fatalf("应该匹配正确的后缀")
}
// 测试多个后缀匹配
if !checkSuffixMatch("/test.png", ".jpg;.png;.gif") {
t.Fatalf("应该匹配多个后缀中的一个")
}
// 测试不匹配的后缀
if checkSuffixMatch("/test.txt", ".jpg;.png;.gif") {
t.Fatalf("不应该匹配不在列表中的后缀")
}
// 测试大小写不敏感
if !checkSuffixMatch("/test.JPG", ".jpg") {
t.Fatalf("后缀匹配应该不区分大小写")
}
}
// 测试目录匹配
func TestDirectoryMatching(t *testing.T) {
// 测试空字符串
if checkDirectoryMatch("/static/test.jpg", "") {
t.Fatalf("空字符串不应该匹配任何路径")
}
// 测试单个目录匹配
if !checkDirectoryMatch("/static/test.jpg", "/static/") {
t.Fatalf("应该匹配正确的目录前缀")
}
// 测试多个目录匹配
if !checkDirectoryMatch("/assets/test.png", "/static/;/assets/;/images/") {
t.Fatalf("应该匹配多个目录前缀中的一个")
}
// 测试不匹配的目录
if checkDirectoryMatch("/uploads/test.jpg", "/static/;/assets/;/images/") {
t.Fatalf("不应该匹配不在列表中的目录前缀")
}
// 测试大小写不敏感
if !checkDirectoryMatch("/STATIC/test.jpg", "/static/") {
t.Fatalf("目录匹配应该不区分大小写")
}
}
// 测试文件匹配
func TestExactFileMatching(t *testing.T) {
// 测试空字符串
if checkExactFileMatch("/index.html", "") {
t.Fatalf("空字符串不应该匹配任何文件")
}
// 测试单个文件匹配
if !checkExactFileMatch("/index.html", "/index.html") {
t.Fatalf("应该匹配完全相同的文件路径")
}
// 测试多个文件匹配
if !checkExactFileMatch("/home.html", "/index.html;/home.html;/about.html") {
t.Fatalf("应该匹配多个文件路径中的一个")
}
// 测试不匹配的文件
if checkExactFileMatch("/contact.html", "/index.html;/home.html;/about.html") {
t.Fatalf("不应该匹配不在列表中的文件路径")
}
// 测试大小写不敏感
if !checkExactFileMatch("/INDEX.HTML", "/index.html") {
t.Fatalf("文件匹配应该不区分大小写")
}
}
+4
View File
@@ -193,6 +193,9 @@ func (waf *WafEngine) LoadHost(inHost model.Hosts) []innerbean.ServerRunTime {
}
}
}
//查询缓存规则
var cacheRuleList []model.CacheRule
global.GWAF_LOCAL_DB.Where("host_code=? ", inHost.Code).Find(&cacheRuleList)
//初始化主机host
hostsafe := &wafenginmodel.HostSafe{
@@ -217,6 +220,7 @@ func (waf *WafEngine) LoadHost(inHost model.Hosts) []innerbean.ServerRunTime {
AntiCCBean: anticcBean,
HttpAuthBases: httpAuthList,
BlockingPage: blockingPageMap,
CacheRule: cacheRuleList,
}
hostsafe.Mux.Lock()
defer hostsafe.Mux.Unlock()
+1
View File
@@ -70,6 +70,7 @@ func (web *WafWebManager) initRouter(r *gin.Engine) {
router.ApiGroupApp.InitWafOtpRouter(RouterGroup)
router.ApiGroupApp.InitAnalysisRouter(RouterGroup)
router.ApiGroupApp.InitWafPrivateInfoRouter(RouterGroup)
router.ApiGroupApp.InitWafCacheRuleRouter(RouterGroup)
}
if global.GWAF_RELEASE == "true" {
+12 -1
View File
@@ -5,6 +5,8 @@ import (
"SamWaf/enums"
"fmt"
"github.com/go-co-op/gocron"
"strconv"
"strings"
"time"
)
@@ -28,7 +30,8 @@ func NewTaskScheduler(registry *TaskRegistry) *TaskScheduler {
// interval 表示时间间隔
// at 具体的时间
// taskMethod 是具体要执行的任务
func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, taskMethod string) error {
// taskDaysOfTheWeek 如果是周级别 此处传入周几
func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, taskMethod string, taskDaysOfTheWeek string) error {
var job *gocron.Job
var err error
@@ -49,6 +52,14 @@ func (ts *TaskScheduler) ScheduleTask(unit string, interval int, at string, task
job, err = ts.Scheduler.Every(interval).Day().At(at).Do(func() {
ts.Registry.ExecuteTask(taskMethod)
})
case enums.TASK_WEEKLY:
dayInt, err := strconv.Atoi(strings.TrimSpace(taskDaysOfTheWeek))
if err != nil {
return fmt.Errorf("无效的星期几格式: %s, 错误: %v", taskDaysOfTheWeek, err)
}
job, err = ts.Scheduler.Every(interval).Weekday(time.Weekday(dayInt)).At(at).Do(func() {
ts.Registry.ExecuteTask(taskMethod)
})
default:
return fmt.Errorf("unsupported unit: %s", unit)
}
+50 -30
View File
@@ -12,6 +12,7 @@ var (
func InitTaskDb() []model.Task {
//每隔秒级别任务
syncTaskToDb(model.Task{
TaskName: "每1秒执行qps清空",
TaskUnit: enums.TASK_SECOND,
@@ -26,13 +27,6 @@ func InitTaskDb() []model.Task {
TaskValue: 1,
TaskMethod: enums.TASK_HOST_QPS_CLEAN,
})
syncTaskToDb(model.Task{
TaskName: "每天凌晨3点进行数据归档操作",
TaskUnit: enums.TASK_DAY,
TaskValue: 1,
TaskAt: "03:00",
TaskMethod: enums.TASK_SHARE_DB,
})
syncTaskToDb(model.Task{
TaskName: "每天10s进行一次统计",
TaskUnit: enums.TASK_SECOND,
@@ -40,6 +34,15 @@ func InitTaskDb() []model.Task {
TaskAt: "",
TaskMethod: enums.TASK_COUNTER,
})
syncTaskToDb(model.Task{
TaskName: "每30s进行健康度检测",
TaskUnit: enums.TASK_SECOND,
TaskValue: 30,
TaskAt: "",
TaskMethod: enums.TASK_HEALTH,
})
//每分钟级别
syncTaskToDb(model.Task{
TaskName: "每1分钟进行一次延迟信息提取",
TaskUnit: enums.TASK_MIN,
@@ -55,18 +58,11 @@ func InitTaskDb() []model.Task {
TaskMethod: enums.TASK_LOAD_CONFIG,
})
syncTaskToDb(model.Task{
TaskName: "每1小时进行一次微信提取accesstoken",
TaskUnit: enums.TASK_HOUR,
TaskValue: 1,
TaskName: "每5分钟进行CCWindows旧信息清除",
TaskUnit: enums.TASK_MIN,
TaskValue: 5,
TaskAt: "",
TaskMethod: enums.TASK_REFLUSH_WECHAT_ACCESS_TOKEN,
})
syncTaskToDb(model.Task{
TaskName: "每天早5点删除历史信息",
TaskUnit: enums.TASK_DAY,
TaskValue: 1,
TaskAt: "05:00",
TaskMethod: enums.TASK_DELETE_HISTORY_INFO,
TaskMethod: enums.TASK_CLEAR_CC_WINDOWS,
})
syncTaskToDb(model.Task{
TaskName: "每天30分钟删除历史下载文件",
@@ -75,6 +71,33 @@ func InitTaskDb() []model.Task {
TaskAt: "",
TaskMethod: enums.TASK_DELETE_HISTORY_DOWNLOAD_FILE,
})
//每小时级别
syncTaskToDb(model.Task{
TaskName: "每1小时进行一次微信提取accesstoken",
TaskUnit: enums.TASK_HOUR,
TaskValue: 1,
TaskAt: "",
TaskMethod: enums.TASK_REFLUSH_WECHAT_ACCESS_TOKEN,
})
//每日定时级别
syncTaskToDb(model.Task{
TaskName: "每天凌晨3点进行数据归档操作",
TaskUnit: enums.TASK_DAY,
TaskValue: 1,
TaskAt: "03:00",
TaskMethod: enums.TASK_SHARE_DB,
})
syncTaskToDb(model.Task{
TaskName: "每天早5点删除历史信息",
TaskUnit: enums.TASK_DAY,
TaskValue: 1,
TaskAt: "05:00",
TaskMethod: enums.TASK_DELETE_HISTORY_INFO,
})
syncTaskToDb(model.Task{
TaskName: "每天02:13进行SSL证书申请",
TaskUnit: enums.TASK_DAY,
@@ -117,20 +140,17 @@ func InitTaskDb() []model.Task {
TaskAt: "08:00;20:00",
TaskMethod: enums.TASK_NOTICE,
})
//每周的级别
syncTaskToDb(model.Task{
TaskName: "每30s进行健康度检测",
TaskUnit: enums.TASK_SECOND,
TaskValue: 30,
TaskAt: "",
TaskMethod: enums.TASK_HEALTH,
})
syncTaskToDb(model.Task{
TaskName: "每5分钟进行CCWindows旧信息清除",
TaskUnit: enums.TASK_MIN,
TaskValue: 5,
TaskAt: "",
TaskMethod: enums.TASK_CLEAR_CC_WINDOWS,
TaskName: "每周六晚上23:00进行web文件缓存清除",
TaskUnit: enums.TASK_WEEKLY,
TaskValue: 1,
TaskAt: "23:00",
TaskDaysOfTheWeek: "6",
TaskMethod: enums.TASK_CLEAR_WEBCACHE,
})
list, _, _ := wafTaskService.GetList()
return list
}
+50
View File
@@ -0,0 +1,50 @@
package waftask
import (
"SamWaf/common/zlog"
"SamWaf/model"
"SamWaf/service/waf_service"
"SamWaf/wafenginecore/wafwebcache"
"encoding/json"
"fmt"
)
func TaskClearWebcache() {
innerLogName := "TaskClearWebcache"
zlog.Info(innerLogName, "开始执行网站缓存清理任务")
// 获取所有运行中的主机
hosts := waf_service.WafHostServiceApp.GetAllRunningHostApi()
if len(hosts) == 0 {
zlog.Info(innerLogName, "没有运行中的主机,无需清理缓存")
return
}
// 遍历所有主机
for _, host := range hosts {
// 检查主机是否启用了缓存
var cacheConfig model.CacheConfig
err := json.Unmarshal([]byte(host.CacheJSON), &cacheConfig)
if err != nil {
zlog.Debug(innerLogName, fmt.Sprintf("解析主机 %s 的缓存配置失败: %v", host.Code, err))
continue
}
// 如果主机启用了缓存且缓存位置包含文件缓存
if cacheConfig.IsEnableCache == 1 && (cacheConfig.CacheLocation == "file" || cacheConfig.CacheLocation == "all") {
// 确保缓存目录不为空
if cacheConfig.CacheDir == "" {
zlog.Warn(innerLogName, fmt.Sprintf("主机 %s 的缓存目录为空", host.Code))
continue
}
zlog.Info(innerLogName, fmt.Sprintf("清理主机 %s 的缓存目录: %s", host.Code, cacheConfig.CacheDir))
// 调用清理函数
wafwebcache.CleanExpiredCache(cacheConfig.CacheDir)
} else {
zlog.Debug(innerLogName, fmt.Sprintf("主机 %s 未启用文件缓存,跳过清理", host.Code))
}
}
zlog.Info(innerLogName, "网站缓存清理任务完成")
}