opensource/SamWaf
3
0
Fork 0
mirror of https://gitee.com/samwaf/SamWaf.git synced 2026-11-02 19:21:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #45 from samwafgo/feat_iprankanalysis

Browse Source 
feat:add ip rank
This commit is contained in:
samwafgo
2024-11-06 16:55:10 +08:00
committed by GitHub
parent d50d5d92bd 3bf8329932
commit 7cb73faec6
12 changed files with 278 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/IPTag.go
+15
View File
@@ -0,0 +1,15 @@
package model
import "SamWaf/model/baseorm"
/*
*
IP Tag 信息
*/
type IPTag struct {
baseorm.BaseOrm
IP string `json:"ip"` //ip
IPTag string `json:"ip_tag"`
Cnt int64 `json:"cnt"` //触发次数
Remarks string `json:"remarks"` //备注
}
model/stats.go
+4 -2
View File
@@ -73,6 +73,8 @@ type StatsDayCount struct {
域名对应的数量[临时]
*/
type StatsIPCount struct {
IP string `json:"ip"` //ip
Count int64 `json:"count"` //数量
IP string `json:"ip"` //ip
IPBelong string `json:"ip_belong"` //归属地
IPTag []IPTag `json:"ip_tags"` //IP标签
Count int64 `json:"count"` //数量
}
service/waf_service/waf_stat.go
+31 -5
View File
@@ -104,12 +104,38 @@ func (receiver *WafStatService) StatHomeSumDayRangeApi(req request.WafStatsDayRa
}
func (receiver *WafStatService) StatHomeSumDayTopIPRangeApi(req request.WafStatsDayRangeReq) (response2.WafIPStats, error) {
var AttackCountOfRange []model.StatsIPCount
global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).Where("day between ? and ? and type = ? ",
req.StartDay, req.EndDay, "阻止").Select("ip,sum(count) as count").Group("ip").Order("sum(count) desc").Limit(10).Scan(&AttackCountOfRange)
var NormalCountOfRange []model.StatsIPCount
global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).Where("day between ? and ? and type = ? ",
req.StartDay, req.EndDay, "放行").Select("ip,sum(count) as count").Group("ip").Order("sum(count) desc").Limit(10).Scan(&NormalCountOfRange)
global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).
Where("day between ? and ? and type = ? ", req.StartDay, req.EndDay, "阻止").
Select("ip,sum(count) as count").Group("ip").Order("sum(count) desc").
Limit(10).
Scan(&AttackCountOfRange)
for i := range AttackCountOfRange {
region := utils.GetCountry(AttackCountOfRange[i].IP)
AttackCountOfRange[i].IPBelong = region[0]
//查询IP标签
var ipTags []model.IPTag
global.GWAF_LOCAL_DB.Where("tenant_id = ? and user_code = ? and ip=?",
global.GWAF_TENANT_ID, global.GWAF_USER_CODE, AttackCountOfRange[i].IP).Find(&ipTags)
AttackCountOfRange[i].IPTag = ipTags
}
var NormalCountOfRange []model.StatsIPCount
global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).
Where("day between ? and ? and type = ? ",
req.StartDay, req.EndDay, "放行").Select("ip,sum(count) as count").
Group("ip").Order("sum(count) desc").
Limit(10).
Scan(&NormalCountOfRange)
for i := range NormalCountOfRange {
region := utils.GetCountry(NormalCountOfRange[i].IP)
NormalCountOfRange[i].IPBelong = region[0]
//查询IP标签
var ipTags []model.IPTag
global.GWAF_LOCAL_DB.Where("tenant_id = ? and user_code = ? and ip=?",
global.GWAF_TENANT_ID, global.GWAF_USER_CODE, NormalCountOfRange[i].IP).Find(&ipTags)
NormalCountOfRange[i].IPTag = ipTags
}
return response2.WafIPStats{
AttackIPOfRange: AttackCountOfRange,
NormalIPOfRange: NormalCountOfRange,
wafdb/localdb.go
+3
View File
@@ -140,6 +140,9 @@ func InitCoreDb(currentDir string) {
//SSL证书
db.AutoMigrate(&model.SslConfig{})
//IPTag
db.AutoMigrate(&model.IPTag{})
global.GWAF_LOCAL_DB.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query)
global.GWAF_LOCAL_DB.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update)
wafdb/patch_sql.go
+18 -4
View File
@@ -17,28 +17,28 @@ func pathLogSql(db *gorm.DB) {
zlog.Info("ready create index maybe use a few minutes ")
err := db.Exec("CREATE INDEX IF NOT EXISTS idx_web_logs_task_flag_time ON web_logs (task_flag, unix_add_time)").Error
if err != nil {
panic("failed to create index: " + err.Error())
panic("failed to create index: idx_web_logs_task_flag_time " + err.Error())
} else {
zlog.Info("db", "idx_web_logs_task_flag_time created")
}
// 创建联合索引
err = db.Exec("CREATE INDEX IF NOT EXISTS idx_web_time_tenant_user_code ON web_logs (unix_add_time, tenant_id,user_code)").Error
if err != nil {
panic("failed to create index: " + err.Error())
panic("failed to create index: idx_web_time_tenant_user_code " + err.Error())
} else {
zlog.Info("db", "idx_web_time_tenant_user_code created")
}
// 详情索引
err = db.Exec("CREATE INDEX IF NOT EXISTS idx_req_uuid_web_logs ON web_logs (REQ_UUID, tenant_id, user_code)").Error
if err != nil {
panic("failed to create index: " + err.Error())
panic("failed to create index:idx_req_uuid_web_logs " + err.Error())
} else {
zlog.Info("db", "idx_unique_req_uuid_web_logs created")
}
// 整体索引
err = db.Exec("CREATE INDEX IF NOT EXISTS idx_tenant_usercode_web_logs ON web_logs ( tenant_id, user_code)").Error
if err != nil {
panic("failed to create index: " + err.Error())
panic("failed to create index: idx_tenant_usercode_web_logs " + err.Error())
} else {
zlog.Info("db", "idx_tenant_usercode_web_logs created")
}
@@ -68,4 +68,18 @@ func pathCoreSql(db *gorm.DB) {
} else {
zlog.Info("db", "hosts :start_status init successfully")
}
//20241106 创建iptagtag索引
err = db.Exec("CREATE UNIQUE INDEX IF NOT EXISTS uni_iptags_full ON ip_tags (user_code, tenant_id, ip, ip_tag)").Error
if err != nil {
panic("failed to create index : uni_iptags_full " + err.Error())
} else {
zlog.Info("db", "uni_iptags_full created")
}
// 创建iptag ip索引
err = db.Exec("CREATE INDEX IF NOT EXISTS idx_iptag_ip ON ip_tags ( user_code, tenant_id, ip)").Error
if err != nil {
panic("failed to create index: idx_iptag_ip " + err.Error())
} else {
zlog.Info("db", "idx_iptag_ip created")
}
}
wafenginecore/checksensitive.go
+3
View File
@@ -23,6 +23,7 @@ func (waf *WafEngine) CheckSensitive(weblogbean *innerbean.WebLog, formValue url
//敏感词检测
matchURLResult := waf.SensitiveManager.MultiPatternSearch([]rune(weblogbean.URL), true)
if len(matchURLResult) > 0 {
weblogbean.RISK_LEVEL = 1
result.IsBlock = true
result.Title = "敏感词检测：" + string(matchURLResult[0].Word)
result.Content = "敏感词内容"
@@ -30,6 +31,7 @@ func (waf *WafEngine) CheckSensitive(weblogbean *innerbean.WebLog, formValue url
}
matchBodyResult := waf.SensitiveManager.MultiPatternSearch([]rune(weblogbean.BODY), true)
if len(matchBodyResult) > 0 {
weblogbean.RISK_LEVEL = 1
result.IsBlock = true
result.Title = "敏感词检测：" + string(matchBodyResult[0].Word)
result.Content = "敏感词内容"
@@ -37,6 +39,7 @@ func (waf *WafEngine) CheckSensitive(weblogbean *innerbean.WebLog, formValue url
}
matchPostFromResult := waf.SensitiveManager.MultiPatternSearch([]rune(weblogbean.POST_FORM), true)
if len(matchPostFromResult) > 0 {
weblogbean.RISK_LEVEL = 1
result.IsBlock = true
result.Title = "敏感词检测：" + string(matchPostFromResult[0].Word)
result.Content = "敏感词内容"
waftask/localtaskcounter.go
+60 -125
View File
@@ -8,13 +8,7 @@ import (
"SamWaf/model"
"SamWaf/model/baseorm"
"SamWaf/service/waf_service"
"SamWaf/utils"
"SamWaf/utils/wechat"
"SamWaf/wafsec"
"encoding/json"
"fmt"
uuid "github.com/satori/go.uuid"
"os"
"time"
)
@@ -55,6 +49,12 @@ type CountCityResult struct {
Count int `json:"count"` //数量
}
type CountIPRuleResult struct {
Ip string `json:"ip"` //用户ip
Rule string `json:"rule"` //规则
Cnt int64 `json:"cnt"` //数量
}
/**
定时统计
*/
@@ -90,11 +90,8 @@ func TaskCounter() {
//一、 主机聚合统计
{
var resultHosts []CountHostResult
explain := global.GWAF_LOCAL_LOG_DB.Debug().Explain("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host",
1, currenyDayMillisecondsBak)
zlog.Debug(explain)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host",
1, currenyDayMillisecondsBak).Scan(&resultHosts)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultHosts)
/****
1.如果不存在则创建
2.如果存在则累加这个周期的统计数
@@ -139,8 +136,8 @@ func TaskCounter() {
//二、 IP聚合统计
{
var resultIP []CountIPResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,src_ip as ip FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host,ip",
1, currenyDayMillisecondsBak).Scan(&resultIP)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,src_ip as ip FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host,ip",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultIP)
/****
1.如果不存在则创建
2.如果存在则累加这个周期的统计数
@@ -188,8 +185,8 @@ func TaskCounter() {
//三、 城市信息聚合统计
{
var resultCitys []CountCityResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,country,province,city FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host,country,province,city",
1, currenyDayMillisecondsBak).Scan(&resultCitys)
global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,country,province,city FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY host_code, user_code,action,tenant_id,day,host,country,province,city",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultCitys)
/****
1.如果不存在则创建
2.如果存在则累加这个周期的统计数
@@ -235,116 +232,54 @@ func TaskCounter() {
}
}
}
//第四 给IP打标签 开始
{
var resultIPRule []CountIPRuleResult
global.GWAF_LOCAL_LOG_DB.Raw("SELECT src_ip as ip ,rule,count(src_ip) as cnt FROM \"web_logs\" where task_flag = ? and unix_add_time > ? and tenant_id = ? and user_code =? GROUP BY user_code,tenant_id, rule,src_ip",
1, currenyDayMillisecondsBak, global.GWAF_TENANT_ID, global.GWAF_USER_CODE).Scan(&resultIPRule)
/****
1.如果不存在则创建
2.如果存在则累加这个IP这个rule的统计数
*/
for _, value := range resultIPRule {
if value.Rule == "" {
value.Rule = "正常"
}
var ipTag model.IPTag
global.GWAF_LOCAL_DB.Where("tenant_id = ? and user_code = ? and ip=? and rule = ?",
global.GWAF_TENANT_ID, global.GWAF_USER_CODE, value.Ip, value.Rule).Find(&ipTag)
if ipTag.IP == "" {
insertIpTag := &model.IPTag{
BaseOrm: baseorm.BaseOrm{
Id: uuid.NewV4().String(),
USER_CODE: global.GWAF_USER_CODE,
Tenant_ID: global.GWAF_TENANT_ID,
CREATE_TIME: customtype.JsonTime(time.Now()),
UPDATE_TIME: customtype.JsonTime(time.Now()),
},
IP: value.Ip,
IPTag: value.Rule,
Cnt: value.Cnt,
Remarks: "",
}
global.GQEQUE_DB.Enqueue(insertIpTag)
} else {
ipTagUpdateMap := map[string]interface{}{
"Cnt": value.Cnt + ipTag.Cnt,
"UPDATE_TIME": customtype.JsonTime(currenyDayBak),
}
updateBean := innerbean.UpdateModel{
Model: model.IPTag{},
Query: "tenant_id = ? and user_code= ? and ip=? and rule = ?",
Update: ipTagUpdateMap,
}
updateBean.Args = append(updateBean.Args, global.GWAF_TENANT_ID, global.GWAF_USER_CODE, value.Ip, value.Rule)
global.GQEQUE_DB.Enqueue(updateBean)
}
}
} //给IP打标签结束
global.GWAF_LAST_UPDATE_TIME = currenyDayBak
global.GWAF_SWITCH_TASK_COUNTER = false
}
func TaskWechatAccessToken() {
zlog.Debug("TaskWechatAccessToken")
wr, err := wechat.GetAppAccessToken("wx8640c6a135dc4b55", "eb57b4a6c445d3624bac7fa3e85efbaf")
if err != nil {
zlog.Error("请求错误GetAppAccessToken")
} else if wr.ErrCode != 0 {
zlog.Error("Wechat Server:", wr.ErrMsg)
} else {
global.GCACHE_WECHAT_ACCESS = wr.AccessToken
zlog.Debug("TaskWechatAccessToken获取到最新token:" + global.GCACHE_WECHAT_ACCESS)
}
}
func TaskStatusNotify() {
zlog.Debug("TaskStatusNotify")
statHomeInfo, err := waf_service.WafStatServiceApp.StatHomeSumDayApi()
if err == nil {
noticeStr := fmt.Sprintf("今日访问量：%d 今天恶意访问量:%d 昨日恶意访问量:%d", statHomeInfo.VisitCountOfToday, statHomeInfo.AttackCountOfToday, statHomeInfo.AttackCountOfYesterday)
global.GQEQUE_MESSAGE_DB.Enqueue(innerbean.OperatorMessageInfo{
BaseMessageInfo: innerbean.BaseMessageInfo{OperaType: "汇总通知"},
OperaCnt: noticeStr,
})
} else {
zlog.Error("TaskStatusNotifyerror", err)
}
}
/*
*
定时删除指定历史信息 通过开关操作
*/
func TaskDeleteHistoryInfo() {
zlog.Debug("TaskDeleteHistoryInfo")
deleteBeforeDay := time.Now().AddDate(0, 0, -int(global.GDATA_DELETE_INTERVAL)).Format("2006-01-02 15:04")
waf_service.WafLogServiceApp.DeleteHistory(deleteBeforeDay)
}
/*
*
定时发送延迟信息
*/
func TaskDelayInfo() {
zlog.Debug("TaskDelayInfo")
models, count, err := waf_service.WafDelayMsgServiceApp.GetAllList()
if err == nil {
if count > 0 {
for i := 0; i < len(models); i++ {
msg := models[i]
sendSuccess := 0
//发送websocket
for _, ws := range global.GWebSocket.GetAllWebSocket() {
if ws != nil {
cmdType := "Info"
if msg.DelayType == "升级结果" {
cmdType = "RELOAD_PAGE"
}
msgBody, _ := json.Marshal(model.MsgDataPacket{
MessageId: uuid.NewV4().String(),
MessageType: msg.DelayType,
MessageData: msg.DelayContent,
MessageAttach: nil,
MessageDateTime: time.Now().Format("2006-01-02 15:04:05"),
MessageUnReadStatus: true,
})
encryptStr, _ := wafsec.AesEncrypt(msgBody, global.GWAF_COMMUNICATION_KEY)
//写入ws数据
msgBytes, err := json.Marshal(
model.MsgPacket{
MsgCode: "200",
MsgDataPacket: encryptStr,
MsgCmdType: cmdType,
})
err = ws.WriteMessage(1, msgBytes)
if err != nil {
continue
} else {
sendSuccess = sendSuccess + 1
}
}
}
if sendSuccess > 0 {
waf_service.WafDelayMsgServiceApp.DelApi(msg.Id)
}
}
}
}
}
// 删除老旧数据
func TaskHistoryDownload() {
currentDir := utils.GetCurrentDir()
downLoadDir := currentDir + "/download"
// 判断备份目录是否存在，不存在则创建
if _, err := os.Stat(downLoadDir); os.IsNotExist(err) {
if err := os.MkdirAll(downLoadDir, os.ModePerm); err != nil {
zlog.Error("创建下载目录失败:", err)
return
}
}
//处理老旧数据
duration := 30 * time.Minute
utils.DeleteOldFiles(downLoadDir, duration)
}
waftask/task_delay_msg.go
+63
View File
@@ -0,0 +1,63 @@
package waftask
import (
"SamWaf/common/zlog"
"SamWaf/global"
"SamWaf/model"
"SamWaf/service/waf_service"
"SamWaf/wafsec"
"encoding/json"
uuid "github.com/satori/go.uuid"
"time"
)
// TaskDelayInfo 定时发送延迟信息
func TaskDelayInfo() {
zlog.Debug("TaskDelayInfo")
models, count, err := waf_service.WafDelayMsgServiceApp.GetAllList()
if err == nil {
if count > 0 {
for i := 0; i < len(models); i++ {
msg := models[i]
sendSuccess := 0
//发送websocket
for _, ws := range global.GWebSocket.GetAllWebSocket() {
if ws != nil {
cmdType := "Info"
if msg.DelayType == "升级结果" {
cmdType = "RELOAD_PAGE"
}
msgBody, _ := json.Marshal(model.MsgDataPacket{
MessageId: uuid.NewV4().String(),
MessageType: msg.DelayType,
MessageData: msg.DelayContent,
MessageAttach: nil,
MessageDateTime: time.Now().Format("2006-01-02 15:04:05"),
MessageUnReadStatus: true,
})
encryptStr, _ := wafsec.AesEncrypt(msgBody, global.GWAF_COMMUNICATION_KEY)
//写入ws数据
msgBytes, err := json.Marshal(
model.MsgPacket{
MsgCode: "200",
MsgDataPacket: encryptStr,
MsgCmdType: cmdType,
})
err = ws.WriteMessage(1, msgBytes)
if err != nil {
continue
} else {
sendSuccess = sendSuccess + 1
}
}
}
if sendSuccess > 0 {
waf_service.WafDelayMsgServiceApp.DelApi(msg.Id)
}
}
}
}
}
waftask/task_history.go
+33
View File
@@ -0,0 +1,33 @@
package waftask
import (
"SamWaf/common/zlog"
"SamWaf/global"
"SamWaf/service/waf_service"
"SamWaf/utils"
"os"
"time"
)
// TaskDeleteHistoryInfo 定时删除指定历史信息 通过开关操作
func TaskDeleteHistoryInfo() {
zlog.Debug("TaskDeleteHistoryInfo")
deleteBeforeDay := time.Now().AddDate(0, 0, -int(global.GDATA_DELETE_INTERVAL)).Format("2006-01-02 15:04")
waf_service.WafLogServiceApp.DeleteHistory(deleteBeforeDay)
}
// TaskHistoryDownload 删除老旧数据
func TaskHistoryDownload() {
currentDir := utils.GetCurrentDir()
downLoadDir := currentDir + "/download"
// 判断备份目录是否存在，不存在则创建
if _, err := os.Stat(downLoadDir); os.IsNotExist(err) {
if err := os.MkdirAll(downLoadDir, os.ModePerm); err != nil {
zlog.Error("创建下载目录失败:", err)
return
}
}
//处理老旧数据
duration := 30 * time.Minute
utils.DeleteOldFiles(downLoadDir, duration)
}
waftask/localtaskcounter_test.go → waftask/task_history_test.go
View File
waftask/task_notify.go
+26
View File
@@ -0,0 +1,26 @@
package waftask
import (
"SamWaf/common/zlog"
"SamWaf/global"
"SamWaf/innerbean"
"SamWaf/service/waf_service"
"fmt"
)
// TaskStatusNotify 汇总通知
func TaskStatusNotify() {
zlog.Debug("TaskStatusNotify")
statHomeInfo, err := waf_service.WafStatServiceApp.StatHomeSumDayApi()
if err == nil {
noticeStr := fmt.Sprintf("今日访问量：%d 今天恶意访问量:%d 昨日恶意访问量:%d", statHomeInfo.VisitCountOfToday, statHomeInfo.AttackCountOfToday, statHomeInfo.AttackCountOfYesterday)
global.GQEQUE_MESSAGE_DB.Enqueue(innerbean.OperatorMessageInfo{
BaseMessageInfo: innerbean.BaseMessageInfo{OperaType: "汇总通知"},
OperaCnt: noticeStr,
})
} else {
zlog.Error("TaskStatusNotifyerror", err)
}
}
waftask/task_third_info.go
+22
View File
@@ -0,0 +1,22 @@
package waftask
import (
"SamWaf/common/zlog"
"SamWaf/global"
"SamWaf/utils/wechat"
)
// TaskWechatAccessToken 初始化access token
func TaskWechatAccessToken() {
zlog.Debug("TaskWechatAccessToken")
wr, err := wechat.GetAppAccessToken("wx*********", "eb57************")
if err != nil {
zlog.Error("请求错误GetAppAccessToken")
} else if wr.ErrCode != 0 {
zlog.Error("Wechat Server:", wr.ErrMsg)
} else {
global.GCACHE_WECHAT_ACCESS = wr.AccessToken
zlog.Debug("TaskWechatAccessToken获取到最新token:" + global.GCACHE_WECHAT_ACCESS)
}
}