opensource/SamWaf
3
0
Fork 0
mirror of https://gitee.com/samwaf/SamWaf.git synced 2026-10-31 02:01:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat:增加程序退出做清理工作

Browse Source
This commit is contained in:
samwaf
2023-12-30 07:58:22 +08:00
parent 82aa9bb67b
commit cdb512cf96
16 changed files with 230 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/waf_log.go
+17 -3
View File
@@ -4,6 +4,7 @@ import (
"SamWaf/global"
"SamWaf/model/common/response"
"SamWaf/model/request"
response2 "SamWaf/model/response"
"github.com/gin-gonic/gin"
)
@@ -29,9 +30,6 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) {
var req request.WafAttackLogSearch
err := c.ShouldBindJSON(&req)
if err == nil {
/*//TOOD 模拟意外退出
os.Exit(-1) //退出进程*/
if global.GDATA_CURRENT_CHANGE {
//如果正在切换库 跳过
response.FailWithMessage("正在切换数据库请等待", c)
@@ -68,3 +66,19 @@ func (w *WafLogAPi) GetListByHostCodeApi(c *gin.Context) {
response.FailWithMessage("解析失败", c)
}
}
func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) {
wafShareList, _ := wafShareDbService.GetAllShareDbApi()
allShareDbRep := make([]response2.AllShareDbRep, len(wafShareList)) // 创建数组
for i, _ := range wafShareList {
allShareDbRep[i] = response2.AllShareDbRep{
StartTime: wafShareList[i].StartTime,
EndTime: wafShareList[i].EndTime,
FileName: wafShareList[i].FileName,
Cnt: wafShareList[i].Cnt,
}
}
response.OkWithDetailed(allShareDbRep, "获取成功", c)
}
global/global.go
+4 -5
View File
@@ -39,7 +39,6 @@ var (
GWAF_LOCAL_DB *gorm.DB //通用本地数据库，尊重用户隐私
GWAF_LOCAL_LOG_DB *gorm.DB //通用本地数据库存日志数据，尊重用户隐私
GWAF_LOCAL_CUSTOM_LOG_DB *gorm.DB //通用本地数据库存历史日志数据， 尊重用户隐私
GWAF_LOCAL_STATS_DB *gorm.DB //通用本地数据库存放统计数据，尊重用户隐私
GWAF_REMOTE_DB *gorm.DB //仅当用户使用云数据库
GWAF_LOCAL_SERVER_PORT int = 26666 // 本地local端口
@@ -75,10 +74,10 @@ var (
GQEQUE_MESSAGE_DB Dequelib.Deque //发送消息队列
/******数据库处理参数*****/
GDATA_BATCH_INSERT int = 1000 //最大批量插入
GDATA_SHARE_DB_SIZE int64 = 100 * 10000 //100w 进行分库 100*10000
GDATA_CURRENT_CHANGE bool = false //当前是否正在切换
GDATA_BATCH_INSERT int = 1000 //最大批量插入
GDATA_SHARE_DB_SIZE int64 = 100 * 10000 //100w 进行分库 100*10000
GDATA_CURRENT_CHANGE bool = false //当前是否正在切换
GDATA_CURRENT_LOG_DB_MAP map[string]*gorm.DB //当前自定义的数据库连接 TODO 如果用户打开了多个 会不会影响内存
/******WebSocket*********/
GWebSocket *model.WebSocketOnline
localwaf/src/apis/waflog/attacklog.ts
+9
View File
@@ -7,3 +7,12 @@ export function attacklogList(params) {
params: params
})
}
//查询存档日志库列表
export function allsharedblist(params) {
return request({
url: 'waflog/attack/allsharedb',
method: 'get',
params: params
})
}
localwaf/src/pages/waf/attack/detail/index.vue
+14 -4
View File
@@ -93,11 +93,11 @@
<t-list-item >
<t-list-item-meta title="请求BODY" ></t-list-item-meta>
</t-list-item>
<t-textarea v-model="detail_data.body" :autosize="{ minRows: 3, maxRows: 5 }" readonly @blur="handleMouseSelect('body')"/>
<t-textarea v-model="detail_data.body" :autosize="{ minRows: 3, maxRows: 5 }" readonly @blur="handleMouseSelect('body')"/>
<t-list-item >
<t-list-item-meta title="请求Form" ></t-list-item-meta>
</t-list-item>
<t-textarea v-model="detail_data.post_form" :autosize="{ minRows: 3, maxRows: 5 }" readonly />
<t-textarea v-model="detail_data.post_form" :autosize="{ minRows: 3, maxRows: 5 }" readonly />
</t-list>
</t-card>
<t-card title="响应数据">
@@ -156,19 +156,29 @@
watch: {
'$route.query.req_uuid'(newVal, oldVal) {
console.log('route.query.req_uuid changed', newVal, oldVal)
this.getDetail(newVal)
if(newVal!=undefined){
this.getDetail(newVal)
}
},
},
methods: {
backPage(){
　history.go(-1)
},
getDetail(id) {
getDetail(uuid_and_name) {
if(uuid_and_name == undefined){
return
}
let arr = uuid_and_name.split("#");
let id = arr[0]
let current_db_name = arr[1]
let that = this
this.$request
.get('/waflog/attack/detail', {
params: {
REQ_UUID: id,
current_db_name:current_db_name
}
})
.then((res) => {
localwaf/src/pages/waf/attack/index.vue
+32 -3
View File
@@ -55,6 +55,15 @@
placeholder="请输入访问方法" :style="{ width: '100px' }" />
</t-form-item>
</t-col>
<t-col :flex="1">
<t-form-item label="日志归档库" name="sharedb">
<t-select v-model="searchformData.current_db_name" clearable :style="{ width: '150px' }">
<t-option v-for="(item, index) in share_db_dic" :value="index" :label="item" :key="index">
{{ item }}
</t-option>
</t-select>
</t-form-item>
</t-col>
</t-row>
</t-col>
@@ -106,7 +115,7 @@
import { SearchIcon } from 'tdesign-icons-vue';
import Trend from '@/components/trend/index.vue';
import { prefix } from '@/config/global';
import { attacklogList } from '@/apis/waflog/attacklog';
import { allsharedblist } from '@/apis/waflog/attacklog';
import { NowDate, ConvertStringToUnix, ConvertDateToString, ConvertUnixToDate } from '@/utils/date';
import {
@@ -319,9 +328,12 @@
method: "",
unix_add_time_begin: "",
unix_add_time_end: "",
current_db_name:"local_log.db",
},
//主机字典
host_dic: {}
host_dic: {},
//日志存档字典
share_db_dic: {}
};
},
computed: {
@@ -379,6 +391,7 @@
}
this.loadHostList()
this.loadShareDbList()
this.getList("")
},
watch: {
@@ -399,6 +412,22 @@
next(); // 继续后续的导航解析过程
},
methods: {
loadShareDbList() {
let that = this;
allsharedblist("").then((res) => {
let resdata = res
console.log("loadShareDbList",resdata)
if (resdata.code === 0) {
let share_options = resdata.data;
for (let i = 0; i < share_options.length; i++) {
that.share_db_dic[share_options[i].file_name] = share_options[i].file_name+"("+share_options[i].cnt+")"
}
}
})
.catch((e : Error) => {
console.log(e);
})
},
loadHostList() {
let that = this;
allhost().then((res) => {
@@ -481,7 +510,7 @@
{
path: '/waf/wafattacklogdetail',
query: {
req_uuid: req_uuid,
req_uuid: req_uuid+"#"+this.searchformData.current_db_name,
},
},
);
main.go
+39 -6
View File
@@ -10,7 +10,9 @@ import (
"SamWaf/plugin"
"SamWaf/utils"
"SamWaf/utils/zlog"
"SamWaf/wafdb"
"SamWaf/wafenginecore"
"SamWaf/wafsafeclear"
"SamWaf/wafsnowflake"
"SamWaf/waftask"
"crypto/tls"
@@ -20,6 +22,7 @@ import (
"github.com/kardianos/service"
"go.uber.org/zap"
"golang.org/x/time/rate"
"gorm.io/gorm"
"log"
"net/http"
"os"
@@ -43,6 +46,7 @@ func (m *wafSystenService) Start(s service.Service) error {
// Stop 是服务停止时调用的方法
func (m *wafSystenService) Stop(s service.Service) error {
wafsafeclear.SafeClear()
return nil
}
@@ -59,6 +63,29 @@ func NeverExit(name string, f func()) {
// run 是服务的主要逻辑
func (m *wafSystenService) run() {
/*// 启动一个 goroutine 来处理信号
go func() {
// 创建一个通道来接收信号
signalCh := make(chan os.Signal, 1)
// 根据操作系统设置不同的信号监听
if runtime.GOOS == "windows" {
signal.Notify(signalCh, syscall.SIGINT,os.Interrupt, syscall.SIGTERM,os.Kill)
} else { // Linux 系统
signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
}
for {
select {
case sig := <-signalCh:
fmt.Println("接收到信号:", sig)
wafsafeclear.SafeClear()
// 在这里执行你的清理操作或退出应用程序的代码
os.Exit(0)
}
}
}()*/
// 在这里编写你的服务逻辑代码
fmt.Println("Service is running...")
//初始化cache
@@ -67,7 +94,8 @@ func (m *wafSystenService) run() {
global.GWAF_MEASURE_PROCESS_DEQUEENGINE = cache.InitWafOnlyLockWrite()
// 创建 Snowflake 实例
global.GWAF_SNOWFLAKE_GEN = wafsnowflake.NewSnowflake(1609459200000, 1, 1) // 设置epoch时间、机器ID和数据中心ID
//提前初始化
global.GDATA_CURRENT_LOG_DB_MAP = map[string]*gorm.DB{}
rversion := "初始化系统 版本号：" + global.GWAF_RELEASE_VERSION_NAME + "(" + global.GWAF_RELEASE_VERSION + ")"
if global.GWAF_RELEASE == "false" {
rversion = rversion + " 调试版本"
@@ -100,9 +128,9 @@ func (m *wafSystenService) run() {
}
//初始化本地数据库
wafenginecore.InitCoreDb("")
wafenginecore.InitLogDb("")
wafenginecore.InitStatsDb("")
wafdb.InitCoreDb("")
wafdb.InitLogDb("")
wafdb.InitStatsDb("")
//初始化队列引擎
wafenginecore.InitDequeEngine()
//启动队列消费
@@ -198,6 +226,7 @@ func (m *wafSystenService) run() {
//脱敏处理初始化
global.GWAF_DLP, _ = dlp.NewEngine("wafDlp")
global.GWAF_DLP.ApplyConfigDefault()
for {
select {
case msg := <-global.GWAF_CHAN_MSG:
@@ -271,7 +300,6 @@ func (m *wafSystenService) run() {
}
break
case host := <-global.GWAF_CHAN_HOST:
wafEngine.HostTarget[host.Host+":"+strconv.Itoa(host.Port)].Host.GUARD_STATUS = host.GUARD_STATUS
zlog.Debug("规则", zap.Any("主机", host))
break
@@ -347,6 +375,11 @@ func main() {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM)
/*_, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()*/
//doWork(ctx)
// 以服务方式运行
if len(os.Args) > 1 {
command := os.Args[1]
@@ -364,7 +397,7 @@ func main() {
zlog.Info("main server run false")
global.GWAF_RUNTIME_SERVER_TYPE = service.Interactive()
}
//defer wafsafeclear.SafeClear()
// 以常规方式运行
err = s.Run()
if err != nil {
model/request/waf_attack_log_detail.go
+2 -1
View File
@@ -1,5 +1,6 @@
package request
type WafAttackLogDetailReq struct {
REQ_UUID string `json:"req_uuid"`
CurrrentDbName string `json:"current_db_name"`
REQ_UUID string `json:"req_uuid"`
}
model/request/waf_attack_log_search.go
+1
View File
@@ -3,6 +3,7 @@ package request
import "SamWaf/model/common/request"
type WafAttackLogSearch struct {
CurrrentDbName string `json:"current_db_name"`
HostCode string `json:"host_code" form:"host_code"` //主机码
Rule string `json:"rule" form:"rule"` //规则名
Action string `json:"action" form:"action"` //状态
model/response/all_host.go
+9
View File
@@ -1,6 +1,15 @@
package response
import "SamWaf/customtype"
type AllHostRep struct {
Code string `json:"value"` //唯一码
Host string `json:"label"` //域名
}
type AllShareDbRep struct {
StartTime customtype.JsonTime `json:"start_time"` //开始时间
EndTime customtype.JsonTime `json:"end_time"` //结束时间
FileName string `json:"file_name"` //文件名
Cnt int64 `json:"cnt"` //当前数量
}
router/waf_log.go
+2
View File
@@ -13,4 +13,6 @@ func (receiver *LogRouter) InitLogRouter(group *gin.RouterGroup) {
wafLogRouter := group.Group("")
wafLogRouter.POST("/samwaf/waflog/attack/list", logApi.GetListApi)
wafLogRouter.GET("/samwaf/waflog/attack/detail", logApi.GetDetailApi)
wafLogRouter.GET("/samwaf/waflog/attack/allsharedb", logApi.GetAllShareDbApi)
}
service/waf_service/waf_log.go
+25 -17
View File
@@ -3,8 +3,8 @@ package waf_service
import (
"SamWaf/global"
"SamWaf/innerbean"
request2 "SamWaf/model/common/request"
"SamWaf/model/request"
"SamWaf/wafdb"
)
type WafLogService struct{}
@@ -18,31 +18,39 @@ func (receiver *WafLogService) AddApi(log innerbean.WebLog) error {
func (receiver *WafLogService) ModifyApi(log innerbean.WebLog) error {
return nil
}
func (receiver *WafLogService) GetDetailApi(wafAttackDetailReq request.WafAttackLogDetailReq) (innerbean.WebLog, error) {
func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) (innerbean.WebLog, error) {
var weblog innerbean.WebLog
global.GWAF_LOCAL_LOG_DB.Where("REQ_UUID=?", wafAttackDetailReq.REQ_UUID).Find(&weblog)
if len(req.CurrrentDbName) == 0 || req.CurrrentDbName == "local_log.db" {
global.GWAF_LOCAL_LOG_DB.Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog)
} else {
wafdb.InitManaulLogDb("", req.CurrrentDbName)
global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog)
}
return weblog, nil
}
func (receiver *WafLogService) GetListApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) {
func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) {
var total int64 = 0
var weblogs []innerbean.WebLog
whereCondition := &request.WafAttackLogSearch{
HostCode: log.HostCode,
Rule: log.Rule,
Action: log.Action,
SrcIp: log.SrcIp,
StatusCode: log.StatusCode,
Method: log.Method,
PageInfo: request2.PageInfo{
PageIndex: 0,
PageSize: 0,
Keyword: "",
},
HostCode: req.HostCode,
Rule: req.Rule,
Action: req.Action,
SrcIp: req.SrcIp,
StatusCode: req.StatusCode,
Method: req.Method,
}
if len(req.CurrrentDbName) == 0 || req.CurrrentDbName == "local_log.db" {
global.GWAF_LOCAL_LOG_DB.Limit(req.PageSize).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Offset(req.PageSize * (req.PageIndex - 1)).Order("create_time desc").Find(&weblogs)
global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Count(&total)
} else {
wafdb.InitManaulLogDb("", req.CurrrentDbName)
global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Debug().Limit(req.PageSize).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Offset(req.PageSize * (req.PageIndex - 1)).Order("create_time desc").Find(&weblogs)
global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Model(&innerbean.WebLog{}).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Count(&total)
}
global.GWAF_LOCAL_LOG_DB.Limit(log.PageSize).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", log.UnixAddTimeBegin, log.UnixAddTimeEnd).Offset(log.PageSize * (log.PageIndex - 1)).Order("create_time desc").Find(&weblogs)
global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", log.UnixAddTimeBegin, log.UnixAddTimeEnd).Count(&total)
return weblogs, total, nil
}
func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) {
service/waf_service/waf_sharedb.go
+8
View File
@@ -32,3 +32,11 @@ func (receiver *WafShareDbService) GetListApi(req request.WafShareDbReq) ([]mode
return list, total, nil
}
// 获取所有db
func (receiver *WafShareDbService) GetAllShareDbApi() ([]model.ShareDb, error) {
var list []model.ShareDb
global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Find(&list)
return list, nil
}
wafenginecore/localdb.go → wafdb/localdb.go
+9 -6
View File
@@ -1,4 +1,4 @@
package wafenginecore
package wafdb
import (
"SamWaf/customtype"
@@ -123,7 +123,7 @@ func InitLogDb(currentDir string) {
DbLogicType: "log",
StartTime: customtype.JsonTime(time.Now()),
EndTime: customtype.JsonTime(time.Now()),
FileName: "local_log",
FileName: "local_log.db",
Cnt: logtotal,
}
global.GWAF_LOCAL_DB.Create(sharDbBean)
@@ -136,7 +136,8 @@ func InitManaulLogDb(currentDir string, custFileName string) {
if currentDir == "" {
currentDir = utils.GetCurrentDir()
}
if global.GWAF_LOCAL_CUSTOM_LOG_DB == nil {
if global.GDATA_CURRENT_LOG_DB_MAP[custFileName] == nil {
zlog.Debug("初始化自定义的库", custFileName)
path := currentDir + "/data/" + custFileName
key := url.QueryEscape(global.GWAF_PWD_LOGDB)
dns := fmt.Sprintf("%s?_db_key=%s", path, key)
@@ -146,7 +147,7 @@ func InitManaulLogDb(currentDir string, custFileName string) {
}
// 启用 WAL 模式
_ = db.Exec("PRAGMA journal_mode=WAL;")
global.GWAF_LOCAL_CUSTOM_LOG_DB = db
global.GDATA_CURRENT_LOG_DB_MAP[custFileName] = db
//logDB.Use(crypto.NewCryptoPlugin())
// 注册默认的AES加解密策略
//crypto.RegisterCryptoStrategy(strategy.NewAesCryptoStrategy("3Y)(27EtO^tK8Bj~"))
@@ -156,9 +157,11 @@ func InitManaulLogDb(currentDir string, custFileName string) {
db.AutoMigrate(&model.AccountLog{})
db.AutoMigrate(&model.WafSysLog{})
global.GWAF_LOCAL_CUSTOM_LOG_DB.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query)
global.GWAF_LOCAL_CUSTOM_LOG_DB.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update)
global.GDATA_CURRENT_LOG_DB_MAP[custFileName].Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query)
global.GDATA_CURRENT_LOG_DB_MAP[custFileName].Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update)
} else {
zlog.Debug("自定义的库已存在", custFileName)
}
}
wafsafeclear/wafsafeclear.go
+53
View File
@@ -0,0 +1,53 @@
package wafsafeclear
import (
"SamWaf/global"
"SamWaf/utils/zlog"
)
func SafeClear() {
sqlDB, err := global.GWAF_LOCAL_DB.DB()
if err != nil {
zlog.Error("清理异常", err)
} else {
// 关闭数据库连接
if err := sqlDB.Close(); err != nil {
zlog.Error("清理异常关闭错误", err)
}
}
sqlDB, err = global.GWAF_LOCAL_LOG_DB.DB()
if err != nil {
zlog.Error("清理log退出异常", err)
} else {
// 关闭数据库连接
if err := sqlDB.Close(); err != nil {
zlog.Error("清理log异常关闭错误", err)
}
}
sqlDB, err = global.GWAF_LOCAL_STATS_DB.DB()
if err != nil {
zlog.Error("清理stat退出异常", err)
} else {
// 关闭数据库连接
if err := sqlDB.Close(); err != nil {
zlog.Error("清理stat异常关闭错误", err)
}
}
for _, value := range global.GDATA_CURRENT_LOG_DB_MAP {
sqlDB, err = value.DB()
if err != nil {
zlog.Error("清理异常错误存档", err)
} else {
// 关闭数据库连接
if err := sqlDB.Close(); err != nil {
zlog.Error("清理异常错误存档关闭错误", err)
}
}
}
zlog.Info("退出清理完成")
}
waftask/localtaskcounter.go
+2 -2
View File
@@ -10,7 +10,7 @@ import (
"SamWaf/service/waf_service"
"SamWaf/utils"
"SamWaf/utils/zlog"
"SamWaf/wafenginecore"
"SamWaf/wafdb"
"SamWaf/wechat"
"encoding/json"
"fmt"
@@ -504,7 +504,7 @@ func TaskShareDbInfo() {
}
global.GWAF_LOCAL_DB.Create(sharDbBean)
global.GWAF_LOCAL_LOG_DB = nil
wafenginecore.InitLogDb("")
wafdb.InitLogDb("")
global.GDATA_CURRENT_CHANGE = false
zlog.Debug("切库完成...")
}
waftask/localtaskcounter_test.go
+4 -4
View File
@@ -3,7 +3,7 @@ package waftask
import (
"SamWaf/global"
"SamWaf/utils/zlog"
"SamWaf/wafenginecore"
"SamWaf/wafdb"
"github.com/spf13/viper"
"testing"
)
@@ -27,8 +27,8 @@ func TestTaskDeleteHistoryInfo(t *testing.T) {
global.GWAF_LOCAL_SERVER_PORT = config.GetInt("local_port") //读取本地端口
global.GWAF_CUSTOM_SERVER_NAME = config.GetString("custom_server_name") //本地服务器其定义名称
zlog.Debug(" load ini: ", global.GWAF_USER_CODE)
wafenginecore.InitCoreDb(currentPath)
wafenginecore.InitLogDb(currentPath)
wafenginecore.InitStatsDb(currentPath)
wafdb.InitCoreDb(currentPath)
wafdb.InitLogDb(currentPath)
wafdb.InitStatsDb(currentPath)
TaskDeleteHistoryInfo()
}