diff --git a/api/waf_log.go b/api/waf_log.go index 15a6829..e9322bb 100644 --- a/api/waf_log.go +++ b/api/waf_log.go @@ -4,6 +4,7 @@ import ( "SamWaf/global" "SamWaf/model/common/response" "SamWaf/model/request" + response2 "SamWaf/model/response" "github.com/gin-gonic/gin" ) @@ -29,9 +30,6 @@ func (w *WafLogAPi) GetListApi(c *gin.Context) { var req request.WafAttackLogSearch err := c.ShouldBindJSON(&req) if err == nil { - /*//TOOD 模拟意外退出 - - os.Exit(-1) //退出进程*/ if global.GDATA_CURRENT_CHANGE { //如果正在切换库 跳过 response.FailWithMessage("正在切换数据库请等待", c) @@ -68,3 +66,19 @@ func (w *WafLogAPi) GetListByHostCodeApi(c *gin.Context) { response.FailWithMessage("解析失败", c) } } + +func (w *WafLogAPi) GetAllShareDbApi(c *gin.Context) { + wafShareList, _ := wafShareDbService.GetAllShareDbApi() + allShareDbRep := make([]response2.AllShareDbRep, len(wafShareList)) // 创建数组 + for i, _ := range wafShareList { + + allShareDbRep[i] = response2.AllShareDbRep{ + StartTime: wafShareList[i].StartTime, + EndTime: wafShareList[i].EndTime, + FileName: wafShareList[i].FileName, + Cnt: wafShareList[i].Cnt, + } + + } + response.OkWithDetailed(allShareDbRep, "获取成功", c) +} diff --git a/global/global.go b/global/global.go index b6c9b02..c52a08a 100644 --- a/global/global.go +++ b/global/global.go @@ -39,7 +39,6 @@ var ( GWAF_LOCAL_DB *gorm.DB //通用本地数据库，尊重用户隐私 GWAF_LOCAL_LOG_DB *gorm.DB //通用本地数据库存日志数据，尊重用户隐私 - GWAF_LOCAL_CUSTOM_LOG_DB *gorm.DB //通用本地数据库存历史日志数据， 尊重用户隐私 GWAF_LOCAL_STATS_DB *gorm.DB //通用本地数据库存放统计数据，尊重用户隐私 GWAF_REMOTE_DB *gorm.DB //仅当用户使用云数据库 GWAF_LOCAL_SERVER_PORT int = 26666 // 本地local端口 @@ -75,10 +74,10 @@ var ( GQEQUE_MESSAGE_DB Dequelib.Deque //发送消息队列 /******数据库处理参数*****/ - GDATA_BATCH_INSERT int = 1000 //最大批量插入 - GDATA_SHARE_DB_SIZE int64 = 100 * 10000 //100w 进行分库 100*10000 - GDATA_CURRENT_CHANGE bool = false //当前是否正在切换 - + GDATA_BATCH_INSERT int = 1000 //最大批量插入 + GDATA_SHARE_DB_SIZE int64 = 100 * 10000 //100w 进行分库 100*10000 + GDATA_CURRENT_CHANGE bool = false //当前是否正在切换 + GDATA_CURRENT_LOG_DB_MAP map[string]*gorm.DB //当前自定义的数据库连接 TODO 如果用户打开了多个 会不会影响内存 /******WebSocket*********/ GWebSocket *model.WebSocketOnline diff --git a/localwaf/src/apis/waflog/attacklog.ts b/localwaf/src/apis/waflog/attacklog.ts index 5e65f1a..ecdb83a 100644 --- a/localwaf/src/apis/waflog/attacklog.ts +++ b/localwaf/src/apis/waflog/attacklog.ts @@ -7,3 +7,12 @@ export function attacklogList(params) { params: params }) } + +//查询存档日志库列表 +export function allsharedblist(params) { + return request({ + url: 'waflog/attack/allsharedb', + method: 'get', + params: params + }) +} diff --git a/localwaf/src/pages/waf/attack/detail/index.vue b/localwaf/src/pages/waf/attack/detail/index.vue index 88eee3d..3763f3f 100644 --- a/localwaf/src/pages/waf/attack/detail/index.vue +++ b/localwaf/src/pages/waf/attack/detail/index.vue @@ -93,11 +93,11 @@ - + - + @@ -156,19 +156,29 @@ watch: { '$route.query.req_uuid'(newVal, oldVal) { console.log('route.query.req_uuid changed', newVal, oldVal) - this.getDetail(newVal) + if(newVal!=undefined){ + this.getDetail(newVal) + } + }, }, methods: { backPage(){ 　history.go(-1) }, - getDetail(id) { + getDetail(uuid_and_name) { + if(uuid_and_name == undefined){ + return + } + let arr = uuid_and_name.split("#"); + let id = arr[0] + let current_db_name = arr[1] let that = this this.$request .get('/waflog/attack/detail', { params: { REQ_UUID: id, + current_db_name:current_db_name } }) .then((res) => { diff --git a/localwaf/src/pages/waf/attack/index.vue b/localwaf/src/pages/waf/attack/index.vue index 33dd2fb..227cc3b 100644 --- a/localwaf/src/pages/waf/attack/index.vue +++ b/localwaf/src/pages/waf/attack/index.vue @@ -55,6 +55,15 @@ placeholder="请输入访问方法" :style="{ width: '100px' }" /> + + + + + {{ item }} + + + + @@ -106,7 +115,7 @@ import { SearchIcon } from 'tdesign-icons-vue'; import Trend from '@/components/trend/index.vue'; import { prefix } from '@/config/global'; - import { attacklogList } from '@/apis/waflog/attacklog'; + import { allsharedblist } from '@/apis/waflog/attacklog'; import { NowDate, ConvertStringToUnix, ConvertDateToString, ConvertUnixToDate } from '@/utils/date'; import { @@ -319,9 +328,12 @@ method: "", unix_add_time_begin: "", unix_add_time_end: "", + current_db_name:"local_log.db", }, //主机字典 - host_dic: {} + host_dic: {}, + //日志存档字典 + share_db_dic: {} }; }, computed: { @@ -379,6 +391,7 @@ } this.loadHostList() + this.loadShareDbList() this.getList("") }, watch: { @@ -399,6 +412,22 @@ next(); // 继续后续的导航解析过程 }, methods: { + loadShareDbList() { + let that = this; + allsharedblist("").then((res) => { + let resdata = res + console.log("loadShareDbList",resdata) + if (resdata.code === 0) { + let share_options = resdata.data; + for (let i = 0; i < share_options.length; i++) { + that.share_db_dic[share_options[i].file_name] = share_options[i].file_name+"("+share_options[i].cnt+")" + } + } + }) + .catch((e : Error) => { + console.log(e); + }) + }, loadHostList() { let that = this; allhost().then((res) => { @@ -481,7 +510,7 @@ { path: '/waf/wafattacklogdetail', query: { - req_uuid: req_uuid, + req_uuid: req_uuid+"#"+this.searchformData.current_db_name, }, }, ); diff --git a/main.go b/main.go index 45faf48..5c15f33 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,9 @@ import ( "SamWaf/plugin" "SamWaf/utils" "SamWaf/utils/zlog" + "SamWaf/wafdb" "SamWaf/wafenginecore" + "SamWaf/wafsafeclear" "SamWaf/wafsnowflake" "SamWaf/waftask" "crypto/tls" @@ -20,6 +22,7 @@ import ( "github.com/kardianos/service" "go.uber.org/zap" "golang.org/x/time/rate" + "gorm.io/gorm" "log" "net/http" "os" @@ -43,6 +46,7 @@ func (m *wafSystenService) Start(s service.Service) error { // Stop 是服务停止时调用的方法 func (m *wafSystenService) Stop(s service.Service) error { + wafsafeclear.SafeClear() return nil } @@ -59,6 +63,29 @@ func NeverExit(name string, f func()) { // run 是服务的主要逻辑 func (m *wafSystenService) run() { + + /*// 启动一个 goroutine 来处理信号 + go func() { + // 创建一个通道来接收信号 + signalCh := make(chan os.Signal, 1) + + // 根据操作系统设置不同的信号监听 + if runtime.GOOS == "windows" { + signal.Notify(signalCh, syscall.SIGINT,os.Interrupt, syscall.SIGTERM,os.Kill) + } else { // Linux 系统 + signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM) + } + for { + select { + case sig := <-signalCh: + fmt.Println("接收到信号:", sig) + wafsafeclear.SafeClear() + // 在这里执行你的清理操作或退出应用程序的代码 + os.Exit(0) + } + } + }()*/ + // 在这里编写你的服务逻辑代码 fmt.Println("Service is running...") //初始化cache @@ -67,7 +94,8 @@ func (m *wafSystenService) run() { global.GWAF_MEASURE_PROCESS_DEQUEENGINE = cache.InitWafOnlyLockWrite() // 创建 Snowflake 实例 global.GWAF_SNOWFLAKE_GEN = wafsnowflake.NewSnowflake(1609459200000, 1, 1) // 设置epoch时间、机器ID和数据中心ID - + //提前初始化 + global.GDATA_CURRENT_LOG_DB_MAP = map[string]*gorm.DB{} rversion := "初始化系统 版本号：" + global.GWAF_RELEASE_VERSION_NAME + "(" + global.GWAF_RELEASE_VERSION + ")" if global.GWAF_RELEASE == "false" { rversion = rversion + " 调试版本" @@ -100,9 +128,9 @@ func (m *wafSystenService) run() { } //初始化本地数据库 - wafenginecore.InitCoreDb("") - wafenginecore.InitLogDb("") - wafenginecore.InitStatsDb("") + wafdb.InitCoreDb("") + wafdb.InitLogDb("") + wafdb.InitStatsDb("") //初始化队列引擎 wafenginecore.InitDequeEngine() //启动队列消费 @@ -198,6 +226,7 @@ func (m *wafSystenService) run() { //脱敏处理初始化 global.GWAF_DLP, _ = dlp.NewEngine("wafDlp") global.GWAF_DLP.ApplyConfigDefault() + for { select { case msg := <-global.GWAF_CHAN_MSG: @@ -271,7 +300,6 @@ func (m *wafSystenService) run() { } break case host := <-global.GWAF_CHAN_HOST: - wafEngine.HostTarget[host.Host+":"+strconv.Itoa(host.Port)].Host.GUARD_STATUS = host.GUARD_STATUS zlog.Debug("规则", zap.Any("主机", host)) break @@ -347,6 +375,11 @@ func main() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, os.Kill, syscall.SIGTERM) + /*_, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop()*/ + + //doWork(ctx) + // 以服务方式运行 if len(os.Args) > 1 { command := os.Args[1] @@ -364,7 +397,7 @@ func main() { zlog.Info("main server run false") global.GWAF_RUNTIME_SERVER_TYPE = service.Interactive() } - + //defer wafsafeclear.SafeClear() // 以常规方式运行 err = s.Run() if err != nil { diff --git a/model/request/waf_attack_log_detail.go b/model/request/waf_attack_log_detail.go index 3eca419..8964607 100644 --- a/model/request/waf_attack_log_detail.go +++ b/model/request/waf_attack_log_detail.go @@ -1,5 +1,6 @@ package request type WafAttackLogDetailReq struct { - REQ_UUID string `json:"req_uuid"` + CurrrentDbName string `json:"current_db_name"` + REQ_UUID string `json:"req_uuid"` } diff --git a/model/request/waf_attack_log_search.go b/model/request/waf_attack_log_search.go index 08c15c2..f80a57d 100644 --- a/model/request/waf_attack_log_search.go +++ b/model/request/waf_attack_log_search.go @@ -3,6 +3,7 @@ package request import "SamWaf/model/common/request" type WafAttackLogSearch struct { + CurrrentDbName string `json:"current_db_name"` HostCode string `json:"host_code" form:"host_code"` //主机码 Rule string `json:"rule" form:"rule"` //规则名 Action string `json:"action" form:"action"` //状态 diff --git a/model/response/all_host.go b/model/response/all_host.go index 2747539..f8f1647 100644 --- a/model/response/all_host.go +++ b/model/response/all_host.go @@ -1,6 +1,15 @@ package response +import "SamWaf/customtype" + type AllHostRep struct { Code string `json:"value"` //唯一码 Host string `json:"label"` //域名 } + +type AllShareDbRep struct { + StartTime customtype.JsonTime `json:"start_time"` //开始时间 + EndTime customtype.JsonTime `json:"end_time"` //结束时间 + FileName string `json:"file_name"` //文件名 + Cnt int64 `json:"cnt"` //当前数量 +} diff --git a/router/waf_log.go b/router/waf_log.go index ccf4a73..45a31a8 100644 --- a/router/waf_log.go +++ b/router/waf_log.go @@ -13,4 +13,6 @@ func (receiver *LogRouter) InitLogRouter(group *gin.RouterGroup) { wafLogRouter := group.Group("") wafLogRouter.POST("/samwaf/waflog/attack/list", logApi.GetListApi) wafLogRouter.GET("/samwaf/waflog/attack/detail", logApi.GetDetailApi) + wafLogRouter.GET("/samwaf/waflog/attack/allsharedb", logApi.GetAllShareDbApi) + } diff --git a/service/waf_service/waf_log.go b/service/waf_service/waf_log.go index c380b57..b1c00b0 100644 --- a/service/waf_service/waf_log.go +++ b/service/waf_service/waf_log.go @@ -3,8 +3,8 @@ package waf_service import ( "SamWaf/global" "SamWaf/innerbean" - request2 "SamWaf/model/common/request" "SamWaf/model/request" + "SamWaf/wafdb" ) type WafLogService struct{} @@ -18,31 +18,39 @@ func (receiver *WafLogService) AddApi(log innerbean.WebLog) error { func (receiver *WafLogService) ModifyApi(log innerbean.WebLog) error { return nil } -func (receiver *WafLogService) GetDetailApi(wafAttackDetailReq request.WafAttackLogDetailReq) (innerbean.WebLog, error) { +func (receiver *WafLogService) GetDetailApi(req request.WafAttackLogDetailReq) (innerbean.WebLog, error) { var weblog innerbean.WebLog - global.GWAF_LOCAL_LOG_DB.Where("REQ_UUID=?", wafAttackDetailReq.REQ_UUID).Find(&weblog) + if len(req.CurrrentDbName) == 0 || req.CurrrentDbName == "local_log.db" { + global.GWAF_LOCAL_LOG_DB.Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog) + } else { + wafdb.InitManaulLogDb("", req.CurrrentDbName) + global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Where("REQ_UUID=?", req.REQ_UUID).Find(&weblog) + } return weblog, nil } -func (receiver *WafLogService) GetListApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { +func (receiver *WafLogService) GetListApi(req request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { var total int64 = 0 var weblogs []innerbean.WebLog whereCondition := &request.WafAttackLogSearch{ - HostCode: log.HostCode, - Rule: log.Rule, - Action: log.Action, - SrcIp: log.SrcIp, - StatusCode: log.StatusCode, - Method: log.Method, - PageInfo: request2.PageInfo{ - PageIndex: 0, - PageSize: 0, - Keyword: "", - }, + HostCode: req.HostCode, + Rule: req.Rule, + Action: req.Action, + SrcIp: req.SrcIp, + StatusCode: req.StatusCode, + Method: req.Method, + } + if len(req.CurrrentDbName) == 0 || req.CurrrentDbName == "local_log.db" { + global.GWAF_LOCAL_LOG_DB.Limit(req.PageSize).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Offset(req.PageSize * (req.PageIndex - 1)).Order("create_time desc").Find(&weblogs) + global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Count(&total) + + } else { + wafdb.InitManaulLogDb("", req.CurrrentDbName) + global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Debug().Limit(req.PageSize).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Offset(req.PageSize * (req.PageIndex - 1)).Order("create_time desc").Find(&weblogs) + global.GDATA_CURRENT_LOG_DB_MAP[req.CurrrentDbName].Model(&innerbean.WebLog{}).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", req.UnixAddTimeBegin, req.UnixAddTimeEnd).Count(&total) + } - global.GWAF_LOCAL_LOG_DB.Limit(log.PageSize).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", log.UnixAddTimeBegin, log.UnixAddTimeEnd).Offset(log.PageSize * (log.PageIndex - 1)).Order("create_time desc").Find(&weblogs) - global.GWAF_LOCAL_LOG_DB.Model(&innerbean.WebLog{}).Where(whereCondition).Where("unix_add_time>=? and unix_add_time<=?", log.UnixAddTimeBegin, log.UnixAddTimeEnd).Count(&total) return weblogs, total, nil } func (receiver *WafLogService) GetListByHostCodeApi(log request.WafAttackLogSearch) ([]innerbean.WebLog, int64, error) { diff --git a/service/waf_service/waf_sharedb.go b/service/waf_service/waf_sharedb.go index ee5e5b3..18f00b2 100644 --- a/service/waf_service/waf_sharedb.go +++ b/service/waf_service/waf_sharedb.go @@ -32,3 +32,11 @@ func (receiver *WafShareDbService) GetListApi(req request.WafShareDbReq) ([]mode return list, total, nil } + +// 获取所有db +func (receiver *WafShareDbService) GetAllShareDbApi() ([]model.ShareDb, error) { + var list []model.ShareDb + global.GWAF_LOCAL_DB.Model(&model.ShareDb{}).Find(&list) + + return list, nil +} diff --git a/wafenginecore/localdb.go b/wafdb/localdb.go similarity index 91% rename from wafenginecore/localdb.go rename to wafdb/localdb.go index 2a43f00..f8f89f8 100644 --- a/wafenginecore/localdb.go +++ b/wafdb/localdb.go @@ -1,4 +1,4 @@ -package wafenginecore +package wafdb import ( "SamWaf/customtype" @@ -123,7 +123,7 @@ func InitLogDb(currentDir string) { DbLogicType: "log", StartTime: customtype.JsonTime(time.Now()), EndTime: customtype.JsonTime(time.Now()), - FileName: "local_log", + FileName: "local_log.db", Cnt: logtotal, } global.GWAF_LOCAL_DB.Create(sharDbBean) @@ -136,7 +136,8 @@ func InitManaulLogDb(currentDir string, custFileName string) { if currentDir == "" { currentDir = utils.GetCurrentDir() } - if global.GWAF_LOCAL_CUSTOM_LOG_DB == nil { + if global.GDATA_CURRENT_LOG_DB_MAP[custFileName] == nil { + zlog.Debug("初始化自定义的库", custFileName) path := currentDir + "/data/" + custFileName key := url.QueryEscape(global.GWAF_PWD_LOGDB) dns := fmt.Sprintf("%s?_db_key=%s", path, key) @@ -146,7 +147,7 @@ func InitManaulLogDb(currentDir string, custFileName string) { } // 启用 WAL 模式 _ = db.Exec("PRAGMA journal_mode=WAL;") - global.GWAF_LOCAL_CUSTOM_LOG_DB = db + global.GDATA_CURRENT_LOG_DB_MAP[custFileName] = db //logDB.Use(crypto.NewCryptoPlugin()) // 注册默认的AES加解密策略 //crypto.RegisterCryptoStrategy(strategy.NewAesCryptoStrategy("3Y)(27EtO^tK8Bj~")) @@ -156,9 +157,11 @@ func InitManaulLogDb(currentDir string, custFileName string) { db.AutoMigrate(&model.AccountLog{}) db.AutoMigrate(&model.WafSysLog{}) - global.GWAF_LOCAL_CUSTOM_LOG_DB.Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query) - global.GWAF_LOCAL_CUSTOM_LOG_DB.Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) + global.GDATA_CURRENT_LOG_DB_MAP[custFileName].Callback().Query().Before("gorm:query").Register("tenant_plugin:before_query", before_query) + global.GDATA_CURRENT_LOG_DB_MAP[custFileName].Callback().Query().Before("gorm:update").Register("tenant_plugin:before_update", before_update) + } else { + zlog.Debug("自定义的库已存在", custFileName) } } diff --git a/wafsafeclear/wafsafeclear.go b/wafsafeclear/wafsafeclear.go new file mode 100644 index 0000000..cf57193 --- /dev/null +++ b/wafsafeclear/wafsafeclear.go @@ -0,0 +1,53 @@ +package wafsafeclear + +import ( + "SamWaf/global" + "SamWaf/utils/zlog" +) + +func SafeClear() { + + sqlDB, err := global.GWAF_LOCAL_DB.DB() + if err != nil { + zlog.Error("清理异常", err) + } else { + + // 关闭数据库连接 + if err := sqlDB.Close(); err != nil { + zlog.Error("清理异常关闭错误", err) + } + } + sqlDB, err = global.GWAF_LOCAL_LOG_DB.DB() + if err != nil { + zlog.Error("清理log退出异常", err) + } else { + + // 关闭数据库连接 + if err := sqlDB.Close(); err != nil { + zlog.Error("清理log异常关闭错误", err) + } + } + sqlDB, err = global.GWAF_LOCAL_STATS_DB.DB() + if err != nil { + zlog.Error("清理stat退出异常", err) + } else { + + // 关闭数据库连接 + if err := sqlDB.Close(); err != nil { + zlog.Error("清理stat异常关闭错误", err) + } + } + for _, value := range global.GDATA_CURRENT_LOG_DB_MAP { + sqlDB, err = value.DB() + if err != nil { + zlog.Error("清理异常错误存档", err) + } else { + + // 关闭数据库连接 + if err := sqlDB.Close(); err != nil { + zlog.Error("清理异常错误存档关闭错误", err) + } + } + } + zlog.Info("退出清理完成") +} diff --git a/waftask/localtaskcounter.go b/waftask/localtaskcounter.go index c2ccbbd..08e17e7 100644 --- a/waftask/localtaskcounter.go +++ b/waftask/localtaskcounter.go @@ -10,7 +10,7 @@ import ( "SamWaf/service/waf_service" "SamWaf/utils" "SamWaf/utils/zlog" - "SamWaf/wafenginecore" + "SamWaf/wafdb" "SamWaf/wechat" "encoding/json" "fmt" @@ -504,7 +504,7 @@ func TaskShareDbInfo() { } global.GWAF_LOCAL_DB.Create(sharDbBean) global.GWAF_LOCAL_LOG_DB = nil - wafenginecore.InitLogDb("") + wafdb.InitLogDb("") global.GDATA_CURRENT_CHANGE = false zlog.Debug("切库完成...") } diff --git a/waftask/localtaskcounter_test.go b/waftask/localtaskcounter_test.go index a4ca0b0..0e8fcb3 100644 --- a/waftask/localtaskcounter_test.go +++ b/waftask/localtaskcounter_test.go @@ -3,7 +3,7 @@ package waftask import ( "SamWaf/global" "SamWaf/utils/zlog" - "SamWaf/wafenginecore" + "SamWaf/wafdb" "github.com/spf13/viper" "testing" ) @@ -27,8 +27,8 @@ func TestTaskDeleteHistoryInfo(t *testing.T) { global.GWAF_LOCAL_SERVER_PORT = config.GetInt("local_port") //读取本地端口 global.GWAF_CUSTOM_SERVER_NAME = config.GetString("custom_server_name") //本地服务器其定义名称 zlog.Debug(" load ini: ", global.GWAF_USER_CODE) - wafenginecore.InitCoreDb(currentPath) - wafenginecore.InitLogDb(currentPath) - wafenginecore.InitStatsDb(currentPath) + wafdb.InitCoreDb(currentPath) + wafdb.InitLogDb(currentPath) + wafdb.InitStatsDb(currentPath) TaskDeleteHistoryInfo() }