opensource/sealos
3
0
Fork 0
mirror of https://github.com/labring/sealos.git synced 2026-09-21 05:44:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: async clean log (#5434)

Browse Source 
* chore: unify the start index to page

* feat: async clean log

* feat: to limit offset utils
This commit is contained in:
zijiren
2025-03-03 14:42:20 +08:00
committed by GitHub
parent a19203b37d
commit 8f45b6f9fb
15 changed files with 126 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/aiproxy/common/config/config.go
+11 -1
View File
@@ -23,10 +23,11 @@ var (
var (
disableServe atomic.Bool
logStorageHours int64 = 0 // default 0 means no limit
saveAllLogDetail atomic.Bool
logDetailRequestBodyMaxSize int64 = 128 * 1024 // 128KB
logDetailResponseBodyMaxSize int64 = 128 * 1024 // 128KB
logDetailStorageHours int64 = 3 * 24
logDetailStorageHours int64 = 3 * 24 // 3 days
internalToken atomic.Value
)
@@ -99,6 +100,15 @@ func SetTimeoutWithModelType(timeout map[int]int64) {
timeoutWithModelType.Store(timeout)
}
func GetLogStorageHours() int64 {
return atomic.LoadInt64(&logStorageHours)
}
func SetLogStorageHours(hours int64) {
hours = env.Int64("LOG_STORAGE_HOURS", hours)
atomic.StoreInt64(&logStorageHours, hours)
}
func GetLogDetailStorageHours() int64 {
return atomic.LoadInt64(&logDetailStorageHours)
}
service/aiproxy/controller/channel.go
+2 -2
View File
@@ -33,7 +33,7 @@ func GetChannels(c *gin.Context) {
channelType, _ := strconv.Atoi(c.Query("channel_type"))
baseURL := c.Query("base_url")
order := c.Query("order")
channels, total, err := model.GetChannels(page*perPage, perPage, id, name, key, channelType, baseURL, order)
channels, total, err := model.GetChannels(page, perPage, id, name, key, channelType, baseURL, order)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
@@ -86,7 +86,7 @@ func SearchChannels(c *gin.Context) {
channelType, _ := strconv.Atoi(c.Query("channel_type"))
baseURL := c.Query("base_url")
order := c.Query("order")
channels, total, err := model.SearchChannels(keyword, page*perPage, perPage, id, name, key, channelType, baseURL, order)
channels, total, err := model.SearchChannels(keyword, page, perPage, id, name, key, channelType, baseURL, order)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
service/aiproxy/controller/group.go
+2 -2
View File
@@ -32,7 +32,7 @@ func (g *GroupResponse) MarshalJSON() ([]byte, error) {
func GetGroups(c *gin.Context) {
page, perPage := parsePageParams(c)
order := c.DefaultQuery("order", "")
groups, total, err := model.GetGroups(page*perPage, perPage, order, false)
groups, total, err := model.GetGroups(page, perPage, order, false)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
@@ -56,7 +56,7 @@ func SearchGroups(c *gin.Context) {
page, perPage := parsePageParams(c)
order := c.DefaultQuery("order", "")
status, _ := strconv.Atoi(c.Query("status"))
groups, total, err := model.SearchGroup(keyword, page*perPage, perPage, order, status)
groups, total, err := model.SearchGroup(keyword, page, perPage, order, status)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
service/aiproxy/controller/modelconfig.go
+2 -2
View File
@@ -11,7 +11,7 @@ import (
func GetModelConfigs(c *gin.Context) {
page, perPage := parsePageParams(c)
_model := c.Query("model")
configs, total, err := model.GetModelConfigs(page*perPage, perPage, _model)
configs, total, err := model.GetModelConfigs(page, perPage, _model)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
@@ -55,7 +55,7 @@ func SearchModelConfigs(c *gin.Context) {
page, perPage := parsePageParams(c)
_model := c.Query("model")
owner := c.Query("owner")
configs, total, err := model.SearchModelConfigs(keyword, page*perPage, perPage, _model, model.ModelOwner(owner))
configs, total, err := model.SearchModelConfigs(keyword, page, perPage, _model, model.ModelOwner(owner))
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
service/aiproxy/controller/token.go
+4 -4
View File
@@ -111,7 +111,7 @@ func GetTokens(c *gin.Context) {
order := c.Query("order")
status, _ := strconv.Atoi(c.Query("status"))
tokens, total, err := model.GetTokens(group, page*perPage, perPage, order, status)
tokens, total, err := model.GetTokens(group, page, perPage, order, status)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
@@ -134,7 +134,7 @@ func GetGroupTokens(c *gin.Context) {
order := c.Query("order")
status, _ := strconv.Atoi(c.Query("status"))
tokens, total, err := model.GetTokens(group, page*perPage, perPage, order, status)
tokens, total, err := model.GetTokens(group, page, perPage, order, status)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
@@ -155,7 +155,7 @@ func SearchTokens(c *gin.Context) {
status, _ := strconv.Atoi(c.Query("status"))
group := c.Query("group")
tokens, total, err := model.SearchTokens(group, keyword, page*perPage, perPage, order, status, name, key)
tokens, total, err := model.SearchTokens(group, keyword, page, perPage, order, status, name, key)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
@@ -181,7 +181,7 @@ func SearchGroupTokens(c *gin.Context) {
key := c.Query("key")
status, _ := strconv.Atoi(c.Query("status"))
tokens, total, err := model.SearchTokens(group, keyword, page*perPage, perPage, order, status, name, key)
tokens, total, err := model.SearchTokens(group, keyword, page, perPage, order, status, name, key)
if err != nil {
middleware.ErrorResponse(c, http.StatusOK, err.Error())
return
service/aiproxy/controller/utils.go
-10
View File
@@ -8,16 +8,6 @@ import (
func parsePageParams(c *gin.Context) (page, perPage int) {
page, _ = strconv.Atoi(c.Query("p"))
page--
if page < 0 {
page = 0
}
perPage, _ = strconv.Atoi(c.Query("per_page"))
if perPage <= 0 {
perPage = 10
} else if perPage > 100 {
perPage = 100
}
return
}
service/aiproxy/main.go
+31 -5
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"errors"
"fmt"
stdlog "log"
"net/http"
@@ -134,13 +135,36 @@ func setupHTTPServer() (*http.Server, *gin.Engine) {
}, server
}
func autoTestBannedModels() {
func autoTestBannedModels(ctx context.Context) {
log.Info("auto test banned models start")
ticker := time.NewTicker(time.Second * 15)
defer ticker.Stop()
for range ticker.C {
controller.AutoTestBannedModels()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
controller.AutoTestBannedModels()
}
}
}
func cleanLog(ctx context.Context) {
log.Info("clean log start")
ticker := time.NewTicker(time.Minute * 15)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
err := model.CleanLog()
if err != nil {
log.Errorf("clean log failed: %s", err)
}
}
}
}
@@ -165,12 +189,14 @@ func main() {
go func() {
log.Infof("server started on http://localhost:%s", srv.Addr[1:])
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
if err := srv.ListenAndServe(); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
log.Fatal("failed to start HTTP server: " + err.Error())
}
}()
go autoTestBannedModels()
go autoTestBannedModels(ctx)
go cleanLog(ctx)
<-ctx.Done()
service/aiproxy/model/channel.go
+6 -4
View File
@@ -152,7 +152,7 @@ func GetAllChannels() (channels []*Channel, err error) {
return channels, err
}
func GetChannels(startIdx int, num int, id int, name string, key string, channelType int, baseURL string, order string) (channels []*Channel, total int64, err error) {
func GetChannels(page int, perPage int, id int, name string, key string, channelType int, baseURL string, order string) (channels []*Channel, total int64, err error) {
tx := DB.Model(&Channel{})
if id != 0 {
tx = tx.Where("id = ?", id)
@@ -176,11 +176,12 @@ func GetChannels(startIdx int, num int, id int, name string, key string, channel
if total <= 0 {
return nil, 0, nil
}
err = tx.Order(getChannelOrder(order)).Limit(num).Offset(startIdx).Find(&channels).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getChannelOrder(order)).Limit(limit).Offset(offset).Find(&channels).Error
return channels, total, err
}
func SearchChannels(keyword string, startIdx int, num int, id int, name string, key string, channelType int, baseURL string, order string) (channels []*Channel, total int64, err error) {
func SearchChannels(keyword string, page int, perPage int, id int, name string, key string, channelType int, baseURL string, order string) (channels []*Channel, total int64, err error) {
tx := DB.Model(&Channel{})
// Handle exact match conditions for non-zero values
@@ -257,7 +258,8 @@ func SearchChannels(keyword string, startIdx int, num int, id int, name string,
if total <= 0 {
return nil, 0, nil
}
err = tx.Order(getChannelOrder(order)).Limit(num).Offset(startIdx).Find(&channels).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getChannelOrder(order)).Limit(limit).Offset(offset).Find(&channels).Error
return channels, total, err
}
service/aiproxy/model/consumeerr.go
+2 -6
View File
@@ -138,12 +138,8 @@ func SearchConsumeError(keyword string, requestID string, group string, tokenNam
return nil, 0, nil
}
page--
if page < 0 {
page = 0
}
var errors []*ConsumeError
err = tx.Order(getLogOrder(order)).Limit(perPage).Offset(page * perPage).Find(&errors).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getLogOrder(order)).Limit(limit).Offset(offset).Find(&errors).Error
return errors, total, err
}
service/aiproxy/model/group.go
+6 -5
View File
@@ -55,7 +55,7 @@ func getGroupOrder(order string) string {
}
}
func GetGroups(startIdx int, num int, order string, onlyDisabled bool) (groups []*Group, total int64, err error) {
func GetGroups(page int, perPage int, order string, onlyDisabled bool) (groups []*Group, total int64, err error) {
tx := DB.Model(&Group{})
if onlyDisabled {
tx = tx.Where("status = ?", GroupStatusDisabled)
@@ -69,8 +69,8 @@ func GetGroups(startIdx int, num int, order string, onlyDisabled bool) (groups [
if total <= 0 {
return nil, 0, nil
}
err = tx.Order(getGroupOrder(order)).Limit(num).Offset(startIdx).Find(&groups).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getGroupOrder(order)).Limit(limit).Offset(offset).Find(&groups).Error
return groups, total, err
}
@@ -242,7 +242,7 @@ func UpdateGroupStatus(id string, status int) (err error) {
return HandleUpdateResult(result, ErrGroupNotFound)
}
func SearchGroup(keyword string, startIdx int, num int, order string, status int) (groups []*Group, total int64, err error) {
func SearchGroup(keyword string, page int, perPage int, order string, status int) (groups []*Group, total int64, err error) {
tx := DB.Model(&Group{})
if status != 0 {
tx = tx.Where("status = ?", status)
@@ -259,7 +259,8 @@ func SearchGroup(keyword string, startIdx int, num int, order string, status int
if total <= 0 {
return nil, 0, nil
}
err = tx.Order(getGroupOrder(order)).Limit(num).Offset(startIdx).Find(&groups).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getGroupOrder(order)).Limit(limit).Offset(offset).Find(&groups).Error
return groups, total, err
}
service/aiproxy/model/log.go
+28 -25
View File
@@ -12,7 +12,6 @@ import (
"github.com/labring/sealos/service/aiproxy/common"
"github.com/labring/sealos/service/aiproxy/common/config"
"github.com/shopspring/decimal"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"gorm.io/gorm"
)
@@ -204,7 +203,28 @@ func GetGroupLogDetail(logID int, group string) (*RequestDetail, error) {
return &detail, nil
}
func cleanRequestDetail() error {
func CleanLog() error {
err := cleanLog()
if err != nil {
return err
}
return cleanLogDetail()
}
func cleanLog() error {
logStorageHours := config.GetLogStorageHours()
if logStorageHours <= 0 {
return nil
}
return LogDB.
Where(
"created_at < ?",
time.Now().Add(-time.Duration(logStorageHours)*time.Hour),
).
Delete(&Log{}).Error
}
func cleanLogDetail() error {
detailStorageHours := config.GetLogDetailStorageHours()
if detailStorageHours <= 0 {
return nil
@@ -237,15 +257,6 @@ func RecordConsumeLog(
ip string,
requestDetail *RequestDetail,
) error {
defer func() {
if requestDetail == nil {
return
}
err := cleanRequestDetail()
if err != nil {
log.Errorf("delete request detail failed: %s", err)
}
}()
log := &Log{
RequestID: requestID,
RequestAt: requestAt,
@@ -404,11 +415,6 @@ func getLogs(
})
g.Go(func() error {
page--
if page < 0 {
page = 0
}
query := buildGetLogsQuery(
group,
startTimestamp,
@@ -431,10 +437,11 @@ func getLogs(
})
}
limit, offset := toLimitOffset(page, perPage)
return query.
Order(getLogOrder(order)).
Limit(perPage).
Offset(page * perPage).
Limit(limit).
Offset(offset).
Find(&logs).Error
})
@@ -721,11 +728,6 @@ func searchLogs(
})
g.Go(func() error {
page--
if page < 0 {
page = 0
}
query := buildSearchLogsQuery(
group,
keyword,
@@ -750,10 +752,11 @@ func searchLogs(
})
}
limit, offset := toLimitOffset(page, perPage)
return query.
Order(getLogOrder(order)).
Limit(perPage).
Offset(page * perPage).
Limit(limit).
Offset(offset).
Find(&logs).Error
})
service/aiproxy/model/modelconfig.go
+6 -4
View File
@@ -78,7 +78,7 @@ func (c *ModelConfig) SupportFormats() ([]string, bool) {
return GetModelConfigStringSlice(c.Config, ModelConfigSupportFormatsKey)
}
func GetModelConfigs(startIdx int, num int, model string) (configs []*ModelConfig, total int64, err error) {
func GetModelConfigs(page int, perPage int, model string) (configs []*ModelConfig, total int64, err error) {
tx := DB.Model(&ModelConfig{})
if model != "" {
tx = tx.Where("model = ?", model)
@@ -90,7 +90,8 @@ func GetModelConfigs(startIdx int, num int, model string) (configs []*ModelConfi
if total <= 0 {
return nil, 0, nil
}
err = tx.Order("created_at desc").Limit(num).Offset(startIdx).Find(&configs).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order("created_at desc").Limit(limit).Offset(offset).Find(&configs).Error
return configs, total, err
}
@@ -112,7 +113,7 @@ func GetModelConfig(model string) (*ModelConfig, error) {
return config, HandleNotFound(err, ErrModelConfigNotFound)
}
func SearchModelConfigs(keyword string, startIdx int, num int, model string, owner ModelOwner) (configs []*ModelConfig, total int64, err error) {
func SearchModelConfigs(keyword string, page int, perPage int, model string, owner ModelOwner) (configs []*ModelConfig, total int64, err error) {
tx := DB.Model(&ModelConfig{}).Where("model LIKE ?", "%"+keyword+"%")
if model != "" {
tx = tx.Where("model = ?", model)
@@ -153,7 +154,8 @@ func SearchModelConfigs(keyword string, startIdx int, num int, model string, own
if total <= 0 {
return nil, 0, nil
}
err = tx.Order("created_at desc").Limit(num).Offset(startIdx).Find(&configs).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order("created_at desc").Limit(limit).Offset(offset).Find(&configs).Error
return configs, total, err
}
service/aiproxy/model/option.go
+7 -3
View File
@@ -56,6 +56,7 @@ func InitOption2DB() error {
}
func initOptionMap() error {
optionMap["LogStorageHours"] = strconv.FormatInt(config.GetLogStorageHours(), 10)
optionMap["LogDetailStorageHours"] = strconv.FormatInt(config.GetLogDetailStorageHours(), 10)
optionMap["SaveAllLogDetail"] = strconv.FormatBool(config.GetSaveAllLogDetail())
optionMap["LogDetailRequestBodyMaxSize"] = strconv.FormatInt(config.GetLogDetailRequestBodyMaxSize(), 10)
@@ -189,14 +190,17 @@ func updateOption(key string, value string, isInit bool) (err error) {
switch key {
case "InternalToken":
config.SetInternalToken(value)
case "LogStorageHours":
logStorageHours, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return err
}
config.SetLogStorageHours(logStorageHours)
case "LogDetailStorageHours":
logDetailStorageHours, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return err
}
if logDetailStorageHours < 0 {
return errors.New("log detail storage hours must be greater than 0")
}
config.SetLogDetailStorageHours(logDetailStorageHours)
case "SaveAllLogDetail":
config.SetSaveAllLogDetail(toBool(value))
service/aiproxy/model/token.go
+6 -4
View File
@@ -86,7 +86,7 @@ func InsertToken(token *Token, autoCreateGroup bool) error {
return nil
}
func GetTokens(group string, startIdx int, num int, order string, status int) (tokens []*Token, total int64, err error) {
func GetTokens(group string, page int, perPage int, order string, status int) (tokens []*Token, total int64, err error) {
tx := DB.Model(&Token{})
if group != "" {
tx = tx.Where("group_id = ?", group)
@@ -104,11 +104,12 @@ func GetTokens(group string, startIdx int, num int, order string, status int) (t
if total <= 0 {
return nil, 0, nil
}
err = tx.Order(getTokenOrder(order)).Limit(num).Offset(startIdx).Find(&tokens).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getTokenOrder(order)).Limit(limit).Offset(offset).Find(&tokens).Error
return tokens, total, err
}
func SearchTokens(group string, keyword string, startIdx int, num int, order string, status int, name string, key string) (tokens []*Token, total int64, err error) {
func SearchTokens(group string, keyword string, page int, perPage int, order string, status int, name string, key string) (tokens []*Token, total int64, err error) {
tx := DB.Model(&Token{})
if group != "" {
tx = tx.Where("group_id = ?", group)
@@ -163,7 +164,8 @@ func SearchTokens(group string, keyword string, startIdx int, num int, order str
if total <= 0 {
return nil, 0, nil
}
err = tx.Order(getTokenOrder(order)).Limit(num).Offset(startIdx).Find(&tokens).Error
limit, offset := toLimitOffset(page, perPage)
err = tx.Order(getTokenOrder(order)).Limit(limit).Offset(offset).Find(&tokens).Error
return tokens, total, err
}
service/aiproxy/model/utils.go
+13
View File
@@ -151,3 +151,16 @@ func String2Int(keyword string) int {
}
return i
}
func toLimitOffset(page int, perPage int) (limit int, offset int) {
page--
if page < 0 {
page = 0
}
if perPage <= 0 {
perPage = 10
} else if perPage > 100 {
perPage = 100
}
return perPage, page * perPage
}