diff --git a/.github/workflows/services.yml b/.github/workflows/services.yml index be250e34d..52719670e 100644 --- a/.github/workflows/services.yml +++ b/.github/workflows/services.yml @@ -88,7 +88,7 @@ jobs: strategy: matrix: ## TODO: add more modules - module: [ database, pay, account, minio, launchpad, exceptionmonitor, aiproxy, devbox, vlogs ] + module: [ database, pay, account, minio, launchpad, exceptionmonitor, devbox, vlogs ] steps: - name: Checkout uses: actions/checkout@v4 @@ -185,7 +185,7 @@ jobs: strategy: matrix: ## TODO: add more modules - module: [ database, pay, account, minio, launchpad, exceptionmonitor, aiproxy, devbox, vlogs ] + module: [ database, pay, account, minio, launchpad, exceptionmonitor, devbox, vlogs ] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/service/aiproxy/.gitignore b/service/aiproxy/.gitignore deleted file mode 100644 index 8df7a82da..000000000 --- a/service/aiproxy/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -aiproxy.db* -aiproxy \ No newline at end of file diff --git a/service/aiproxy/Dockerfile b/service/aiproxy/Dockerfile deleted file mode 100644 index 69b9899ab..000000000 --- a/service/aiproxy/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM alpine:latest - -ARG TARGETARCH -COPY bin/service-aiproxy-$TARGETARCH /aiproxy - -ENV PUID=0 PGID=0 UMASK=022 - -ENV FFPROBE_ENABLED=true - -EXPOSE 3000 - -RUN apk add --no-cache ca-certificates tzdata ffmpeg curl && \ - rm -rf /var/cache/apk/* - -ENTRYPOINT ["/aiproxy"] diff --git a/service/aiproxy/Makefile b/service/aiproxy/Makefile deleted file mode 100644 index fae59539d..000000000 --- a/service/aiproxy/Makefile +++ /dev/null @@ -1,53 +0,0 @@ -IMG ?= ghcr.io/labring/sealos-aiproxy-service:latest - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# only support linux, non cgo -PLATFORMS ?= linux_arm64 linux_amd64 -GOOS=linux -GOARCH=$(shell go env GOARCH) - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Build - -.PHONY: clean -clean: - rm -f $(SERVICE_NAME) - -.PHONY: build -build: ## Build service-hub binary. - LD_FLAGS="-s -w -extldflags '-static'"; \ - CGO_ENABLED=0 GOOS=linux go build -tags "jsoniter" -ldflags "$${LD_FLAGS}" -trimpath -o bin/manager main.go - -.PHONY: docker-build -docker-build: build - mv bin/manager bin/service-aiproxy-${TARGETARCH} - docker build -t $(IMG) . - -.PHONY: docker-push -docker-push: - docker push $(IMG) diff --git a/service/aiproxy/README.md b/service/aiproxy/README.md index 50c0e8729..2a8b03093 100644 --- a/service/aiproxy/README.md +++ b/service/aiproxy/README.md @@ -1,26 +1,3 @@ -# Use Sealos to Deploy +# AIProxy -```bash -sealos run ghcr.io/labring/sealos-cloud-aiproxy-service:latest \ - -e ADMIN_KEY= \ - -e cloudDomain= -``` - -# Use One PostgreSQL - -```bash -sealos run ghcr.io/labring/sealos-cloud-aiproxy-service:latest \ - -e ADMIN_KEY= \ - -e cloudDomain= \ - -e LOG_SQL_DSN="" -``` - -# Envs - -- `ADMIN_KEY`: The admin key for the AI Proxy Service, admin key is used to admin api and relay api, default is empty -- `SEALOS_JWT_KEY`: Used to sealos balance service, default is empty -- `SQL_DSN`: The database connection string, default is empty -- `LOG_SQL_DSN`: The log database connection string, default is empty -- `REDIS_CONN_STRING`: The redis connection string, default is empty -- `BALANCE_SEALOS_CHECK_REAL_NAME_ENABLE`: Whether to check real name, default is `false` -- `BALANCE_SEALOS_NO_REAL_NAME_USED_AMOUNT_LIMIT`: The amount of used balance when the user has no real name, default is `1` +**Note: This project has been moved to [labring/aiproxy](https://github.com/labring/aiproxy)** diff --git a/service/aiproxy/common/audio/audio.go b/service/aiproxy/common/audio/audio.go deleted file mode 100644 index d7b2d416d..000000000 --- a/service/aiproxy/common/audio/audio.go +++ /dev/null @@ -1,76 +0,0 @@ -package audio - -import ( - "errors" - "io" - "os/exec" - "strconv" - "strings" - - "github.com/labring/sealos/service/aiproxy/common/config" -) - -var ErrAudioDurationNAN = errors.New("audio duration is N/A") - -func GetAudioDuration(audio io.Reader) (float64, error) { - if !config.FfprobeEnabled { - return 0, nil - } - - ffprobeCmd := exec.Command( - "ffprobe", - "-v", "error", - "-select_streams", "a:0", - "-show_entries", "stream=duration", - "-of", "default=noprint_wrappers=1:nokey=1", - "-i", "-", - ) - ffprobeCmd.Stdin = audio - output, err := ffprobeCmd.Output() - if err != nil { - return 0, err - } - - str := strings.TrimSpace(string(output)) - - if str == "" || str == "N/A" { - return 0, ErrAudioDurationNAN - } - - duration, err := strconv.ParseFloat(str, 64) - if err != nil { - return 0, err - } - return duration, nil -} - -func GetAudioDurationFromFilePath(filePath string) (float64, error) { - if !config.FfprobeEnabled { - return 0, nil - } - - ffprobeCmd := exec.Command( - "ffprobe", - "-v", "error", - "-select_streams", "a:0", - "-show_entries", "format=duration", - "-of", "default=noprint_wrappers=1:nokey=1", - "-i", filePath, - ) - output, err := ffprobeCmd.Output() - if err != nil { - return 0, err - } - - str := strings.TrimSpace(string(output)) - - if str == "" || str == "N/A" { - return 0, ErrAudioDurationNAN - } - - duration, err := strconv.ParseFloat(str, 64) - if err != nil { - return 0, err - } - return duration, nil -} diff --git a/service/aiproxy/common/balance/balance.go b/service/aiproxy/common/balance/balance.go deleted file mode 100644 index 553f33b29..000000000 --- a/service/aiproxy/common/balance/balance.go +++ /dev/null @@ -1,28 +0,0 @@ -package balance - -import ( - "context" - - "github.com/labring/sealos/service/aiproxy/model" -) - -type GroupBalance interface { - GetGroupRemainBalance(ctx context.Context, group model.GroupCache) (float64, PostGroupConsumer, error) -} - -type PostGroupConsumer interface { - PostGroupConsume(ctx context.Context, tokenName string, usage float64) (float64, error) -} - -var ( - mock GroupBalance = NewMockGroupBalance() - Default = mock -) - -func MockGetGroupRemainBalance(ctx context.Context, group model.GroupCache) (float64, PostGroupConsumer, error) { - return mock.GetGroupRemainBalance(ctx, group) -} - -func GetGroupRemainBalance(ctx context.Context, group model.GroupCache) (float64, PostGroupConsumer, error) { - return Default.GetGroupRemainBalance(ctx, group) -} diff --git a/service/aiproxy/common/balance/mock.go b/service/aiproxy/common/balance/mock.go deleted file mode 100644 index fe94e2247..000000000 --- a/service/aiproxy/common/balance/mock.go +++ /dev/null @@ -1,27 +0,0 @@ -package balance - -import ( - "context" - - "github.com/labring/sealos/service/aiproxy/model" -) - -var _ GroupBalance = (*MockGroupBalance)(nil) - -const ( - mockBalance = 10000000 -) - -type MockGroupBalance struct{} - -func NewMockGroupBalance() *MockGroupBalance { - return &MockGroupBalance{} -} - -func (q *MockGroupBalance) GetGroupRemainBalance(_ context.Context, _ model.GroupCache) (float64, PostGroupConsumer, error) { - return mockBalance, q, nil -} - -func (q *MockGroupBalance) PostGroupConsume(_ context.Context, _ string, usage float64) (float64, error) { - return usage, nil -} diff --git a/service/aiproxy/common/balance/sealos.go b/service/aiproxy/common/balance/sealos.go deleted file mode 100644 index 6fab89015..000000000 --- a/service/aiproxy/common/balance/sealos.go +++ /dev/null @@ -1,384 +0,0 @@ -package balance - -import ( - "bytes" - "context" - "errors" - "fmt" - "math/rand/v2" - "net/http" - "time" - - "github.com/bytedance/sonic" - "github.com/golang-jwt/jwt/v5" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/env" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/redis/go-redis/v9" - "github.com/shopspring/decimal" - log "github.com/sirupsen/logrus" -) - -const ( - defaultAccountURL = "http://account-service.account-system.svc.cluster.local:2333" - balancePrecision = 1000000 - appType = "LLM-TOKEN" - sealosRequester = "sealos-admin" - sealosGroupBalanceKey = "sealos:balance:%s" - sealosUserRealNameKey = "sealos:realName:%s" - getBalanceRetry = 3 -) - -var ( - _ GroupBalance = (*Sealos)(nil) - sealosHTTPClient = &http.Client{} - decimalBalancePrecision = decimal.NewFromInt(balancePrecision) - minConsumeAmount = decimal.NewFromInt(1) - jwtToken string - sealosRedisCacheEnable = env.Bool("BALANCE_SEALOS_REDIS_CACHE_ENABLE", true) - sealosCacheExpire = 3 * time.Minute -) - -var ( - sealosCheckRealNameEnable = env.Bool("BALANCE_SEALOS_CHECK_REAL_NAME_ENABLE", false) - sealosNoRealNameUsedAmountLimit = env.Float64("BALANCE_SEALOS_NO_REAL_NAME_USED_AMOUNT_LIMIT", 1) -) - -type Sealos struct { - accountURL string -} - -// FIXME: 如果获取余额能成功，但是消费永远失败，需要加一个失败次数限制，如果失败次数超过一定阈值，暂停服务 -func InitSealos(jwtKey string, accountURL string) error { - token, err := newSealosToken(jwtKey) - if err != nil { - return fmt.Errorf("failed to generate sealos jwt token: %w", err) - } - jwtToken = token - Default = NewSealos(accountURL) - return nil -} - -func NewSealos(accountURL string) *Sealos { - if accountURL == "" { - accountURL = defaultAccountURL - } - return &Sealos{accountURL: accountURL} -} - -type sealosClaims struct { - Requester string `json:"requester"` - jwt.RegisteredClaims -} - -func newSealosToken(key string) (string, error) { - claims := &sealosClaims{ - Requester: sealosRequester, - RegisteredClaims: jwt.RegisteredClaims{ - NotBefore: jwt.NewNumericDate(time.Now()), - }, - } - return jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString(conv.StringToBytes(key)) -} - -type sealosGetGroupBalanceResp struct { - UserUID string `json:"userUID"` - Error string `json:"error"` - Balance int64 `json:"balance"` -} - -type sealosPostGroupConsumeReq struct { - Namespace string `json:"namespace"` - AppType string `json:"appType"` - AppName string `json:"appName"` - UserUID string `json:"userUID"` - Amount int64 `json:"amount"` -} - -type sealosPostGroupConsumeResp struct { - Error string `json:"error"` -} - -type sealosCache struct { - UserUID string `redis:"u"` - Balance int64 `redis:"b"` -} - -//nolint:gosec -func cacheSetGroupBalance(ctx context.Context, group string, balance int64, userUID string) error { - if !common.RedisEnabled || !sealosRedisCacheEnable { - return nil - } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - pipe := common.RDB.Pipeline() - pipe.HSet(ctx, fmt.Sprintf(sealosGroupBalanceKey, group), sealosCache{ - Balance: balance, - UserUID: userUID, - }) - expireTime := sealosCacheExpire + time.Duration(rand.Int64N(10)-5)*time.Second - pipe.Expire(ctx, fmt.Sprintf(sealosGroupBalanceKey, group), expireTime) - _, err := pipe.Exec(ctx) - return err -} - -func cacheGetGroupBalance(ctx context.Context, group string) (*sealosCache, error) { - if !common.RedisEnabled || !sealosRedisCacheEnable { - return nil, redis.Nil - } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - var cache sealosCache - if err := common.RDB.HGetAll(ctx, fmt.Sprintf(sealosGroupBalanceKey, group)).Scan(&cache); err != nil { - return nil, err - } - return &cache, nil -} - -var decreaseGroupBalanceScript = redis.NewScript(` - local balance = redis.call("HGet", KEYS[1], "balance") - if balance == false then - return redis.status_reply("ok") - end - redis.call("HSet", KEYS[1], "balance", balance - ARGV[1]) - return redis.status_reply("ok") -`) - -func cacheDecreaseGroupBalance(ctx context.Context, group string, amount int64) error { - if !common.RedisEnabled || !sealosRedisCacheEnable { - return nil - } - return decreaseGroupBalanceScript.Run(ctx, common.RDB, []string{fmt.Sprintf(sealosGroupBalanceKey, group)}, amount).Err() -} - -var ErrNoRealNameUsedAmountLimit = errors.New("达到未实名用户使用额度限制，请实名认证") - -func (s *Sealos) GetGroupRemainBalance(ctx context.Context, group model.GroupCache) (float64, PostGroupConsumer, error) { - var errs []error - for i := 0; ; i++ { - balance, userUID, err := s.getGroupRemainBalance(ctx, group.ID) - if err == nil { - if sealosCheckRealNameEnable && - group.UsedAmount > sealosNoRealNameUsedAmountLimit && - !s.checkRealName(ctx, userUID) { - return 0, nil, ErrNoRealNameUsedAmountLimit - } - return decimal.NewFromInt(balance).Div(decimalBalancePrecision).InexactFloat64(), - newSealosPostGroupConsumer(s.accountURL, group.ID, userUID), nil - } - errs = append(errs, err) - if i == getBalanceRetry-1 { - return 0, nil, errors.Join(errs...) - } - time.Sleep(time.Second) - } -} - -func cacheGetUserRealName(ctx context.Context, userUID string) (bool, error) { - if !common.RedisEnabled || !sealosRedisCacheEnable { - return true, redis.Nil - } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - realName, err := common.RDB.Get(ctx, fmt.Sprintf(sealosUserRealNameKey, userUID)).Bool() - if err != nil { - return false, err - } - return realName, nil -} - -func cacheSetUserRealName(ctx context.Context, userUID string, realName bool) error { - if !common.RedisEnabled || !sealosRedisCacheEnable { - return nil - } - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - var expireTime time.Duration - if realName { - expireTime = time.Hour * 12 - } else { - expireTime = time.Minute * 1 - } - return common.RDB.Set(ctx, fmt.Sprintf(sealosUserRealNameKey, userUID), realName, expireTime).Err() -} - -func (s *Sealos) checkRealName(ctx context.Context, userUID string) bool { - if cache, err := cacheGetUserRealName(ctx, userUID); err == nil { - return cache - } else if !errors.Is(err, redis.Nil) { - log.Errorf("get user (%s) real name cache failed: %s", userUID, err) - } - - realName, err := s.fetchRealNameFromAPI(ctx, userUID) - if err != nil { - log.Errorf("fetch user (%s) real name failed: %s", userUID, err) - return true - } - - if err := cacheSetUserRealName(ctx, userUID, realName); err != nil { - log.Errorf("set user (%s) real name cache failed: %s", userUID, err) - } - - return realName -} - -type sealosGetRealNameInfoResp struct { - IsRealName bool `json:"isRealName"` - Error string `json:"error"` -} - -func (s *Sealos) fetchRealNameFromAPI(ctx context.Context, userUID string) (bool, error) { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, - fmt.Sprintf("%s/admin/v1alpha1/real-name-info?userUID=%s", s.accountURL, userUID), nil) - if err != nil { - return false, err - } - - req.Header.Set("Authorization", "Bearer "+jwtToken) - resp, err := sealosHTTPClient.Do(req) - if err != nil { - return false, err - } - defer resp.Body.Close() - - var sealosResp sealosGetRealNameInfoResp - if err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&sealosResp); err != nil { - return false, err - } - - if resp.StatusCode != http.StatusOK || sealosResp.Error != "" { - return false, fmt.Errorf("get user (%s) real name failed with status code %d, error: %s", userUID, resp.StatusCode, sealosResp.Error) - } - - return sealosResp.IsRealName, nil -} - -// GroupBalance interface implementation -func (s *Sealos) getGroupRemainBalance(ctx context.Context, group string) (int64, string, error) { - if cache, err := cacheGetGroupBalance(ctx, group); err == nil && cache.UserUID != "" { - return cache.Balance, cache.UserUID, nil - } else if err != nil && !errors.Is(err, redis.Nil) { - log.Errorf("get group (%s) balance cache failed: %s", group, err) - } - - balance, userUID, err := s.fetchBalanceFromAPI(ctx, group) - if err != nil { - return 0, "", err - } - - if err := cacheSetGroupBalance(ctx, group, balance, userUID); err != nil { - log.Errorf("set group (%s) balance cache failed: %s", group, err) - } - - return balance, userUID, nil -} - -func (s *Sealos) fetchBalanceFromAPI(ctx context.Context, group string) (balance int64, userUID string, err error) { - ctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, - fmt.Sprintf("%s/admin/v1alpha1/account-with-workspace?namespace=%s", s.accountURL, group), nil) - if err != nil { - return 0, "", err - } - - req.Header.Set("Authorization", "Bearer "+jwtToken) - resp, err := sealosHTTPClient.Do(req) - if err != nil { - return 0, "", err - } - defer resp.Body.Close() - - var sealosResp sealosGetGroupBalanceResp - if err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&sealosResp); err != nil { - return 0, "", err - } - - if sealosResp.Error != "" { - return 0, "", errors.New(sealosResp.Error) - } - - if resp.StatusCode != http.StatusOK { - return 0, "", fmt.Errorf("get group (%s) balance failed with status code %d", group, resp.StatusCode) - } - - return sealosResp.Balance, sealosResp.UserUID, nil -} - -type SealosPostGroupConsumer struct { - accountURL string - group string - uid string -} - -func newSealosPostGroupConsumer(accountURL, group, uid string) *SealosPostGroupConsumer { - return &SealosPostGroupConsumer{ - accountURL: accountURL, - group: group, - uid: uid, - } -} - -func (s *SealosPostGroupConsumer) PostGroupConsume(ctx context.Context, tokenName string, usage float64) (float64, error) { - amount := s.calculateAmount(usage) - - if err := cacheDecreaseGroupBalance(ctx, s.group, amount.IntPart()); err != nil { - log.Errorf("decrease group (%s) balance cache failed: %s", s.group, err) - } - - if err := s.postConsume(ctx, amount.IntPart(), tokenName); err != nil { - return 0, err - } - - return amount.Div(decimalBalancePrecision).InexactFloat64(), nil -} - -func (s *SealosPostGroupConsumer) calculateAmount(usage float64) decimal.Decimal { - amount := decimal.NewFromFloat(usage).Mul(decimalBalancePrecision).Ceil() - if amount.LessThan(minConsumeAmount) { - amount = minConsumeAmount - } - return amount -} - -func (s *SealosPostGroupConsumer) postConsume(ctx context.Context, amount int64, tokenName string) error { - reqBody, err := sonic.Marshal(sealosPostGroupConsumeReq{ - Namespace: s.group, - Amount: amount, - AppType: appType, - AppName: tokenName, - UserUID: s.uid, - }) - if err != nil { - return err - } - - req, err := http.NewRequestWithContext(ctx, - http.MethodPost, - s.accountURL+"/admin/v1alpha1/charge-billing", - bytes.NewBuffer(reqBody)) - if err != nil { - return err - } - - req.Header.Set("Authorization", "Bearer "+jwtToken) - resp, err := sealosHTTPClient.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - var sealosResp sealosPostGroupConsumeResp - if err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&sealosResp); err != nil { - return err - } - - if resp.StatusCode != http.StatusOK || sealosResp.Error != "" { - return fmt.Errorf("status code: %d, error: %s", resp.StatusCode, sealosResp.Error) - } - - return nil -} diff --git a/service/aiproxy/common/color.go b/service/aiproxy/common/color.go deleted file mode 100644 index 476015b25..000000000 --- a/service/aiproxy/common/color.go +++ /dev/null @@ -1,20 +0,0 @@ -package common - -import ( - "os" - "sync" - - "github.com/mattn/go-isatty" -) - -var ( - needColor bool - needColorOnce sync.Once -) - -func NeedColor() bool { - needColorOnce.Do(func() { - needColor = isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) - }) - return needColor -} diff --git a/service/aiproxy/common/config/config.go b/service/aiproxy/common/config/config.go deleted file mode 100644 index 09b53d2f1..000000000 --- a/service/aiproxy/common/config/config.go +++ /dev/null @@ -1,244 +0,0 @@ -package config - -import ( - "math" - "os" - "slices" - "strconv" - "sync/atomic" - - "github.com/labring/sealos/service/aiproxy/common/env" -) - -var ( - DebugEnabled = env.Bool("DEBUG", false) - DebugSQLEnabled = env.Bool("DEBUG_SQL", false) -) - -var ( - DisableAutoMigrateDB = env.Bool("DISABLE_AUTO_MIGRATE_DB", false) - OnlyOneLogFile = env.Bool("ONLY_ONE_LOG_FILE", false) - AdminKey = os.Getenv("ADMIN_KEY") - FfprobeEnabled = env.Bool("FFPROBE_ENABLED", false) -) - -var ( - disableServe atomic.Bool - logStorageHours int64 = 0 // default 0 means no limit - saveAllLogDetail atomic.Bool - logDetailRequestBodyMaxSize int64 = 128 * 1024 // 128KB - logDetailResponseBodyMaxSize int64 = 128 * 1024 // 128KB - logDetailStorageHours int64 = 3 * 24 // 3 days - internalToken atomic.Value - notifyNote atomic.Value -) - -var ( - retryTimes atomic.Int64 - enableModelErrorAutoBan atomic.Bool - modelErrorAutoBanRate = math.Float64bits(0.3) - timeoutWithModelType atomic.Value - disableModelConfig = env.Bool("DISABLE_MODEL_CONFIG", false) -) - -var ( - defaultChannelModels atomic.Value - defaultChannelModelMapping atomic.Value - groupMaxTokenNum atomic.Int64 - groupConsumeLevelRatio atomic.Value -) - -var geminiSafetySetting atomic.Value - -var billingEnabled atomic.Bool - -func init() { - timeoutWithModelType.Store(make(map[int]int64)) - defaultChannelModels.Store(make(map[int][]string)) - defaultChannelModelMapping.Store(make(map[int]map[string]string)) - groupConsumeLevelRatio.Store(make(map[float64]float64)) - geminiSafetySetting.Store("BLOCK_NONE") - billingEnabled.Store(true) - internalToken.Store(os.Getenv("INTERNAL_TOKEN")) - notifyNote.Store(os.Getenv("NOTIFY_NOTE")) -} - -func GetDisableModelConfig() bool { - return disableModelConfig -} - -func GetRetryTimes() int64 { - return retryTimes.Load() -} - -func SetRetryTimes(times int64) { - times = env.Int64("RETRY_TIMES", times) - retryTimes.Store(times) -} - -func GetEnableModelErrorAutoBan() bool { - return enableModelErrorAutoBan.Load() -} - -func SetEnableModelErrorAutoBan(enabled bool) { - enabled = env.Bool("ENABLE_MODEL_ERROR_AUTO_BAN", enabled) - enableModelErrorAutoBan.Store(enabled) -} - -func GetModelErrorAutoBanRate() float64 { - return math.Float64frombits(atomic.LoadUint64(&modelErrorAutoBanRate)) -} - -func SetModelErrorAutoBanRate(rate float64) { - rate = env.Float64("MODEL_ERROR_AUTO_BAN_RATE", rate) - atomic.StoreUint64(&modelErrorAutoBanRate, math.Float64bits(rate)) -} - -func GetTimeoutWithModelType() map[int]int64 { - return timeoutWithModelType.Load().(map[int]int64) -} - -func SetTimeoutWithModelType(timeout map[int]int64) { - timeout = env.JSON("TIMEOUT_WITH_MODEL_TYPE", timeout) - timeoutWithModelType.Store(timeout) -} - -func GetLogStorageHours() int64 { - return atomic.LoadInt64(&logStorageHours) -} - -func SetLogStorageHours(hours int64) { - hours = env.Int64("LOG_STORAGE_HOURS", hours) - atomic.StoreInt64(&logStorageHours, hours) -} - -func GetLogDetailStorageHours() int64 { - return atomic.LoadInt64(&logDetailStorageHours) -} - -func SetLogDetailStorageHours(hours int64) { - hours = env.Int64("LOG_DETAIL_STORAGE_HOURS", hours) - atomic.StoreInt64(&logDetailStorageHours, hours) -} - -func GetSaveAllLogDetail() bool { - return saveAllLogDetail.Load() -} - -func SetSaveAllLogDetail(enabled bool) { - enabled = env.Bool("SAVE_ALL_LOG_DETAIL", enabled) - saveAllLogDetail.Store(enabled) -} - -func GetLogDetailRequestBodyMaxSize() int64 { - return atomic.LoadInt64(&logDetailRequestBodyMaxSize) -} - -func SetLogDetailRequestBodyMaxSize(size int64) { - size = env.Int64("LOG_DETAIL_REQUEST_BODY_MAX_SIZE", size) - atomic.StoreInt64(&logDetailRequestBodyMaxSize, size) -} - -func GetLogDetailResponseBodyMaxSize() int64 { - return atomic.LoadInt64(&logDetailResponseBodyMaxSize) -} - -func SetLogDetailResponseBodyMaxSize(size int64) { - size = env.Int64("LOG_DETAIL_RESPONSE_BODY_MAX_SIZE", size) - atomic.StoreInt64(&logDetailResponseBodyMaxSize, size) -} - -func GetDisableServe() bool { - return disableServe.Load() -} - -func SetDisableServe(disabled bool) { - disabled = env.Bool("DISABLE_SERVE", disabled) - disableServe.Store(disabled) -} - -func GetDefaultChannelModels() map[int][]string { - return defaultChannelModels.Load().(map[int][]string) -} - -func SetDefaultChannelModels(models map[int][]string) { - models = env.JSON("DEFAULT_CHANNEL_MODELS", models) - for key, ms := range models { - slices.Sort(ms) - models[key] = slices.Compact(ms) - } - defaultChannelModels.Store(models) -} - -func GetDefaultChannelModelMapping() map[int]map[string]string { - return defaultChannelModelMapping.Load().(map[int]map[string]string) -} - -func SetDefaultChannelModelMapping(mapping map[int]map[string]string) { - mapping = env.JSON("DEFAULT_CHANNEL_MODEL_MAPPING", mapping) - defaultChannelModelMapping.Store(mapping) -} - -func GetGroupConsumeLevelRatio() map[float64]float64 { - return groupConsumeLevelRatio.Load().(map[float64]float64) -} - -func GetGroupConsumeLevelRatioStringKeyMap() map[string]float64 { - ratio := GetGroupConsumeLevelRatio() - stringMap := make(map[string]float64) - for k, v := range ratio { - stringMap[strconv.FormatFloat(k, 'f', -1, 64)] = v - } - return stringMap -} - -func SetGroupConsumeLevelRatio(ratio map[float64]float64) { - ratio = env.JSON("GROUP_CONSUME_LEVEL_RATIO", ratio) - groupConsumeLevelRatio.Store(ratio) -} - -// GetGroupMaxTokenNum returns max number of tokens per group, 0 means unlimited -func GetGroupMaxTokenNum() int64 { - return groupMaxTokenNum.Load() -} - -func SetGroupMaxTokenNum(num int64) { - num = env.Int64("GROUP_MAX_TOKEN_NUM", num) - groupMaxTokenNum.Store(num) -} - -func GetGeminiSafetySetting() string { - return geminiSafetySetting.Load().(string) -} - -func SetGeminiSafetySetting(setting string) { - setting = env.String("GEMINI_SAFETY_SETTING", setting) - geminiSafetySetting.Store(setting) -} - -func GetBillingEnabled() bool { - return billingEnabled.Load() -} - -func SetBillingEnabled(enabled bool) { - enabled = env.Bool("BILLING_ENABLED", enabled) - billingEnabled.Store(enabled) -} - -func GetInternalToken() string { - return internalToken.Load().(string) -} - -func SetInternalToken(token string) { - token = env.String("INTERNAL_TOKEN", token) - internalToken.Store(token) -} - -func GetNotifyNote() string { - return notifyNote.Load().(string) -} - -func SetNotifyNote(note string) { - note = env.String("NOTIFY_NOTE", note) - notifyNote.Store(note) -} diff --git a/service/aiproxy/common/constants.go b/service/aiproxy/common/constants.go deleted file mode 100644 index 65d61413e..000000000 --- a/service/aiproxy/common/constants.go +++ /dev/null @@ -1,5 +0,0 @@ -package common - -import "time" - -var StartTime = time.Now().UnixMilli() // unit: millisecond diff --git a/service/aiproxy/common/consume/consume.go b/service/aiproxy/common/consume/consume.go deleted file mode 100644 index ba778a646..000000000 --- a/service/aiproxy/common/consume/consume.go +++ /dev/null @@ -1,199 +0,0 @@ -package consume - -import ( - "context" - "sync" - "time" - - "github.com/labring/sealos/service/aiproxy/common/balance" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/shopspring/decimal" - log "github.com/sirupsen/logrus" -) - -var consumeWaitGroup sync.WaitGroup - -func Wait() { - consumeWaitGroup.Wait() -} - -func AsyncConsume( - postGroupConsumer balance.PostGroupConsumer, - code int, - usage *relaymodel.Usage, - meta *meta.Meta, - inputPrice, - outputPrice float64, - content string, - ip string, - retryTimes int, - requestDetail *model.RequestDetail, -) { - if meta.IsChannelTest { - return - } - - consumeWaitGroup.Add(1) - defer func() { - consumeWaitGroup.Done() - if r := recover(); r != nil { - log.Errorf("panic in consume: %v", r) - } - }() - - go Consume( - context.Background(), - postGroupConsumer, - code, - usage, - meta, - inputPrice, - outputPrice, - content, - ip, - retryTimes, - requestDetail, - ) -} - -func Consume( - ctx context.Context, - postGroupConsumer balance.PostGroupConsumer, - code int, - usage *relaymodel.Usage, - meta *meta.Meta, - inputPrice, - outputPrice float64, - content string, - ip string, - retryTimes int, - requestDetail *model.RequestDetail, -) { - if meta.IsChannelTest { - return - } - - amount := CalculateAmount(usage, inputPrice, outputPrice) - - amount = consumeAmount(ctx, amount, postGroupConsumer, meta) - - err := recordConsume(meta, code, usage, inputPrice, outputPrice, content, ip, requestDetail, amount, retryTimes) - if err != nil { - log.Error("error batch record consume: " + err.Error()) - notify.ErrorThrottle("recordConsume", time.Minute, "record consume failed", err.Error()) - } -} - -func consumeAmount( - ctx context.Context, - amount float64, - postGroupConsumer balance.PostGroupConsumer, - meta *meta.Meta, -) float64 { - if amount > 0 { - return processGroupConsume(ctx, amount, postGroupConsumer, meta) - } - return 0 -} - -func CalculateAmount( - usage *relaymodel.Usage, - inputPrice, outputPrice float64, -) float64 { - if usage == nil { - return 0 - } - - promptTokens := usage.PromptTokens - completionTokens := usage.CompletionTokens - totalTokens := promptTokens + completionTokens - - if totalTokens == 0 { - return 0 - } - - promptAmount := decimal.NewFromInt(int64(promptTokens)). - Mul(decimal.NewFromFloat(inputPrice)). - Div(decimal.NewFromInt(model.PriceUnit)) - completionAmount := decimal.NewFromInt(int64(completionTokens)). - Mul(decimal.NewFromFloat(outputPrice)). - Div(decimal.NewFromInt(model.PriceUnit)) - - return promptAmount.Add(completionAmount).InexactFloat64() -} - -func processGroupConsume( - ctx context.Context, - amount float64, - postGroupConsumer balance.PostGroupConsumer, - meta *meta.Meta, -) float64 { - consumedAmount, err := postGroupConsumer.PostGroupConsume(ctx, meta.Token.Name, amount) - if err != nil { - log.Error("error consuming token remain amount: " + err.Error()) - if err := model.CreateConsumeError( - meta.RequestID, - meta.RequestAt, - meta.Group.ID, - meta.Token.Name, - meta.OriginModel, - err.Error(), - amount, - meta.Token.ID, - ); err != nil { - log.Error("failed to create consume error: " + err.Error()) - } - return amount - } - return consumedAmount -} - -func recordConsume( - meta *meta.Meta, - code int, - usage *relaymodel.Usage, - inputPrice, - outputPrice float64, - content string, - ip string, - requestDetail *model.RequestDetail, - amount float64, - retryTimes int, -) error { - promptTokens := 0 - completionTokens := 0 - if usage != nil { - promptTokens = usage.PromptTokens - completionTokens = usage.CompletionTokens - } - - var channelID int - if meta.Channel != nil { - channelID = meta.Channel.ID - } - - return model.BatchRecordConsume( - meta.RequestID, - meta.RequestAt, - meta.Group.ID, - code, - channelID, - promptTokens, - completionTokens, - meta.OriginModel, - meta.Token.ID, - meta.Token.Name, - amount, - inputPrice, - outputPrice, - meta.Endpoint, - content, - int(meta.Mode), - ip, - retryTimes, - requestDetail, - ) -} diff --git a/service/aiproxy/common/conv/any.go b/service/aiproxy/common/conv/any.go deleted file mode 100644 index d5e3bc037..000000000 --- a/service/aiproxy/common/conv/any.go +++ /dev/null @@ -1,18 +0,0 @@ -package conv - -import "unsafe" - -func AsString(v any) string { - str, _ := v.(string) - return str -} - -// The change of bytes will cause the change of string synchronously -func BytesToString(b []byte) string { - return unsafe.String(unsafe.SliceData(b), len(b)) -} - -// If string is readonly, modifying bytes will cause panic -func StringToBytes(s string) []byte { - return unsafe.Slice(unsafe.StringData(s), len(s)) -} diff --git a/service/aiproxy/common/ctxkey/key.go b/service/aiproxy/common/ctxkey/key.go deleted file mode 100644 index a1275b875..000000000 --- a/service/aiproxy/common/ctxkey/key.go +++ /dev/null @@ -1,11 +0,0 @@ -package ctxkey - -const ( - Group = "group" - Token = "token" - GroupBalance = "group_balance" - OriginalModel = "original_model" - RequestID = "X-Request-Id" - ModelCaches = "model_caches" - ModelConfig = "model_config" -) diff --git a/service/aiproxy/common/database.go b/service/aiproxy/common/database.go deleted file mode 100644 index b89c89568..000000000 --- a/service/aiproxy/common/database.go +++ /dev/null @@ -1,16 +0,0 @@ -package common - -import ( - "github.com/labring/sealos/service/aiproxy/common/env" -) - -var ( - UsingSQLite = false - UsingPostgreSQL = false - UsingMySQL = false -) - -var ( - SQLitePath = env.String("SQLITE_PATH", "aiproxy.db") - SQLiteBusyTimeout = env.Int64("SQLITE_BUSY_TIMEOUT", 3000) -) diff --git a/service/aiproxy/common/env/helper.go b/service/aiproxy/common/env/helper.go deleted file mode 100644 index dfb9f2b77..000000000 --- a/service/aiproxy/common/env/helper.go +++ /dev/null @@ -1,85 +0,0 @@ -package env - -import ( - "os" - "strconv" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common/conv" - log "github.com/sirupsen/logrus" -) - -func Bool(env string, defaultValue bool) bool { - if env == "" { - return defaultValue - } - e := os.Getenv(env) - if e == "" { - return defaultValue - } - p, err := strconv.ParseBool(e) - if err != nil { - log.Errorf("invalid %s: %s", env, e) - return defaultValue - } - return p -} - -func Int64(env string, defaultValue int64) int64 { - if env == "" { - return defaultValue - } - e := os.Getenv(env) - if e == "" { - return defaultValue - } - num, err := strconv.ParseInt(e, 10, 64) - if err != nil { - log.Errorf("invalid %s: %s", env, e) - return defaultValue - } - return num -} - -func Float64(env string, defaultValue float64) float64 { - if env == "" { - return defaultValue - } - e := os.Getenv(env) - if e == "" { - return defaultValue - } - num, err := strconv.ParseFloat(e, 64) - if err != nil { - log.Errorf("invalid %s: %s", env, e) - return defaultValue - } - return num -} - -func String(env string, defaultValue string) string { - if env == "" { - return defaultValue - } - e := os.Getenv(env) - if e == "" { - return defaultValue - } - return e -} - -func JSON[T any](env string, defaultValue T) T { - if env == "" { - return defaultValue - } - e := os.Getenv(env) - if e == "" { - return defaultValue - } - var t T - if err := sonic.Unmarshal(conv.StringToBytes(e), &t); err != nil { - log.Errorf("invalid %s: %s", env, e) - return defaultValue - } - return t -} diff --git a/service/aiproxy/common/fastJSONSerializer/fastJSONSerializer.go b/service/aiproxy/common/fastJSONSerializer/fastJSONSerializer.go deleted file mode 100644 index a0fdb2598..000000000 --- a/service/aiproxy/common/fastJSONSerializer/fastJSONSerializer.go +++ /dev/null @@ -1,47 +0,0 @@ -package fastjsonserializer - -import ( - "context" - "fmt" - "reflect" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common/conv" - "gorm.io/gorm/schema" -) - -type JSONSerializer struct{} - -func (*JSONSerializer) Scan(ctx context.Context, field *schema.Field, dst reflect.Value, dbValue any) (err error) { - fieldValue := reflect.New(field.FieldType) - - if dbValue != nil { - var bytes []byte - switch v := dbValue.(type) { - case []byte: - bytes = v - case string: - bytes = conv.StringToBytes(v) - default: - return fmt.Errorf("failed to unmarshal JSONB value: %#v", dbValue) - } - - if len(bytes) == 0 { - field.ReflectValueOf(ctx, dst).Set(reflect.Zero(field.FieldType)) - return nil - } - - err = sonic.Unmarshal(bytes, fieldValue.Interface()) - } - - field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem()) - return -} - -func (*JSONSerializer) Value(_ context.Context, _ *schema.Field, _ reflect.Value, fieldValue any) (any, error) { - return sonic.Marshal(fieldValue) -} - -func init() { - schema.RegisterSerializer("fastjson", new(JSONSerializer)) -} diff --git a/service/aiproxy/common/gin.go b/service/aiproxy/common/gin.go deleted file mode 100644 index deb1bbd67..000000000 --- a/service/aiproxy/common/gin.go +++ /dev/null @@ -1,109 +0,0 @@ -package common - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/bytedance/sonic/ast" - "github.com/gin-gonic/gin" -) - -type RequestBodyKey struct{} - -const ( - MaxRequestBodySize = 1024 * 1024 * 50 // 50MB -) - -func LimitReader(r io.Reader, n int64) io.Reader { return &LimitedReader{r, n} } - -type LimitedReader struct { - R io.Reader - N int64 -} - -var ErrLimitedReaderExceeded = errors.New("limited reader exceeded") - -func (l *LimitedReader) Read(p []byte) (n int, err error) { - if l.N <= 0 { - return 0, ErrLimitedReaderExceeded - } - if int64(len(p)) > l.N { - p = p[0:l.N] - } - n, err = l.R.Read(p) - l.N -= int64(n) - return -} - -func GetRequestBody(req *http.Request) ([]byte, error) { - contentType := req.Header.Get("Content-Type") - if contentType == "application/x-www-form-urlencoded" || - strings.HasPrefix(contentType, "multipart/form-data") { - return nil, nil - } - - requestBody := req.Context().Value(RequestBodyKey{}) - if requestBody != nil { - return requestBody.([]byte), nil - } - var buf []byte - var err error - defer func() { - req.Body.Close() - if err == nil { - req.Body = io.NopCloser(bytes.NewBuffer(buf)) - } - }() - if req.ContentLength <= 0 || req.Header.Get("Content-Type") != "application/json" { - buf, err = io.ReadAll(LimitReader(req.Body, MaxRequestBodySize)) - if err != nil { - if errors.Is(err, ErrLimitedReaderExceeded) { - return nil, fmt.Errorf("request body too large, max: %d", MaxRequestBodySize) - } - return nil, fmt.Errorf("request body read failed: %w", err) - } - } else { - if req.ContentLength > MaxRequestBodySize { - return nil, fmt.Errorf("request body too large: %d, max: %d", req.ContentLength, MaxRequestBodySize) - } - buf = make([]byte, req.ContentLength) - _, err = io.ReadFull(req.Body, buf) - } - if err != nil { - return nil, fmt.Errorf("request body read failed: %w", err) - } - ctx := req.Context() - bufCtx := context.WithValue(ctx, RequestBodyKey{}, buf) - *req = *req.WithContext(bufCtx) - return buf, nil -} - -func UnmarshalBodyReusable(req *http.Request, v any) error { - requestBody, err := GetRequestBody(req) - if err != nil { - return err - } - return sonic.Unmarshal(requestBody, &v) -} - -func UnmarshalBody2Node(req *http.Request) (ast.Node, error) { - requestBody, err := GetRequestBody(req) - if err != nil { - return ast.Node{}, err - } - return sonic.Get(requestBody) -} - -func SetEventStreamHeaders(c *gin.Context) { - c.Writer.Header().Set("Content-Type", "text/event-stream") - c.Writer.Header().Set("Cache-Control", "no-cache") - c.Writer.Header().Set("Connection", "keep-alive") - c.Writer.Header().Set("Transfer-Encoding", "chunked") - c.Writer.Header().Set("X-Accel-Buffering", "no") -} diff --git a/service/aiproxy/common/image/image.go b/service/aiproxy/common/image/image.go deleted file mode 100644 index 8eac5f595..000000000 --- a/service/aiproxy/common/image/image.go +++ /dev/null @@ -1,131 +0,0 @@ -package image - -import ( - "bytes" - "context" - "encoding/base64" - "errors" - "fmt" - "image" - - // import gif decoder - _ "image/gif" - // import jpeg decoder - _ "image/jpeg" - // import png decoder - _ "image/png" - "io" - "net/http" - "regexp" - "strings" - - "github.com/labring/sealos/service/aiproxy/common" - // import webp decoder - _ "golang.org/x/image/webp" -) - -// Regex to match data URL pattern -var dataURLPattern = regexp.MustCompile(`data:image/([^;]+);base64,(.*)`) - -func IsImageURL(resp *http.Response) bool { - return strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") -} - -func GetImageSizeFromURL(url string) (width int, height int, err error) { - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - return 0, 0, err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return 0, 0, fmt.Errorf("status code: %d", resp.StatusCode) - } - - isImage := IsImageURL(resp) - if !isImage { - return - } - img, _, err := image.DecodeConfig(resp.Body) - if err != nil { - return - } - return img.Width, img.Height, nil -} - -const ( - MaxImageSize = 1024 * 1024 * 5 // 5MB -) - -func GetImageFromURL(ctx context.Context, url string) (string, string, error) { - // Check if the URL is a data URL - matches := dataURLPattern.FindStringSubmatch(url) - if len(matches) == 3 { - // URL is a data URL - return "image/" + matches[1], matches[2], nil - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return "", "", err - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return "", "", err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return "", "", fmt.Errorf("status code: %d", resp.StatusCode) - } - isImage := IsImageURL(resp) - if !isImage { - return "", "", errors.New("not an image") - } - var buf []byte - if resp.ContentLength <= 0 { - buf, err = io.ReadAll(common.LimitReader(resp.Body, MaxImageSize)) - if err != nil { - if errors.Is(err, common.ErrLimitedReaderExceeded) { - return "", "", fmt.Errorf("image too large, max: %d", MaxImageSize) - } - return "", "", fmt.Errorf("image read failed: %w", err) - } - } else { - if resp.ContentLength > MaxImageSize { - return "", "", fmt.Errorf("image too large: %d, max: %d", resp.ContentLength, MaxImageSize) - } - buf = make([]byte, resp.ContentLength) - _, err = io.ReadFull(resp.Body, buf) - } - if err != nil { - return "", "", err - } - return resp.Header.Get("Content-Type"), base64.StdEncoding.EncodeToString(buf), nil -} - -var reg = regexp.MustCompile(`data:image/([^;]+);base64,`) - -func GetImageSizeFromBase64(encoded string) (width int, height int, err error) { - decoded, err := base64.StdEncoding.DecodeString(reg.ReplaceAllString(encoded, "")) - if err != nil { - return 0, 0, err - } - - img, _, err := image.DecodeConfig(bytes.NewReader(decoded)) - if err != nil { - return 0, 0, err - } - - return img.Width, img.Height, nil -} - -func GetImageSize(image string) (width int, height int, err error) { - if strings.HasPrefix(image, "data:image/") { - return GetImageSizeFromBase64(image) - } - return GetImageSizeFromURL(image) -} diff --git a/service/aiproxy/common/image/svg.go b/service/aiproxy/common/image/svg.go deleted file mode 100644 index a183d2b6f..000000000 --- a/service/aiproxy/common/image/svg.go +++ /dev/null @@ -1,45 +0,0 @@ -package image - -import ( - "image" - "image/color" - "io" - - "github.com/srwiley/oksvg" - "github.com/srwiley/rasterx" -) - -func Decode(r io.Reader) (image.Image, error) { - icon, err := oksvg.ReadIconStream(r) - if err != nil { - return nil, err - } - - w, h := int(icon.ViewBox.W), int(icon.ViewBox.H) - icon.SetTarget(0, 0, float64(w), float64(h)) - - rgba := image.NewRGBA(image.Rect(0, 0, w, h)) - icon.Draw(rasterx.NewDasher(w, h, rasterx.NewScannerGV(w, h, rgba, rgba.Bounds())), 1) - - return rgba, err -} - -func DecodeConfig(r io.Reader) (image.Config, error) { - var config image.Config - - icon, err := oksvg.ReadIconStream(r) - if err != nil { - return config, err - } - - config.ColorModel = color.RGBAModel - config.Width = int(icon.ViewBox.W) - config.Height = int(icon.ViewBox.H) - - return config, nil -} - -func init() { - image.RegisterFormat("svg", "= 'a' && c <= 'z' { - c = c - 'a' + 'A' - } - key[i+16] = c - } - return conv.BytesToString(key) -} - -//nolint:gosec -func GetRandomString(length int) string { - key := make([]byte, length) - for i := 0; i < length; i++ { - key[i] = keyChars[rand.IntN(len(keyChars))] - } - return conv.BytesToString(key) -} - -//nolint:gosec -func GetRandomNumberString(length int) string { - key := make([]byte, length) - for i := 0; i < length; i++ { - key[i] = keyNumbers[rand.IntN(len(keyNumbers))] - } - return conv.BytesToString(key) -} - -// RandRange returns a random number between min and max (max is not included) -// -//nolint:gosec -func RandRange(_min, _max int) int { - return _min + rand.IntN(_max-_min) -} diff --git a/service/aiproxy/common/redis.go b/service/aiproxy/common/redis.go deleted file mode 100644 index e83e9d8f4..000000000 --- a/service/aiproxy/common/redis.go +++ /dev/null @@ -1,55 +0,0 @@ -package common - -import ( - "context" - "os" - "time" - - "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" -) - -var ( - RDB *redis.Client - RedisEnabled = false -) - -// InitRedisClient This function is called after init() -func InitRedisClient() (err error) { - if os.Getenv("REDIS_CONN_STRING") == "" { - log.Info("REDIS_CONN_STRING not set, redis is not enabled") - return nil - } - RedisEnabled = true - log.Info("redis is enabled") - opt, err := redis.ParseURL(os.Getenv("REDIS_CONN_STRING")) - if err != nil { - log.Fatal("failed to parse redis connection string: " + err.Error()) - } - RDB = redis.NewClient(opt) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - _, err = RDB.Ping(ctx).Result() - if err != nil { - log.Errorf("failed to ping redis: %s", err.Error()) - } - - return nil -} - -func RedisSet(key string, value string, expiration time.Duration) error { - ctx := context.Background() - return RDB.Set(ctx, key, value, expiration).Err() -} - -func RedisGet(key string) (string, error) { - ctx := context.Background() - return RDB.Get(ctx, key).Result() -} - -func RedisDel(key string) error { - ctx := context.Background() - return RDB.Del(ctx, key).Err() -} diff --git a/service/aiproxy/common/render/event.go b/service/aiproxy/common/render/event.go deleted file mode 100644 index adc9016cd..000000000 --- a/service/aiproxy/common/render/event.go +++ /dev/null @@ -1,51 +0,0 @@ -package render - -import ( - "net/http" - - "github.com/labring/sealos/service/aiproxy/common/conv" -) - -var ( - contentType = []string{"text/event-stream"} - noCache = []string{"no-cache"} -) - -type OpenAISSE struct { - Data string -} - -const ( - nn = "\n\n" - data = "data: " -) - -var ( - nnBytes = conv.StringToBytes(nn) - dataBytes = conv.StringToBytes(data) -) - -func (r *OpenAISSE) Render(w http.ResponseWriter) error { - r.WriteContentType(w) - - for _, bytes := range [][]byte{ - dataBytes, - conv.StringToBytes(r.Data), - nnBytes, - } { - // nosemgrep: go.lang.security.audit.xss.no-direct-write-to-responsewriter.no-direct-write-to-responsewriter - if _, err := w.Write(bytes); err != nil { - return err - } - } - return nil -} - -func (r *OpenAISSE) WriteContentType(w http.ResponseWriter) { - header := w.Header() - header["Content-Type"] = contentType - - if _, exist := header["Cache-Control"]; !exist { - header["Cache-Control"] = noCache - } -} diff --git a/service/aiproxy/common/render/render.go b/service/aiproxy/common/render/render.go deleted file mode 100644 index b8db50910..000000000 --- a/service/aiproxy/common/render/render.go +++ /dev/null @@ -1,43 +0,0 @@ -package render - -import ( - "errors" - "fmt" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/conv" -) - -func StringData(c *gin.Context, str string) { - if len(c.Errors) > 0 { - return - } - if c.IsAborted() { - return - } - c.Render(-1, &OpenAISSE{Data: str}) - c.Writer.Flush() -} - -func ObjectData(c *gin.Context, object any) error { - if len(c.Errors) > 0 { - return c.Errors.Last() - } - if c.IsAborted() { - return errors.New("context aborted") - } - jsonData, err := sonic.Marshal(object) - if err != nil { - return fmt.Errorf("error marshalling object: %w", err) - } - c.Render(-1, &OpenAISSE{Data: conv.BytesToString(jsonData)}) - c.Writer.Flush() - return nil -} - -const DONE = "[DONE]" - -func Done(c *gin.Context) { - StringData(c, DONE) -} diff --git a/service/aiproxy/common/rpmlimit/mem.go b/service/aiproxy/common/rpmlimit/mem.go deleted file mode 100644 index 5463d164b..000000000 --- a/service/aiproxy/common/rpmlimit/mem.go +++ /dev/null @@ -1,93 +0,0 @@ -package rpmlimit - -import ( - "sync" - "time" -) - -type InMemoryRateLimiter struct { - store map[string]*RateLimitWindow - mutex sync.RWMutex - expirationDuration time.Duration -} - -type RateLimitWindow struct { - timestamps []int64 - lastAccess int64 -} - -func (l *InMemoryRateLimiter) Init(expirationDuration time.Duration) { - if l.store == nil { - l.mutex.Lock() - if l.store == nil { - l.store = make(map[string]*RateLimitWindow) - l.expirationDuration = expirationDuration - if expirationDuration > 0 { - go l.clearExpiredItems() - } - } - l.mutex.Unlock() - } -} - -func (l *InMemoryRateLimiter) clearExpiredItems() { - ticker := time.NewTicker(l.expirationDuration) - defer ticker.Stop() - - for range ticker.C { - l.mutex.Lock() - now := time.Now().Unix() - for key, window := range l.store { - if now-window.lastAccess > int64(l.expirationDuration.Seconds()) { - delete(l.store, key) - } - } - l.mutex.Unlock() - } -} - -// Request parameter duration's unit is seconds -func (l *InMemoryRateLimiter) Request(key string, maxRequestNum int, duration time.Duration) bool { - now := time.Now().Unix() - cutoff := now - int64(duration.Seconds()) - - l.mutex.RLock() - window, exists := l.store[key] - l.mutex.RUnlock() - - if !exists { - l.mutex.Lock() - window = &RateLimitWindow{ - timestamps: make([]int64, 0, maxRequestNum), - lastAccess: now, - } - l.store[key] = window - window.timestamps = append(window.timestamps, now) - l.mutex.Unlock() - return true - } - - l.mutex.Lock() - defer l.mutex.Unlock() - - // Update last access time - window.lastAccess = now - - // Remove expired timestamps - idx := 0 - for i, ts := range window.timestamps { - if ts > cutoff { - idx = i - break - } - } - window.timestamps = window.timestamps[idx:] - - // Check if we can add a new request - if len(window.timestamps) < maxRequestNum { - window.timestamps = append(window.timestamps, now) - return true - } - - return false -} diff --git a/service/aiproxy/common/rpmlimit/rate-limit.go b/service/aiproxy/common/rpmlimit/rate-limit.go deleted file mode 100644 index 32668438e..000000000 --- a/service/aiproxy/common/rpmlimit/rate-limit.go +++ /dev/null @@ -1,174 +0,0 @@ -package rpmlimit - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/labring/sealos/service/aiproxy/common" - log "github.com/sirupsen/logrus" -) - -var inMemoryRateLimiter InMemoryRateLimiter - -const ( - groupModelRPMKey = "group_model_rpm:%s:%s" - overLimitRPMKey = "over_limit_rpm:%s:%s" -) - -var pushRequestScript = ` -local key = KEYS[1] -local over_limit_key = KEYS[2] -local window = tonumber(ARGV[1]) -local current_time = tonumber(ARGV[2]) -local max_requests = tonumber(ARGV[3]) -local cutoff = current_time - window - -redis.call('ZREMRANGEBYSCORE', key, '-inf', cutoff) -redis.call('ZREMRANGEBYSCORE', over_limit_key, '-inf', cutoff) -local count = redis.call('ZCOUNT', key, cutoff, current_time) -local over_limit_count = redis.call('ZCOUNT', over_limit_key, cutoff, current_time) - -if count <= max_requests then - redis.call('ZADD', key, current_time, current_time) - redis.call('PEXPIRE', key, window / 1000) - count = count + 1 -else - redis.call('ZADD', over_limit_key, current_time, current_time) - redis.call('PEXPIRE', over_limit_key, window / 1000) - over_limit_count = over_limit_count + 1 -end - -return string.format("%d:%d", count, over_limit_count) -` - -var getRequestCountScript = ` -local pattern = KEYS[1] -local over_limit_pattern = KEYS[2] -local window = tonumber(ARGV[1]) -local current_time = tonumber(ARGV[2]) -local cutoff = current_time - window - -local total = 0 - -local keys = redis.call('KEYS', pattern) -for _, key in ipairs(keys) do - redis.call('ZREMRANGEBYSCORE', key, '-inf', cutoff) - total = total + redis.call('ZCOUNT', key, cutoff, current_time) -end - -local over_limit_keys = redis.call('KEYS', over_limit_pattern) -for _, key in ipairs(over_limit_keys) do - redis.call('ZREMRANGEBYSCORE', key, '-inf', cutoff) - total = total + redis.call('ZCOUNT', key, cutoff, current_time) -end - -return total -` - -func GetRPM(ctx context.Context, group, model string) (int64, error) { - if !common.RedisEnabled { - return 0, nil - } - - var pattern string - var overLimitPattern string - if group == "" && model == "" { - pattern = "group_model_rpm:*:*" - overLimitPattern = "over_limit_rpm:*:*" - } else if group == "" { - pattern = "group_model_rpm:*:" + model - overLimitPattern = "over_limit_rpm:*:" + model - } else if model == "" { - pattern = fmt.Sprintf("group_model_rpm:%s:*", group) - overLimitPattern = fmt.Sprintf("over_limit_rpm:%s:*", group) - } else { - pattern = fmt.Sprintf("group_model_rpm:%s:%s", group, model) - overLimitPattern = fmt.Sprintf("over_limit_rpm:%s:%s", group, model) - } - - rdb := common.RDB - result, err := rdb.Eval( - ctx, - getRequestCountScript, - []string{pattern, overLimitPattern}, - time.Minute.Microseconds(), - time.Now().UnixMicro(), - ).Int64() - if err != nil { - return 0, err - } - return result, nil -} - -func redisRateLimitRequest(ctx context.Context, group, model string, maxRequestNum int64, duration time.Duration) (bool, error) { - result, _, err := PushRequest(ctx, group, model, maxRequestNum, duration) - if err != nil { - return false, err - } - return result <= maxRequestNum, nil -} - -func PushRequest(ctx context.Context, group, model string, maxRequestNum int64, duration time.Duration) (int64, int64, error) { - result, err := common.RDB.Eval( - ctx, - pushRequestScript, - []string{ - fmt.Sprintf(groupModelRPMKey, group, model), - fmt.Sprintf(overLimitRPMKey, group, model), - }, - duration.Microseconds(), - time.Now().UnixMicro(), - maxRequestNum, - ).Text() - if err != nil { - return 0, 0, err - } - count, overLimitCount, ok := strings.Cut(result, ":") - if !ok { - return 0, 0, errors.New("invalid result") - } - countInt, err := strconv.ParseInt(count, 10, 64) - if err != nil { - return 0, 0, err - } - overLimitCountInt, err := strconv.ParseInt(overLimitCount, 10, 64) - if err != nil { - return 0, 0, err - } - return countInt, overLimitCountInt, nil -} - -func RateLimit(ctx context.Context, group, model string, maxRequestNum int64, duration time.Duration) (bool, error) { - if maxRequestNum == 0 { - return true, nil - } - if common.RedisEnabled { - return redisRateLimitRequest(ctx, group, model, maxRequestNum, duration) - } - return MemoryRateLimit(ctx, group, model, maxRequestNum, duration), nil -} - -// ignore redis error -func ForceRateLimit(ctx context.Context, group, model string, maxRequestNum int64, duration time.Duration) bool { - if maxRequestNum == 0 { - return true - } - if common.RedisEnabled { - ok, err := redisRateLimitRequest(ctx, group, model, maxRequestNum, duration) - if err == nil { - return ok - } - log.Error("rate limit error: " + err.Error()) - } - return MemoryRateLimit(ctx, group, model, maxRequestNum, duration) -} - -func MemoryRateLimit(_ context.Context, group, model string, maxRequestNum int64, duration time.Duration) bool { - // It's safe to call multi times. - inMemoryRateLimiter.Init(3 * time.Minute) - return inMemoryRateLimiter.Request(fmt.Sprintf(groupModelRPMKey, group, model), int(maxRequestNum), duration) -} diff --git a/service/aiproxy/common/splitter/splitter.go b/service/aiproxy/common/splitter/splitter.go deleted file mode 100644 index 18ac4b9c2..000000000 --- a/service/aiproxy/common/splitter/splitter.go +++ /dev/null @@ -1,122 +0,0 @@ -package splitter - -import "bytes" - -type Splitter struct { - head []byte - tail []byte - headLen int - tailLen int - buffer []byte - state int - partialTailPos int - kmpNext []int -} - -func NewSplitter(head, tail []byte) *Splitter { - return &Splitter{ - head: head, - tail: tail, - headLen: len(head), - tailLen: len(tail), - kmpNext: computeKMPNext(tail), - } -} - -func computeKMPNext(pattern []byte) []int { - n := len(pattern) - next := make([]int, n) - if n == 0 { - return next - } - next[0] = 0 - for i := 1; i < n; i++ { - j := next[i-1] - for j > 0 && pattern[i] != pattern[j] { - j = next[j-1] - } - if pattern[i] == pattern[j] { - j++ - } - next[i] = j - } - return next -} - -func (s *Splitter) Process(data []byte) ([]byte, []byte) { - if len(data) == 0 { - return nil, nil - } - switch s.state { - case 0: - s.buffer = append(s.buffer, data...) - bufferLen := len(s.buffer) - minLen := bufferLen - if minLen > s.headLen { - minLen = s.headLen - } - if minLen > 0 { - if !bytes.Equal(s.buffer[:minLen], s.head[:minLen]) { - s.state = 2 - remaining := s.buffer - s.buffer = nil - return nil, remaining - } - } - - if bufferLen < s.headLen { - return nil, nil - } - - s.state = 1 - s.buffer = s.buffer[s.headLen:] - if len(s.buffer) == 0 { - return nil, nil - } - return s.processSeekTail() - case 1: - s.buffer = append(s.buffer, data...) - return s.processSeekTail() - default: - return nil, data - } -} - -func (s *Splitter) processSeekTail() ([]byte, []byte) { - data := s.buffer - j := s.partialTailPos - tail := s.tail - tailLen := s.tailLen - kmpNext := s.kmpNext - - var i int - for i = 0; i < len(data); i++ { - for j > 0 && data[i] != tail[j] { - j = kmpNext[j-1] - } - if data[i] == tail[j] { - j++ - if j == tailLen { - end := i - tailLen + 1 - if end < 0 { - end = 0 - } - result := data[:end] - remaining := data[i+1:] - s.buffer = nil - s.state = 2 - s.partialTailPos = 0 - return result, remaining - } - } - } - splitAt := len(data) - j - if splitAt < 0 { - splitAt = 0 - } - result := data[:splitAt] - remainingPart := data[splitAt:] - s.partialTailPos = j - s.buffer = remainingPart - return result, nil -} diff --git a/service/aiproxy/common/splitter/think.go b/service/aiproxy/common/splitter/think.go deleted file mode 100644 index 99b1f3d32..000000000 --- a/service/aiproxy/common/splitter/think.go +++ /dev/null @@ -1,17 +0,0 @@ -package splitter - -import "github.com/labring/sealos/service/aiproxy/common/conv" - -const ( - ThinkHead = "\n" - ThinkTail = "\n" -) - -var ( - thinkHeadBytes = conv.StringToBytes(ThinkHead) - thinkTailBytes = conv.StringToBytes(ThinkTail) -) - -func NewThinkSplitter() *Splitter { - return NewSplitter(thinkHeadBytes, thinkTailBytes) -} diff --git a/service/aiproxy/common/trunc.go b/service/aiproxy/common/trunc.go deleted file mode 100644 index e4d85bfe8..000000000 --- a/service/aiproxy/common/trunc.go +++ /dev/null @@ -1,31 +0,0 @@ -package common - -import ( - "unicode/utf8" - - "github.com/labring/sealos/service/aiproxy/common/conv" -) - -func TruncateByRune(s string, length int) string { - total := 0 - for _, r := range s { - runeLen := utf8.RuneLen(r) - if runeLen == -1 || total+runeLen > length { - return s[:total] - } - total += runeLen - } - return s[:total] -} - -func TruncateBytesByRune(b []byte, length int) []byte { - total := 0 - for _, r := range conv.BytesToString(b) { - runeLen := utf8.RuneLen(r) - if runeLen == -1 || total+runeLen > length { - return b[:total] - } - total += runeLen - } - return b[:total] -} diff --git a/service/aiproxy/common/trylock/lock.go b/service/aiproxy/common/trylock/lock.go deleted file mode 100644 index 5bad5d072..000000000 --- a/service/aiproxy/common/trylock/lock.go +++ /dev/null @@ -1,74 +0,0 @@ -package trylock - -import ( - "context" - "sync" - "time" - - "github.com/labring/sealos/service/aiproxy/common" - log "github.com/sirupsen/logrus" -) - -var memRecord = sync.Map{} - -func init() { - go cleanMemLock() -} - -func cleanMemLock() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for now := range ticker.C { - memRecord.Range(func(key, value any) bool { - if exp, ok := value.(time.Time); ok { - if now.After(exp) { - memRecord.Delete(key) - } - } else { - memRecord.Delete(key) - } - return true - }) - } -} - -func MemLock(key string, expiration time.Duration) bool { - now := time.Now() - newExpiration := now.Add(expiration) - - for { - actual, loaded := memRecord.LoadOrStore(key, newExpiration) - if !loaded { - return true - } - oldExpiration, ok := actual.(time.Time) - if !ok { - memRecord.Delete(key) - continue - } - if now.After(oldExpiration) { - if memRecord.CompareAndSwap(key, actual, newExpiration) { - return true - } - continue - } - return false - } -} - -func Lock(key string, expiration time.Duration) bool { - if !common.RedisEnabled { - return MemLock(key, expiration) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - result, err := common.RDB.SetNX(ctx, key, true, expiration).Result() - if err != nil { - if MemLock("lockerror", time.Second*3) { - log.Errorf("try notify error: %v", err) - } - return MemLock(key, expiration) - } - return result -} diff --git a/service/aiproxy/common/trylock/lock_test.go b/service/aiproxy/common/trylock/lock_test.go deleted file mode 100644 index 808d565f7..000000000 --- a/service/aiproxy/common/trylock/lock_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package trylock_test - -import ( - "testing" - "time" - - "github.com/labring/sealos/service/aiproxy/common/trylock" -) - -func TestMemLock(t *testing.T) { - if !trylock.MemLock("", time.Second) { - t.Error("Expected true, Got false") - } - if trylock.MemLock("", time.Second) { - t.Error("Expected false, Got true") - } - if trylock.MemLock("", time.Second) { - t.Error("Expected false, Got true") - } - time.Sleep(time.Second) - if !trylock.MemLock("", time.Second) { - t.Error("Expected true, Got false") - } -} diff --git a/service/aiproxy/controller/channel-billing.go b/service/aiproxy/controller/channel-billing.go deleted file mode 100644 index eac975a3a..000000000 --- a/service/aiproxy/controller/channel-billing.go +++ /dev/null @@ -1,152 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "net/http" - "strconv" - "sync" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/balance" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/channeltype" - log "github.com/sirupsen/logrus" -) - -// https://github.com/labring/sealos/service/aiproxy/issues/79 - -func updateChannelBalance(channel *model.Channel) (float64, error) { - adaptorI, ok := channeltype.GetAdaptor(channel.Type) - if !ok { - return 0, fmt.Errorf("invalid channel type: %d, channel: %s(%d)", channel.Type, channel.Name, channel.ID) - } - if getBalance, ok := adaptorI.(adaptor.Balancer); ok { - balance, err := getBalance.GetBalance(channel) - if err != nil && !errors.Is(err, adaptor.ErrGetBalanceNotImplemented) { - return 0, fmt.Errorf("failed to get channel[%d] %s(%d) balance: %s", channel.Type, channel.Name, channel.ID, err.Error()) - } - if err := channel.UpdateBalance(balance); err != nil { - return 0, fmt.Errorf("failed to update channel [%d] %s(%d) balance: %s", channel.Type, channel.Name, channel.ID, err.Error()) - } - if !errors.Is(err, adaptor.ErrGetBalanceNotImplemented) && - balance < channel.GetBalanceThreshold() { - return 0, fmt.Errorf("channel[%d] %s(%d) balance is less than threshold: %f", channel.Type, channel.Name, channel.ID, balance) - } - return balance, nil - } - return 0, nil -} - -func UpdateChannelBalance(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: err.Error(), - }) - return - } - channel, err := model.GetChannelByID(id) - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: err.Error(), - }) - return - } - balance, err := updateChannelBalance(channel) - if err != nil { - notify.Error(fmt.Sprintf("check channel[%d] %s(%d) balance error", channel.Type, channel.Name, channel.ID), err.Error()) - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: err.Error(), - }) - return - } - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: true, - Message: "", - Data: balance, - }) -} - -func updateAllChannelsBalance() error { - channels, err := model.GetAllChannels() - if err != nil { - return err - } - - var wg sync.WaitGroup - semaphore := make(chan struct{}, 10) - - for _, channel := range channels { - if !channel.EnabledAutoBalanceCheck { - continue - } - wg.Add(1) - semaphore <- struct{}{} - go func(ch *model.Channel) { - defer wg.Done() - defer func() { <-semaphore }() - _, err := updateChannelBalance(ch) - if err != nil { - notify.Error(fmt.Sprintf("check channel[%d] %s(%d) balance error", ch.Type, ch.Name, ch.ID), err.Error()) - } - }(channel) - } - - wg.Wait() - return nil -} - -func UpdateAllChannelsBalance(c *gin.Context) { - err := updateAllChannelsBalance() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func UpdateChannelsBalance(frequency time.Duration) { - for { - time.Sleep(frequency) - _ = updateAllChannelsBalance() - } -} - -// subscription -func GetSubscription(c *gin.Context) { - group := middleware.GetGroup(c) - b, _, err := balance.GetGroupRemainBalance(c, *group) - if err != nil { - if errors.Is(err, balance.ErrNoRealNameUsedAmountLimit) { - middleware.ErrorResponse(c, http.StatusForbidden, err.Error()) - return - } - log.Errorf("get group (%s) balance failed: %s", group.ID, err) - middleware.ErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("get group (%s) balance failed", group.ID)) - return - } - token := middleware.GetToken(c) - quota := token.Quota - if quota <= 0 { - quota = b - } - c.JSON(http.StatusOK, openai.SubscriptionResponse{ - HardLimitUSD: quota + token.UsedAmount, - SoftLimitUSD: b, - SystemHardLimitUSD: quota + token.UsedAmount, - }) -} - -func GetUsage(c *gin.Context) { - token := middleware.GetToken(c) - c.JSON(http.StatusOK, openai.UsageResponse{TotalUsage: token.UsedAmount * 100}) -} diff --git a/service/aiproxy/controller/channel-test.go b/service/aiproxy/controller/channel-test.go deleted file mode 100644 index e1561068b..000000000 --- a/service/aiproxy/controller/channel-test.go +++ /dev/null @@ -1,448 +0,0 @@ -package controller - -import ( - "context" - "errors" - "fmt" - "io" - "math/rand/v2" - "net/http" - "net/http/httptest" - "net/url" - "slices" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/common/trylock" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/monitor" - "github.com/labring/sealos/service/aiproxy/relay/channeltype" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" - log "github.com/sirupsen/logrus" -) - -const channelTestRequestID = "channel-test" - -var ( - modelTypeCache map[string]relaymode.Mode = make(map[string]relaymode.Mode) - modelTypeCacheOnce sync.Once -) - -func guessModelType(model string) relaymode.Mode { - modelTypeCacheOnce.Do(func() { - for _, c := range channeltype.ChannelAdaptor { - for _, m := range c.GetModelList() { - if _, ok := modelTypeCache[m.Model]; !ok { - modelTypeCache[m.Model] = m.Type - } - } - } - }) - - if cachedType, ok := modelTypeCache[model]; ok { - return cachedType - } - return relaymode.Unknown -} - -// testSingleModel tests a single model in the channel -func testSingleModel(mc *model.ModelCaches, channel *model.Channel, modelName string) (*model.ChannelTest, error) { - modelConfig, ok := mc.ModelConfig.GetModelConfig(modelName) - if !ok { - return nil, errors.New(modelName + " model config not found") - } - if modelConfig.Type == relaymode.Unknown { - newModelConfig := *modelConfig - newModelConfig.Type = guessModelType(modelName) - modelConfig = &newModelConfig - } - - body, mode, err := utils.BuildRequest(modelConfig) - if err != nil { - return nil, err - } - - w := httptest.NewRecorder() - newc, _ := gin.CreateTestContext(w) - newc.Request = &http.Request{ - URL: &url.URL{}, - Body: io.NopCloser(body), - Header: make(http.Header), - } - middleware.SetRequestID(newc, channelTestRequestID) - - meta := meta.NewMeta( - channel, - mode, - modelName, - modelConfig, - meta.WithRequestID(channelTestRequestID), - meta.WithChannelTest(true), - ) - relayController, ok := relayController(mode) - if !ok { - return nil, fmt.Errorf("relay mode %d not implemented", mode) - } - bizErr := relayController(meta, newc) - success := bizErr == nil - var respStr string - var code int - if success { - switch meta.Mode { - case relaymode.AudioSpeech, - relaymode.ImagesGenerations: - respStr = "" - default: - respStr = w.Body.String() - } - code = w.Code - } else { - respStr = bizErr.Error.JSONOrEmpty() - code = bizErr.StatusCode - } - - return channel.UpdateModelTest( - meta.RequestAt, - meta.OriginModel, - meta.ActualModel, - meta.Mode, - time.Since(meta.RequestAt).Seconds(), - success, - respStr, - code, - ) -} - -//nolint:goconst -func TestChannel(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: err.Error(), - }) - return - } - - modelName := c.Param("model") - if modelName == "" { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: "model is required", - }) - return - } - - channel, err := model.LoadChannelByID(id) - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: "channel not found", - }) - return - } - - if !slices.Contains(channel.Models, modelName) { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: "model not supported by channel", - }) - return - } - - ct, err := testSingleModel(model.LoadModelCaches(), channel, modelName) - if err != nil { - log.Errorf("failed to test channel %s(%d) model %s: %s", channel.Name, channel.ID, modelName, err.Error()) - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: fmt.Sprintf("failed to test channel %s(%d) model %s: %s", channel.Name, channel.ID, modelName, err.Error()), - }) - return - } - - if c.Query("success_body") != "true" && ct.Success { - ct.Response = "" - } - - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: true, - Data: ct, - }) -} - -type testResult struct { - Data *model.ChannelTest `json:"data,omitempty"` - Message string `json:"message,omitempty"` - Success bool `json:"success"` -} - -func processTestResult(mc *model.ModelCaches, channel *model.Channel, modelName string, returnSuccess bool, successResponseBody bool) *testResult { - ct, err := testSingleModel(mc, channel, modelName) - - e := &utils.UnsupportedModelTypeError{} - if errors.As(err, &e) { - log.Errorf("model %s not supported test: %s", modelName, err.Error()) - return nil - } - - result := &testResult{ - Success: err == nil, - } - if err != nil { - result.Message = fmt.Sprintf("failed to test channel %s(%d) model %s: %s", channel.Name, channel.ID, modelName, err.Error()) - return result - } - - if !ct.Success { - result.Data = ct - return result - } - - if !returnSuccess { - return nil - } - - if !successResponseBody { - ct.Response = "" - } - result.Data = ct - return result -} - -func TestChannelModels(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: err.Error(), - }) - return - } - - channel, err := model.LoadChannelByID(id) - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: "channel not found", - }) - return - } - - returnSuccess := c.Query("return_success") == "true" - successResponseBody := c.Query("success_body") == "true" - isStream := c.Query("stream") == "true" - - if isStream { - common.SetEventStreamHeaders(c) - } - - results := make([]*testResult, 0) - resultsMutex := sync.Mutex{} - hasError := atomic.Bool{} - - var wg sync.WaitGroup - semaphore := make(chan struct{}, 5) - - models := slices.Clone(channel.Models) - rand.Shuffle(len(models), func(i, j int) { - models[i], models[j] = models[j], models[i] - }) - - mc := model.LoadModelCaches() - - for _, modelName := range models { - wg.Add(1) - semaphore <- struct{}{} - - go func(model string) { - defer wg.Done() - defer func() { <-semaphore }() - - result := processTestResult(mc, channel, model, returnSuccess, successResponseBody) - if result == nil { - return - } - if !result.Success || (result.Data != nil && !result.Data.Success) { - hasError.Store(true) - } - resultsMutex.Lock() - if isStream { - err := render.ObjectData(c, result) - if err != nil { - log.Errorf("failed to render result: %s", err.Error()) - } - } else { - results = append(results, result) - } - resultsMutex.Unlock() - }(modelName) - } - - wg.Wait() - - if !hasError.Load() { - err := model.ClearLastTestErrorAt(channel.ID) - if err != nil { - log.Errorf("failed to clear last test error at for channel %s(%d): %s", channel.Name, channel.ID, err.Error()) - } - } - - if !isStream { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: true, - Data: results, - }) - } -} - -func TestAllChannels(c *gin.Context) { - testDisabled := c.Query("test_disabled") == "true" - var channels []*model.Channel - var err error - if testDisabled { - channels, err = model.LoadChannels() - } else { - channels, err = model.LoadEnabledChannels() - } - if err != nil { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: false, - Message: err.Error(), - }) - return - } - returnSuccess := c.Query("return_success") == "true" - successResponseBody := c.Query("success_body") == "true" - isStream := c.Query("stream") == "true" - - if isStream { - common.SetEventStreamHeaders(c) - } - - results := make([]*testResult, 0) - resultsMutex := sync.Mutex{} - hasErrorMap := make(map[int]*atomic.Bool) - - var wg sync.WaitGroup - semaphore := make(chan struct{}, 5) - - newChannels := slices.Clone(channels) - rand.Shuffle(len(newChannels), func(i, j int) { - newChannels[i], newChannels[j] = newChannels[j], newChannels[i] - }) - - mc := model.LoadModelCaches() - - for _, channel := range newChannels { - channelHasError := &atomic.Bool{} - hasErrorMap[channel.ID] = channelHasError - - models := slices.Clone(channel.Models) - rand.Shuffle(len(models), func(i, j int) { - models[i], models[j] = models[j], models[i] - }) - - for _, modelName := range models { - wg.Add(1) - semaphore <- struct{}{} - - go func(model string, ch *model.Channel, hasError *atomic.Bool) { - defer wg.Done() - defer func() { <-semaphore }() - - result := processTestResult(mc, ch, model, returnSuccess, successResponseBody) - if result == nil { - return - } - if !result.Success || (result.Data != nil && !result.Data.Success) { - hasError.Store(true) - } - resultsMutex.Lock() - if isStream { - err := render.ObjectData(c, result) - if err != nil { - log.Errorf("failed to render result: %s", err.Error()) - } - } else { - results = append(results, result) - } - resultsMutex.Unlock() - }(modelName, channel, channelHasError) - } - } - - wg.Wait() - - for id, hasError := range hasErrorMap { - if !hasError.Load() { - err := model.ClearLastTestErrorAt(id) - if err != nil { - log.Errorf("failed to clear last test error at for channel %d: %s", id, err.Error()) - } - } - } - - if !isStream { - c.JSON(http.StatusOK, middleware.APIResponse{ - Success: true, - Data: results, - }) - } -} - -func tryTestChannel(channelID int, modelName string) bool { - return trylock.Lock(fmt.Sprintf("channel_test_lock:%d:%s", channelID, modelName), 30*time.Second) -} - -func AutoTestBannedModels() { - log := log.WithFields(log.Fields{ - "auto_test_banned_models": "true", - }) - channels, err := monitor.GetAllBannedModelChannels(context.Background()) - if err != nil { - log.Errorf("failed to get banned channels: %s", err.Error()) - return - } - if len(channels) == 0 { - return - } - - mc := model.LoadModelCaches() - - for modelName, ids := range channels { - for _, id := range ids { - if !tryTestChannel(int(id), modelName) { - continue - } - channel, err := model.LoadChannelByID(int(id)) - if err != nil { - log.Errorf("failed to get channel by model %s: %s", modelName, err.Error()) - continue - } - result, err := testSingleModel(mc, channel, modelName) - if err != nil { - notify.Error(fmt.Sprintf("channel[%d] %s(%d) model %s test failed", channel.Type, channel.Name, channel.ID, modelName), err.Error()) - continue - } - if result.Success { - notify.Info(fmt.Sprintf("channel[%d] %s(%d) model %s test success", channel.Type, channel.Name, channel.ID, modelName), "unban it") - err = monitor.ClearChannelModelErrors(context.Background(), modelName, channel.ID) - if err != nil { - log.Errorf("clear channel errors failed: %+v", err) - } - } else { - notify.Error(fmt.Sprintf("channel[%d] %s(%d) model %s test failed", channel.Type, channel.Name, channel.ID, modelName), - fmt.Sprintf("code: %d, response: %s", result.Code, result.Response)) - } - } - } -} diff --git a/service/aiproxy/controller/channel.go b/service/aiproxy/controller/channel.go deleted file mode 100644 index 1335f4092..000000000 --- a/service/aiproxy/controller/channel.go +++ /dev/null @@ -1,281 +0,0 @@ -package controller - -import ( - "fmt" - "maps" - "net/http" - "slices" - "strconv" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/monitor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/channeltype" - log "github.com/sirupsen/logrus" -) - -func ChannelTypeNames(c *gin.Context) { - middleware.SuccessResponse(c, channeltype.ChannelNames) -} - -func ChannelTypeMetas(c *gin.Context) { - middleware.SuccessResponse(c, channeltype.ChannelMetas) -} - -func GetChannels(c *gin.Context) { - page, perPage := parsePageParams(c) - id, _ := strconv.Atoi(c.Query("id")) - name := c.Query("name") - key := c.Query("key") - channelType, _ := strconv.Atoi(c.Query("channel_type")) - baseURL := c.Query("base_url") - order := c.Query("order") - channels, total, err := model.GetChannels(page, perPage, id, name, key, channelType, baseURL, order) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, gin.H{ - "channels": channels, - "total": total, - }) -} - -func GetAllChannels(c *gin.Context) { - channels, err := model.GetAllChannels() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, channels) -} - -func AddChannels(c *gin.Context) { - channels := make([]*AddChannelRequest, 0) - err := c.ShouldBindJSON(&channels) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - _channels := make([]*model.Channel, 0, len(channels)) - for _, channel := range channels { - channels, err := channel.ToChannels() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - _channels = append(_channels, channels...) - } - err = model.BatchInsertChannels(_channels) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func SearchChannels(c *gin.Context) { - keyword := c.Query("keyword") - page, perPage := parsePageParams(c) - id, _ := strconv.Atoi(c.Query("id")) - name := c.Query("name") - key := c.Query("key") - channelType, _ := strconv.Atoi(c.Query("channel_type")) - baseURL := c.Query("base_url") - order := c.Query("order") - channels, total, err := model.SearchChannels(keyword, page, perPage, id, name, key, channelType, baseURL, order) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, gin.H{ - "channels": channels, - "total": total, - }) -} - -func GetChannel(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - channel, err := model.GetChannelByID(id) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, channel) -} - -type AddChannelRequest struct { - ModelMapping map[string]string `json:"model_mapping"` - Config *model.ChannelConfig `json:"config"` - Name string `json:"name"` - Key string `json:"key"` - BaseURL string `json:"base_url"` - Other string `json:"other"` - Models []string `json:"models"` - Type int `json:"type"` - Priority int32 `json:"priority"` - Status int `json:"status"` -} - -func (r *AddChannelRequest) ToChannel() (*model.Channel, error) { - channelType, ok := channeltype.GetAdaptor(r.Type) - if !ok { - return nil, fmt.Errorf("invalid channel type: %d", r.Type) - } - if validator, ok := channelType.(adaptor.KeyValidator); ok { - err := validator.ValidateKey(r.Key) - if err != nil { - keyHelp := validator.KeyHelp() - if keyHelp == "" { - return nil, fmt.Errorf("%s [%s(%d)] invalid key: %w", r.Name, channeltype.ChannelNames[r.Type], r.Type, err) - } - return nil, fmt.Errorf("%s [%s(%d)] invalid key: %w, %s", r.Name, channeltype.ChannelNames[r.Type], r.Type, err, keyHelp) - } - } - return &model.Channel{ - Type: r.Type, - Name: r.Name, - Key: r.Key, - BaseURL: r.BaseURL, - Models: slices.Clone(r.Models), - ModelMapping: maps.Clone(r.ModelMapping), - Priority: r.Priority, - Status: r.Status, - Config: r.Config, - }, nil -} - -func (r *AddChannelRequest) ToChannels() ([]*model.Channel, error) { - keys := strings.Split(r.Key, "\n") - channels := make([]*model.Channel, 0, len(keys)) - for _, key := range keys { - if key == "" { - continue - } - c, err := r.ToChannel() - if err != nil { - return nil, err - } - c.Key = key - channels = append(channels, c) - } - if len(channels) == 0 { - ch, err := r.ToChannel() - if err != nil { - return nil, err - } - return []*model.Channel{ch}, nil - } - return channels, nil -} - -func AddChannel(c *gin.Context) { - channel := AddChannelRequest{} - err := c.ShouldBindJSON(&channel) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - channels, err := channel.ToChannels() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.BatchInsertChannels(channels) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func DeleteChannel(c *gin.Context) { - id, _ := strconv.Atoi(c.Param("id")) - err := model.DeleteChannelByID(id) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func DeleteChannels(c *gin.Context) { - ids := []int{} - err := c.ShouldBindJSON(&ids) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.DeleteChannelsByIDs(ids) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func UpdateChannel(c *gin.Context) { - idStr := c.Param("id") - if idStr == "" { - middleware.ErrorResponse(c, http.StatusOK, "id is required") - return - } - id, err := strconv.Atoi(idStr) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - channel := AddChannelRequest{} - err = c.ShouldBindJSON(&channel) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - ch, err := channel.ToChannel() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - ch.ID = id - err = model.UpdateChannel(ch) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = monitor.ClearChannelAllModelErrors(c.Request.Context(), id) - if err != nil { - log.Errorf("failed to clear channel all model errors: %+v", err) - } - middleware.SuccessResponse(c, ch) -} - -type UpdateChannelStatusRequest struct { - Status int `json:"status"` -} - -func UpdateChannelStatus(c *gin.Context) { - id, _ := strconv.Atoi(c.Param("id")) - status := UpdateChannelStatusRequest{} - err := c.ShouldBindJSON(&status) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.UpdateChannelStatusByID(id, status.Status) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = monitor.ClearChannelAllModelErrors(c.Request.Context(), id) - if err != nil { - log.Errorf("failed to clear channel all model errors: %+v", err) - } - middleware.SuccessResponse(c, nil) -} diff --git a/service/aiproxy/controller/dashboard.go b/service/aiproxy/controller/dashboard.go deleted file mode 100644 index 4630d5411..000000000 --- a/service/aiproxy/controller/dashboard.go +++ /dev/null @@ -1,233 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "net/http" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/rpmlimit" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "gorm.io/gorm" -) - -func getDashboardTime(t string) (time.Time, time.Time, model.TimeSpanType) { - end := time.Now() - var start time.Time - var timeSpan model.TimeSpanType - switch t { - case "month": - start = end.AddDate(0, 0, -30) - timeSpan = model.TimeSpanDay - case "two_week": - start = end.AddDate(0, 0, -15) - timeSpan = model.TimeSpanDay - case "week": - start = end.AddDate(0, 0, -7) - timeSpan = model.TimeSpanDay - case "day": - fallthrough - default: - start = end.AddDate(0, 0, -1) - timeSpan = model.TimeSpanHour - } - return start, end, timeSpan -} - -func fillGaps(data []*model.ChartData, start, end time.Time, t model.TimeSpanType) []*model.ChartData { - if len(data) == 0 { - return data - } - - var timeSpan time.Duration - switch t { - case model.TimeSpanDay: - timeSpan = time.Hour * 24 - default: - timeSpan = time.Hour - } - - // Handle first point - firstPoint := time.Unix(data[0].Timestamp, 0) - firstAlignedTime := firstPoint - for !firstAlignedTime.Add(-timeSpan).Before(start) { - firstAlignedTime = firstAlignedTime.Add(-timeSpan) - } - var firstIsZero bool - if !firstAlignedTime.Equal(firstPoint) { - data = append([]*model.ChartData{ - { - Timestamp: firstAlignedTime.Unix(), - }, - }, data...) - firstIsZero = true - } - - // Handle last point - lastPoint := time.Unix(data[len(data)-1].Timestamp, 0) - lastAlignedTime := lastPoint - for !lastAlignedTime.Add(timeSpan).After(end) { - lastAlignedTime = lastAlignedTime.Add(timeSpan) - } - var lastIsZero bool - if !lastAlignedTime.Equal(lastPoint) { - data = append(data, &model.ChartData{ - Timestamp: lastAlignedTime.Unix(), - }) - lastIsZero = true - } - - result := make([]*model.ChartData, 0, len(data)) - result = append(result, data[0]) - - for i := 1; i < len(data); i++ { - curr := data[i] - prev := data[i-1] - hourDiff := (curr.Timestamp - prev.Timestamp) / int64(timeSpan.Seconds()) - - // If gap is 1 hour or less, continue - if hourDiff <= 1 { - result = append(result, curr) - continue - } - - // If gap is more than 3 hours, only add boundary points - if hourDiff > 3 { - // Add point for hour after prev - if i != 1 || (i == 1 && !firstIsZero) { - result = append(result, &model.ChartData{ - Timestamp: prev.Timestamp + int64(timeSpan.Seconds()), - }) - } - // Add point for hour before curr - if i != len(data)-1 || (i == len(data)-1 && !lastIsZero) { - result = append(result, &model.ChartData{ - Timestamp: curr.Timestamp - int64(timeSpan.Seconds()), - }) - } - result = append(result, curr) - continue - } - - // Fill gaps of 2-3 hours with zero points - for j := prev.Timestamp + int64(timeSpan.Seconds()); j < curr.Timestamp; j += int64(timeSpan.Seconds()) { - result = append(result, &model.ChartData{ - Timestamp: j, - }) - } - result = append(result, curr) - } - - return result -} - -func GetDashboard(c *gin.Context) { - log := middleware.GetLogger(c) - - start, end, timeSpan := getDashboardTime(c.Query("type")) - modelName := c.Query("model") - - dashboards, err := model.GetDashboardData(start, end, modelName, timeSpan) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - dashboards.ChartData = fillGaps(dashboards.ChartData, start, end, timeSpan) - - if common.RedisEnabled { - rpm, err := rpmlimit.GetRPM(c.Request.Context(), "", modelName) - if err != nil { - log.Errorf("failed to get rpm: %v", err) - } else { - dashboards.RPM = rpm - } - } - - middleware.SuccessResponse(c, dashboards) -} - -func GetGroupDashboard(c *gin.Context) { - log := middleware.GetLogger(c) - - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - - start, end, timeSpan := getDashboardTime(c.Query("type")) - tokenName := c.Query("token_name") - modelName := c.Query("model") - - dashboards, err := model.GetGroupDashboardData(group, start, end, tokenName, modelName, timeSpan) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "failed to get statistics") - return - } - - dashboards.ChartData = fillGaps(dashboards.ChartData, start, end, timeSpan) - - if common.RedisEnabled && tokenName == "" { - rpm, err := rpmlimit.GetRPM(c.Request.Context(), group, modelName) - if err != nil { - log.Errorf("failed to get rpm: %v", err) - } else { - dashboards.RPM = rpm - } - } - - middleware.SuccessResponse(c, dashboards) -} - -func GetGroupDashboardModels(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - groupCache, err := model.CacheGetGroup(group) - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - middleware.SuccessResponse(c, model.LoadModelCaches().EnabledModelConfigs) - } else { - middleware.ErrorResponse(c, http.StatusOK, fmt.Sprintf("failed to get group: %v", err)) - } - return - } - - enabledModelConfigs := model.LoadModelCaches().EnabledModelConfigs - newEnabledModelConfigs := make([]*model.ModelConfig, len(enabledModelConfigs)) - for i, mc := range enabledModelConfigs { - newEnabledModelConfigs[i] = middleware.GetGroupAdjustedModelConfig(groupCache, mc) - } - middleware.SuccessResponse(c, newEnabledModelConfigs) -} - -func GetModelCostRank(c *gin.Context) { - startTime, endTime := parseTimeRange(c) - models, err := model.GetModelCostRank("", startTime, endTime) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, models) -} - -func GetGroupModelCostRank(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - startTime, endTime := parseTimeRange(c) - models, err := model.GetModelCostRank(group, startTime, endTime) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, models) -} diff --git a/service/aiproxy/controller/group.go b/service/aiproxy/controller/group.go deleted file mode 100644 index 4ab8409c6..000000000 --- a/service/aiproxy/controller/group.go +++ /dev/null @@ -1,301 +0,0 @@ -package controller - -import ( - "net/http" - "strconv" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" -) - -type GroupResponse struct { - *model.Group - AccessedAt time.Time `json:"accessed_at,omitempty"` -} - -func (g *GroupResponse) MarshalJSON() ([]byte, error) { - type Alias model.Group - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at,omitempty"` - AccessedAt int64 `json:"accessed_at,omitempty"` - }{ - Alias: (*Alias)(g.Group), - CreatedAt: g.CreatedAt.UnixMilli(), - AccessedAt: g.AccessedAt.UnixMilli(), - }) -} - -func GetGroups(c *gin.Context) { - page, perPage := parsePageParams(c) - order := c.DefaultQuery("order", "") - groups, total, err := model.GetGroups(page, perPage, order, false) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - groupResponses := make([]*GroupResponse, len(groups)) - for i, group := range groups { - lastRequestAt, _ := model.GetGroupLastRequestTime(group.ID) - groupResponses[i] = &GroupResponse{ - Group: group, - AccessedAt: lastRequestAt, - } - } - middleware.SuccessResponse(c, gin.H{ - "groups": groupResponses, - "total": total, - }) -} - -func SearchGroups(c *gin.Context) { - keyword := c.Query("keyword") - page, perPage := parsePageParams(c) - order := c.DefaultQuery("order", "") - status, _ := strconv.Atoi(c.Query("status")) - groups, total, err := model.SearchGroup(keyword, page, perPage, order, status) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - groupResponses := make([]*GroupResponse, len(groups)) - for i, group := range groups { - lastRequestAt, _ := model.GetGroupLastRequestTime(group.ID) - groupResponses[i] = &GroupResponse{ - Group: group, - AccessedAt: lastRequestAt, - } - } - middleware.SuccessResponse(c, gin.H{ - "groups": groupResponses, - "total": total, - }) -} - -func GetGroup(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group id is empty") - return - } - _group, err := model.GetGroupByID(group) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - lastRequestAt, _ := model.GetGroupLastRequestTime(group) - groupResponse := &GroupResponse{ - Group: _group, - AccessedAt: lastRequestAt, - } - middleware.SuccessResponse(c, groupResponse) -} - -type UpdateGroupRPMRatioRequest struct { - RPMRatio float64 `json:"rpm_ratio"` -} - -func UpdateGroupRPMRatio(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := UpdateGroupRPMRatioRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err = model.UpdateGroupRPMRatio(group, req.RPMRatio) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -type UpdateGroupRPMRequest struct { - RPM map[string]int64 `json:"rpm"` -} - -func UpdateGroupRPM(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := UpdateGroupRPMRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err = model.UpdateGroupRPM(group, req.RPM) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -type UpdateGroupTPMRequest struct { - TPM map[string]int64 `json:"tpm"` -} - -func UpdateGroupTPM(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := UpdateGroupTPMRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err = model.UpdateGroupTPM(group, req.TPM) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -type UpdateGroupTPMRatioRequest struct { - TPMRatio float64 `json:"tpm_ratio"` -} - -func UpdateGroupTPMRatio(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := UpdateGroupTPMRatioRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err = model.UpdateGroupTPMRatio(group, req.TPMRatio) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -type UpdateGroupStatusRequest struct { - Status int `json:"status"` -} - -func UpdateGroupStatus(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := UpdateGroupStatusRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err = model.UpdateGroupStatus(group, req.Status) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func DeleteGroup(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err := model.DeleteGroupByID(group) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func DeleteGroups(c *gin.Context) { - ids := []string{} - err := c.ShouldBindJSON(&ids) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.DeleteGroupsByIDs(ids) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -type CreateGroupRequest struct { - RPM map[string]int64 `json:"rpm"` - RPMRatio float64 `json:"rpm_ratio"` - TPM map[string]int64 `json:"tpm"` - TPMRatio float64 `json:"tpm_ratio"` -} - -func CreateGroup(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := CreateGroupRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - if err := model.CreateGroup(&model.Group{ - ID: group, - RPMRatio: req.RPMRatio, - RPM: req.RPM, - TPMRatio: req.TPMRatio, - TPM: req.TPM, - }); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func UpdateGroup(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - req := CreateGroupRequest{} - err := sonic.ConfigDefault.NewDecoder(c.Request.Body).Decode(&req) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid parameter") - return - } - err = model.UpdateGroup(group, &model.Group{ - RPMRatio: req.RPMRatio, - RPM: req.RPM, - TPMRatio: req.TPMRatio, - TPM: req.TPM, - }) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} diff --git a/service/aiproxy/controller/import.go b/service/aiproxy/controller/import.go deleted file mode 100644 index ef1c19f08..000000000 --- a/service/aiproxy/controller/import.go +++ /dev/null @@ -1,210 +0,0 @@ -package controller - -import ( - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "gorm.io/gorm" -) - -type OneAPIChannel struct { - Type int `gorm:"default:0" json:"type"` - Key string `gorm:"type:text" json:"key"` - Status int `gorm:"default:1" json:"status"` - Name string `gorm:"index" json:"name"` - BaseURL string `gorm:"column:base_url;default:''"` - Models string `json:"models"` - ModelMapping map[string]string `gorm:"type:varchar(1024);serializer:fastjson"` - Priority int32 `gorm:"bigint;default:0"` - Config ChannelConfig `gorm:"serializer:fastjson"` -} - -func (c *OneAPIChannel) TableName() string { - return "channels" -} - -type ChannelConfig struct { - Region string `json:"region,omitempty"` - SK string `json:"sk,omitempty"` - AK string `json:"ak,omitempty"` - UserID string `json:"user_id,omitempty"` - APIVersion string `json:"api_version,omitempty"` - LibraryID string `json:"library_id,omitempty"` - VertexAIProjectID string `json:"vertex_ai_project_id,omitempty"` - VertexAIADC string `json:"vertex_ai_adc,omitempty"` -} - -// https://github.com/songquanpeng/one-api/blob/main/relay/channeltype/define.go -const ( - OneAPIOpenAI = iota + 1 - OneAPIAPI2D - OneAPIAzure - OneAPICloseAI - OneAPIOpenAISB - OneAPIOpenAIMax - OneAPIOhMyGPT - OneAPICustom - OneAPIAils - OneAPIAIProxy - OneAPIPaLM - OneAPIAPI2GPT - OneAPIAIGC2D - OneAPIAnthropic - OneAPIBaidu - OneAPIZhipu - OneAPIAli - OneAPIXunfei - OneAPIAI360 - OneAPIOpenRouter - OneAPIAIProxyLibrary - OneAPIFastGPT - OneAPITencent - OneAPIGemini - OneAPIMoonshot - OneAPIBaichuan - OneAPIMinimax - OneAPIMistral - OneAPIGroq - OneAPIOllama - OneAPILingYiWanWu - OneAPIStepFun - OneAPIAwsClaude - OneAPICoze - OneAPICohere - OneAPIDeepSeek - OneAPICloudflare - OneAPIDeepL - OneAPITogetherAI - OneAPIDoubao - OneAPINovita - OneAPIVertextAI - OneAPIProxy - OneAPISiliconFlow - OneAPIXAI - OneAPIReplicate - OneAPIBaiduV2 - OneAPIXunfeiV2 - OneAPIAliBailian - OneAPIOpenAICompatible - OneAPIGeminiOpenAICompatible -) - -// relay/channeltype/define.go - -var OneAPIChannelType2AIProxyMap = map[int]int{ - OneAPIOpenAI: 1, - OneAPIAzure: 3, - OneAPIAnthropic: 14, - OneAPIBaidu: 15, - OneAPIZhipu: 16, - OneAPIAli: 17, - OneAPIAI360: 19, - OneAPITencent: 23, - OneAPIGemini: 24, - OneAPIMoonshot: 25, - OneAPIBaichuan: 26, - OneAPIMinimax: 27, - OneAPIMistral: 28, - OneAPIGroq: 29, - OneAPIOllama: 30, - OneAPILingYiWanWu: 31, - OneAPIStepFun: 32, - OneAPIAwsClaude: 33, - OneAPICoze: 34, - OneAPICohere: 35, - OneAPIDeepSeek: 36, - OneAPICloudflare: 37, - OneAPIDoubao: 40, - OneAPINovita: 41, - OneAPIVertextAI: 42, - OneAPISiliconFlow: 43, - OneAPIBaiduV2: 13, - OneAPIXunfeiV2: 18, - OneAPIAliBailian: 17, - OneAPIGeminiOpenAICompatible: 12, - OneAPIXAI: 45, -} - -type ImportChannelFromOneAPIRequest struct { - DSN string `json:"dsn"` -} - -func AddOneAPIChannel(ch OneAPIChannel) error { - add := AddChannelRequest{ - Type: ch.Type, - Name: ch.Name, - Key: ch.Key, - BaseURL: ch.BaseURL, - Models: strings.Split(ch.Models, ","), - ModelMapping: ch.ModelMapping, - Priority: ch.Priority, - Status: ch.Status, - } - if t, ok := OneAPIChannelType2AIProxyMap[ch.Type]; ok { - add.Type = t - } else { - add.Type = 1 - } - if add.Type == 1 && add.BaseURL != "" { - add.BaseURL += "/v1" - } - chs, err := add.ToChannels() - if err != nil { - return err - } - return model.BatchInsertChannels(chs) -} - -func ImportChannelFromOneAPI(c *gin.Context) { - var req ImportChannelFromOneAPIRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusBadRequest, err.Error()) - return - } - - if req.DSN == "" { - middleware.ErrorResponse(c, http.StatusBadRequest, "sql dsn is required") - return - } - - var db *gorm.DB - var err error - if strings.HasPrefix(req.DSN, "mysql") { - db, err = model.OpenMySQL(req.DSN) - } else if strings.HasPrefix(req.DSN, "postgres") { - db, err = model.OpenPostgreSQL(req.DSN) - } else { - middleware.ErrorResponse(c, http.StatusBadRequest, "invalid dsn, only mysql and postgres are supported") - return - } - if err != nil { - middleware.ErrorResponse(c, http.StatusBadRequest, err.Error()) - return - } - sqlDB, err := db.DB() - if err != nil { - middleware.ErrorResponse(c, http.StatusBadRequest, err.Error()) - return - } - defer sqlDB.Close() - - allChannels := make([]*OneAPIChannel, 0) - err = db.Model(&OneAPIChannel{}).Find(&allChannels).Error - if err != nil { - middleware.ErrorResponse(c, http.StatusBadRequest, err.Error()) - return - } - - errs := make([]error, 0) - for _, ch := range allChannels { - err := AddOneAPIChannel(*ch) - if err != nil { - errs = append(errs, err) - } - } - - middleware.SuccessResponse(c, errs) -} diff --git a/service/aiproxy/controller/log.go b/service/aiproxy/controller/log.go deleted file mode 100644 index 6f98a3b1d..000000000 --- a/service/aiproxy/controller/log.go +++ /dev/null @@ -1,319 +0,0 @@ -package controller - -import ( - "net/http" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" -) - -func parseTimeRange(c *gin.Context) (startTime, endTime time.Time) { - startTimestamp, _ := strconv.ParseInt(c.Query("start_timestamp"), 10, 64) - endTimestamp, _ := strconv.ParseInt(c.Query("end_timestamp"), 10, 64) - - if startTimestamp != 0 { - startTime = time.UnixMilli(startTimestamp) - } - sevenDaysAgo := time.Now().AddDate(0, 0, -7) - if startTime.IsZero() || startTime.Before(sevenDaysAgo) { - startTime = sevenDaysAgo - } - - if endTimestamp != 0 { - endTime = time.UnixMilli(endTimestamp) - } - return -} - -func parseCommonParams(c *gin.Context) (params struct { - tokenName string - modelName string - channelID int - endpoint string - tokenID int - order string - requestID string - mode int - codeType string - withBody bool - ip string -}, -) { - params.tokenName = c.Query("token_name") - params.modelName = c.Query("model_name") - params.channelID, _ = strconv.Atoi(c.Query("channel")) - params.endpoint = c.Query("endpoint") - params.tokenID, _ = strconv.Atoi(c.Query("token_id")) - params.order = c.Query("order") - params.requestID = c.Query("request_id") - params.mode, _ = strconv.Atoi(c.Query("mode")) - params.codeType = c.Query("code_type") - params.withBody, _ = strconv.ParseBool(c.Query("with_body")) - params.ip = c.Query("ip") - return -} - -// Handler functions -func GetLogs(c *gin.Context) { - page, perPage := parsePageParams(c) - startTime, endTime := parseTimeRange(c) - params := parseCommonParams(c) - group := c.Query("group") - - result, err := model.GetLogs( - group, - startTime, - endTime, - params.modelName, - params.requestID, - params.tokenID, - params.tokenName, - params.channelID, - params.endpoint, - params.order, - params.mode, - model.CodeType(params.codeType), - params.withBody, - params.ip, - page, - perPage, - ) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, result) -} - -func GetGroupLogs(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - - page, perPage := parsePageParams(c) - startTime, endTime := parseTimeRange(c) - params := parseCommonParams(c) - - result, err := model.GetGroupLogs( - group, - startTime, - endTime, - params.modelName, - params.requestID, - params.tokenID, - params.tokenName, - params.channelID, - params.endpoint, - params.order, - params.mode, - model.CodeType(params.codeType), - params.withBody, - params.ip, - page, - perPage, - ) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, result) -} - -func SearchLogs(c *gin.Context) { - page, perPage := parsePageParams(c) - startTime, endTime := parseTimeRange(c) - params := parseCommonParams(c) - - keyword := c.Query("keyword") - group := c.Query("group_id") - - result, err := model.SearchLogs( - group, - keyword, - params.endpoint, - params.requestID, - params.tokenID, - params.tokenName, - params.modelName, - startTime, - endTime, - params.channelID, - params.order, - params.mode, - model.CodeType(params.codeType), - params.withBody, - params.ip, - page, - perPage, - ) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, result) -} - -func SearchGroupLogs(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - - page, perPage := parsePageParams(c) - startTime, endTime := parseTimeRange(c) - params := parseCommonParams(c) - keyword := c.Query("keyword") - - result, err := model.SearchGroupLogs( - group, - keyword, - params.endpoint, - params.requestID, - params.tokenID, - params.tokenName, - params.modelName, - startTime, - endTime, - params.channelID, - params.order, - params.mode, - model.CodeType(params.codeType), - params.withBody, - params.ip, - page, - perPage, - ) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, result) -} - -func GetLogDetail(c *gin.Context) { - logID, _ := strconv.Atoi(c.Param("log_id")) - log, err := model.GetLogDetail(logID) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, log) -} - -func GetUsedModels(c *gin.Context) { - startTime, endTime := parseTimeRange(c) - models, err := model.GetUsedModels("", startTime, endTime) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, models) -} - -func GetGroupLogDetail(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - logID, _ := strconv.Atoi(c.Param("log_id")) - log, err := model.GetGroupLogDetail(logID, group) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, log) -} - -func GetGroupUsedModels(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - startTime, endTime := parseTimeRange(c) - models, err := model.GetUsedModels(group, startTime, endTime) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, models) -} - -func GetGroupUsedTokenNames(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - startTime, endTime := parseTimeRange(c) - tokenNames, err := model.GetUsedTokenNames(group, startTime, endTime) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, tokenNames) -} - -func DeleteHistoryLogs(c *gin.Context) { - timestamp, _ := strconv.ParseInt(c.Query("timestamp"), 10, 64) - if timestamp == 0 { - middleware.ErrorResponse(c, http.StatusOK, "timestamp is required") - return - } - count, err := model.DeleteOldLog(time.UnixMilli(timestamp)) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, count) -} - -func SearchConsumeError(c *gin.Context) { - keyword := c.Query("keyword") - group := c.Query("group") - tokenName := c.Query("token_name") - modelName := c.Query("model_name") - content := c.Query("content") - tokenID, _ := strconv.Atoi(c.Query("token_id")) - usedAmount, _ := strconv.ParseFloat(c.Query("used_amount"), 64) - - page, _ := strconv.Atoi(c.Query("page")) - perPage, _ := strconv.Atoi(c.Query("per_page")) - if perPage <= 0 { - perPage = 10 - } else if perPage > 100 { - perPage = 100 - } - - order := c.Query("order") - requestID := c.Query("request_id") - - logs, total, err := model.SearchConsumeError( - keyword, - requestID, - group, - tokenName, - modelName, - content, - usedAmount, - tokenID, - page, - perPage, - order, - ) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, gin.H{ - "logs": logs, - "total": total, - }) -} diff --git a/service/aiproxy/controller/misc.go b/service/aiproxy/controller/misc.go deleted file mode 100644 index 21ed6f03b..000000000 --- a/service/aiproxy/controller/misc.go +++ /dev/null @@ -1,17 +0,0 @@ -package controller - -import ( - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/middleware" -) - -type StatusData struct { - StartTime int64 `json:"startTime"` -} - -func GetStatus(c *gin.Context) { - middleware.SuccessResponse(c, &StatusData{ - StartTime: common.StartTime, - }) -} diff --git a/service/aiproxy/controller/model.go b/service/aiproxy/controller/model.go deleted file mode 100644 index 4c1ce66aa..000000000 --- a/service/aiproxy/controller/model.go +++ /dev/null @@ -1,249 +0,0 @@ -package controller - -import ( - "fmt" - "net/http" - "slices" - "sort" - "strconv" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/channeltype" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - log "github.com/sirupsen/logrus" -) - -// https://platform.openai.com/docs/api-reference/models/list - -type OpenAIModelPermission struct { - Group *string `json:"group"` - ID string `json:"id"` - Object string `json:"object"` - Organization string `json:"organization"` - Created int `json:"created"` - AllowCreateEngine bool `json:"allow_create_engine"` - AllowSampling bool `json:"allow_sampling"` - AllowLogprobs bool `json:"allow_logprobs"` - AllowSearchIndices bool `json:"allow_search_indices"` - AllowView bool `json:"allow_view"` - AllowFineTuning bool `json:"allow_fine_tuning"` - IsBlocking bool `json:"is_blocking"` -} - -type OpenAIModels struct { - Parent *string `json:"parent"` - ID string `json:"id"` - Object string `json:"object"` - OwnedBy string `json:"owned_by"` - Root string `json:"root"` - Permission []OpenAIModelPermission `json:"permission"` - Created int `json:"created"` -} - -type BuiltinModelConfig model.ModelConfig - -func (c *BuiltinModelConfig) MarshalJSON() ([]byte, error) { - type Alias BuiltinModelConfig - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at,omitempty"` - UpdatedAt int64 `json:"updated_at,omitempty"` - }{ - Alias: (*Alias)(c), - }) -} - -func SortBuiltinModelConfigsFunc(i, j *BuiltinModelConfig) int { - return model.SortModelConfigsFunc((*model.ModelConfig)(i), (*model.ModelConfig)(j)) -} - -var ( - builtinModels []*BuiltinModelConfig - builtinModelsMap map[string]*OpenAIModels - builtinChannelID2Models map[int][]*BuiltinModelConfig -) - -var permission = []OpenAIModelPermission{ - { - ID: "modelperm-LwHkVFn8AcMItP432fKKDIKJ", - Object: "model_permission", - Created: 1626777600, - AllowCreateEngine: true, - AllowSampling: true, - AllowLogprobs: true, - AllowSearchIndices: false, - AllowView: true, - AllowFineTuning: false, - Organization: "*", - Group: nil, - IsBlocking: false, - }, -} - -func init() { - builtinChannelID2Models = make(map[int][]*BuiltinModelConfig) - builtinModelsMap = make(map[string]*OpenAIModels) - // https://platform.openai.com/docs/models/model-endpoint-compatibility - for i, adaptor := range channeltype.ChannelAdaptor { - modelNames := adaptor.GetModelList() - builtinChannelID2Models[i] = make([]*BuiltinModelConfig, len(modelNames)) - for idx, _model := range modelNames { - if _model.Owner == "" { - _model.Owner = model.ModelOwner(adaptor.GetChannelName()) - } - if v, ok := builtinModelsMap[_model.Model]; !ok { - builtinModelsMap[_model.Model] = &OpenAIModels{ - ID: _model.Model, - Object: "model", - Created: 1626777600, - OwnedBy: string(_model.Owner), - Permission: permission, - Root: _model.Model, - Parent: nil, - } - builtinModels = append(builtinModels, (*BuiltinModelConfig)(_model)) - } else if v.OwnedBy != string(_model.Owner) { - log.Fatalf("model %s owner mismatch, expect %s, actual %s", _model.Model, string(_model.Owner), v.OwnedBy) - } - builtinChannelID2Models[i][idx] = (*BuiltinModelConfig)(_model) - } - } - for _, models := range builtinChannelID2Models { - sort.Slice(models, func(i, j int) bool { - return models[i].Model < models[j].Model - }) - slices.SortStableFunc(models, SortBuiltinModelConfigsFunc) - } - slices.SortStableFunc(builtinModels, SortBuiltinModelConfigsFunc) -} - -func BuiltinModels(c *gin.Context) { - middleware.SuccessResponse(c, builtinModels) -} - -func ChannelBuiltinModels(c *gin.Context) { - middleware.SuccessResponse(c, builtinChannelID2Models) -} - -func ChannelBuiltinModelsByType(c *gin.Context) { - channelType := c.Param("type") - if channelType == "" { - middleware.ErrorResponse(c, http.StatusOK, "type is required") - return - } - channelTypeInt, err := strconv.Atoi(channelType) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid type") - return - } - middleware.SuccessResponse(c, builtinChannelID2Models[channelTypeInt]) -} - -func ChannelDefaultModelsAndMapping(c *gin.Context) { - middleware.SuccessResponse(c, gin.H{ - "models": config.GetDefaultChannelModels(), - "mapping": config.GetDefaultChannelModelMapping(), - }) -} - -func ChannelDefaultModelsAndMappingByType(c *gin.Context) { - channelType := c.Param("type") - if channelType == "" { - middleware.ErrorResponse(c, http.StatusOK, "type is required") - return - } - channelTypeInt, err := strconv.Atoi(channelType) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid type") - return - } - middleware.SuccessResponse(c, gin.H{ - "models": config.GetDefaultChannelModels()[channelTypeInt], - "mapping": config.GetDefaultChannelModelMapping()[channelTypeInt], - }) -} - -func EnabledModels(c *gin.Context) { - middleware.SuccessResponse(c, model.LoadModelCaches().EnabledModelConfigs) -} - -func ChannelEnabledModels(c *gin.Context) { - middleware.SuccessResponse(c, model.LoadModelCaches().EnabledChannelType2ModelConfigs) -} - -func ChannelEnabledModelsByType(c *gin.Context) { - channelTypeStr := c.Param("type") - if channelTypeStr == "" { - middleware.ErrorResponse(c, http.StatusOK, "type is required") - return - } - channelTypeInt, err := strconv.Atoi(channelTypeStr) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "invalid type") - return - } - middleware.SuccessResponse(c, model.LoadModelCaches().EnabledChannelType2ModelConfigs[channelTypeInt]) -} - -func ListModels(c *gin.Context) { - enabledModelConfigsMap := middleware.GetModelCaches(c).EnabledModelConfigsMap - token := middleware.GetToken(c) - - availableOpenAIModels := make([]*OpenAIModels, 0, len(token.Models)) - - for _, model := range token.Models { - if mc, ok := enabledModelConfigsMap[model]; ok { - availableOpenAIModels = append(availableOpenAIModels, &OpenAIModels{ - ID: model, - Object: "model", - Created: 1626777600, - OwnedBy: string(mc.Owner), - Root: model, - Permission: permission, - Parent: nil, - }) - } - } - - c.JSON(http.StatusOK, gin.H{ - "object": "list", - "data": availableOpenAIModels, - }) -} - -func RetrieveModel(c *gin.Context) { - modelName := c.Param("model") - enabledModelConfigsMap := middleware.GetModelCaches(c).EnabledModelConfigsMap - - mc, ok := enabledModelConfigsMap[modelName] - if ok { - token := middleware.GetToken(c) - ok = slices.Contains(token.Models, modelName) - } - - if !ok { - c.JSON(200, gin.H{ - "error": &relaymodel.Error{ - Message: fmt.Sprintf("the model '%s' does not exist", modelName), - Type: "invalid_request_error", - Param: "model", - Code: "model_not_found", - }, - }) - return - } - - c.JSON(200, &OpenAIModels{ - ID: modelName, - Object: "model", - Created: 1626777600, - OwnedBy: string(mc.Owner), - Root: modelName, - Permission: permission, - Parent: nil, - }) -} diff --git a/service/aiproxy/controller/modelconfig.go b/service/aiproxy/controller/modelconfig.go deleted file mode 100644 index 32ba6a2fd..000000000 --- a/service/aiproxy/controller/modelconfig.go +++ /dev/null @@ -1,140 +0,0 @@ -package controller - -import ( - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" -) - -func GetModelConfigs(c *gin.Context) { - page, perPage := parsePageParams(c) - _model := c.Query("model") - configs, total, err := model.GetModelConfigs(page, perPage, _model) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, gin.H{ - "configs": configs, - "total": total, - }) -} - -func GetAllModelConfigs(c *gin.Context) { - configs, err := model.GetAllModelConfigs() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, configs) -} - -type GetModelConfigsByModelsContainsRequest struct { - Models []string `json:"models"` -} - -func GetModelConfigsByModelsContains(c *gin.Context) { - request := GetModelConfigsByModelsContainsRequest{} - err := c.ShouldBindJSON(&request) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - configs, err := model.GetModelConfigsByModels(request.Models) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, configs) -} - -func SearchModelConfigs(c *gin.Context) { - keyword := c.Query("keyword") - page, perPage := parsePageParams(c) - _model := c.Query("model") - owner := c.Query("owner") - configs, total, err := model.SearchModelConfigs(keyword, page, perPage, _model, model.ModelOwner(owner)) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, gin.H{ - "configs": configs, - "total": total, - }) -} - -type SaveModelConfigsRequest struct { - CreatedAt int64 `json:"created_at"` - UpdatedAt int64 `json:"updated_at"` - *model.ModelConfig -} - -func SaveModelConfigs(c *gin.Context) { - var configs []*SaveModelConfigsRequest - if err := c.ShouldBindJSON(&configs); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - modelConfigs := make([]*model.ModelConfig, len(configs)) - for i, config := range configs { - modelConfigs[i] = config.ModelConfig - } - err := model.SaveModelConfigs(modelConfigs) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func SaveModelConfig(c *gin.Context) { - var config SaveModelConfigsRequest - if err := c.ShouldBindJSON(&config); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err := model.SaveModelConfig(config.ModelConfig) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func DeleteModelConfig(c *gin.Context) { - _model := c.Param("model") - err := model.DeleteModelConfig(_model) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func DeleteModelConfigs(c *gin.Context) { - models := []string{} - err := c.ShouldBindJSON(&models) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.DeleteModelConfigsByModels(models) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func GetModelConfig(c *gin.Context) { - _model := c.Param("model") - config, err := model.GetModelConfig(_model) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, config) -} diff --git a/service/aiproxy/controller/monitor.go b/service/aiproxy/controller/monitor.go deleted file mode 100644 index 4b7d370aa..000000000 --- a/service/aiproxy/controller/monitor.go +++ /dev/null @@ -1,92 +0,0 @@ -package controller - -import ( - "net/http" - "strconv" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/monitor" -) - -func GetAllChannelModelErrorRates(c *gin.Context) { - rates, err := monitor.GetAllChannelModelErrorRates(c.Request.Context()) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.JSON(http.StatusOK, rates) -} - -func GetChannelModelErrorRates(c *gin.Context) { - channelID := c.Param("id") - channelIDInt, err := strconv.ParseInt(channelID, 10, 64) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "Invalid channel ID") - return - } - rates, err := monitor.GetChannelModelErrorRates(c.Request.Context(), channelIDInt) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.JSON(http.StatusOK, rates) -} - -func ClearAllModelErrors(c *gin.Context) { - err := monitor.ClearAllModelErrors(c.Request.Context()) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.Status(http.StatusNoContent) -} - -func ClearChannelAllModelErrors(c *gin.Context) { - channelID := c.Param("id") - channelIDInt, err := strconv.ParseInt(channelID, 10, 64) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "Invalid channel ID") - return - } - err = monitor.ClearChannelAllModelErrors(c.Request.Context(), int(channelIDInt)) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.Status(http.StatusNoContent) -} - -func ClearChannelModelErrors(c *gin.Context) { - channelID := c.Param("id") - model := c.Param("model") - channelIDInt, err := strconv.ParseInt(channelID, 10, 64) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, "Invalid channel ID") - return - } - err = monitor.ClearChannelModelErrors(c.Request.Context(), model, int(channelIDInt)) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.Status(http.StatusNoContent) -} - -func GetModelsErrorRate(c *gin.Context) { - rates, err := monitor.GetModelsErrorRate(c.Request.Context()) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.JSON(http.StatusOK, rates) -} - -func GetAllBannedModelChannels(c *gin.Context) { - channels, err := monitor.GetAllBannedModelChannels(c.Request.Context()) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - c.JSON(http.StatusOK, channels) -} diff --git a/service/aiproxy/controller/option.go b/service/aiproxy/controller/option.go deleted file mode 100644 index 7de083dab..000000000 --- a/service/aiproxy/controller/option.go +++ /dev/null @@ -1,66 +0,0 @@ -package controller - -import ( - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" -) - -func GetOptions(c *gin.Context) { - dbOptions, err := model.GetAllOption() - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - options := make(map[string]string, len(dbOptions)) - for _, option := range dbOptions { - options[option.Key] = option.Value - } - middleware.SuccessResponse(c, options) -} - -func GetOption(c *gin.Context) { - key := c.Param("key") - if key == "" { - middleware.ErrorResponse(c, http.StatusOK, "key is required") - return - } - option, err := model.GetOption(key) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, option) -} - -func UpdateOption(c *gin.Context) { - var option model.Option - err := c.BindJSON(&option) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.UpdateOption(option.Key, option.Value) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} - -func UpdateOptions(c *gin.Context) { - var options map[string]string - err := c.BindJSON(&options) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - err = model.UpdateOptions(options) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - middleware.SuccessResponse(c, nil) -} diff --git a/service/aiproxy/controller/relay.go b/service/aiproxy/controller/relay.go deleted file mode 100644 index 8d4f255d7..000000000 --- a/service/aiproxy/controller/relay.go +++ /dev/null @@ -1,451 +0,0 @@ -package controller - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "math/rand/v2" - "net/http" - "slices" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/labring/sealos/service/aiproxy/middleware" - dbmodel "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/monitor" - "github.com/labring/sealos/service/aiproxy/relay/controller" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - log "github.com/sirupsen/logrus" -) - -// https://platform.openai.com/docs/api-reference/chat - -type RelayController func(*meta.Meta, *gin.Context) *model.ErrorWithStatusCode - -func relayController(mode relaymode.Mode) (RelayController, bool) { - var relayController RelayController - switch mode { - case relaymode.ImagesGenerations, - relaymode.Edits: - relayController = controller.RelayImageHelper - case relaymode.AudioSpeech: - relayController = controller.RelayTTSHelper - case relaymode.AudioTranslation, - relaymode.AudioTranscription: - relayController = controller.RelaySTTHelper - case relaymode.ParsePdf: - relayController = controller.RelayParsePdfHelper - case relaymode.Rerank: - relayController = controller.RerankHelper - case relaymode.ChatCompletions, - relaymode.Embeddings, - relaymode.Completions, - relaymode.Moderations: - relayController = controller.RelayTextHelper - default: - return nil, false - } - return func(meta *meta.Meta, c *gin.Context) *model.ErrorWithStatusCode { - log := middleware.GetLogger(c) - middleware.SetLogFieldsFromMeta(meta, log.Data) - return relayController(meta, c) - }, true -} - -func RelayHelper(meta *meta.Meta, c *gin.Context, relayController RelayController) (*model.ErrorWithStatusCode, bool) { - relayErr := relayController(meta, c) - if relayErr == nil { - if _, _, err := monitor.AddRequest( - context.Background(), - meta.OriginModel, - int64(meta.Channel.ID), - false, - false, - ); err != nil { - log.Errorf("add request failed: %+v", err) - } - return nil, false - } - if shouldErrorMonitor(relayErr.StatusCode) { - hasPermission := channelHasPermission(relayErr.StatusCode) - beyondThreshold, autoBanned, err := monitor.AddRequest( - context.Background(), - meta.OriginModel, - int64(meta.Channel.ID), - true, - !hasPermission, - ) - if err != nil { - log.Errorf("add request failed: %+v", err) - } - if autoBanned { - notify.ErrorThrottle( - fmt.Sprintf("autoBanned:%d:%s", meta.Channel.ID, meta.OriginModel), - time.Minute, - fmt.Sprintf("channel[%d] %s(%d) model %s is auto banned", - meta.Channel.Type, meta.Channel.Name, meta.Channel.ID, meta.OriginModel), - relayErr.JSONOrEmpty(), - ) - } else if beyondThreshold { - notify.WarnThrottle( - fmt.Sprintf("beyondThreshold:%d:%s", meta.Channel.ID, meta.OriginModel), - time.Minute, - fmt.Sprintf("channel[%d] %s(%d) model %s error rate is beyond threshold", - meta.Channel.Type, meta.Channel.Name, meta.Channel.ID, meta.OriginModel), - relayErr.JSONOrEmpty(), - ) - } else if !hasPermission { - notify.ErrorThrottle( - fmt.Sprintf("channelHasPermission:%d:%s", meta.Channel.ID, meta.OriginModel), - time.Minute, - fmt.Sprintf("channel[%d] %s(%d) model %s has no permission", - meta.Channel.Type, meta.Channel.Name, meta.Channel.ID, meta.OriginModel), - relayErr.JSONOrEmpty(), - ) - } - } - return relayErr, shouldRetry(c, relayErr.StatusCode) -} - -func filterChannels(channels []*dbmodel.Channel, ignoreChannel ...int64) []*dbmodel.Channel { - filtered := make([]*dbmodel.Channel, 0) - for _, channel := range channels { - if channel.Status != dbmodel.ChannelStatusEnabled { - continue - } - if slices.Contains(ignoreChannel, int64(channel.ID)) { - continue - } - filtered = append(filtered, channel) - } - return filtered -} - -var ( - ErrChannelsNotFound = errors.New("channels not found") - ErrChannelsExhausted = errors.New("channels exhausted") -) - -func GetRandomChannel(c *dbmodel.ModelCaches, model string, errorRates map[int64]float64, ignoreChannel ...int64) (*dbmodel.Channel, error) { - return getRandomChannel(c.EnabledModel2channels[model], errorRates, ignoreChannel...) -} - -func getPriority(channel *dbmodel.Channel, errorRate float64) int32 { - priority := channel.GetPriority() - if errorRate > 1 { - errorRate = 1 - } else if errorRate < 0.1 { - errorRate = 0.1 - } - return int32(float64(priority) / errorRate) -} - -//nolint:gosec -func getRandomChannel(channels []*dbmodel.Channel, errorRates map[int64]float64, ignoreChannel ...int64) (*dbmodel.Channel, error) { - if len(channels) == 0 { - return nil, ErrChannelsNotFound - } - - channels = filterChannels(channels, ignoreChannel...) - if len(channels) == 0 { - return nil, ErrChannelsExhausted - } - - if len(channels) == 1 { - return channels[0], nil - } - - var totalWeight int32 - cachedPrioritys := make([]int32, len(channels)) - for i, ch := range channels { - priority := getPriority(ch, errorRates[int64(ch.ID)]) - totalWeight += priority - cachedPrioritys[i] = priority - } - - if totalWeight == 0 { - return channels[rand.IntN(len(channels))], nil - } - - r := rand.Int32N(totalWeight) - for i, ch := range channels { - r -= cachedPrioritys[i] - if r < 0 { - return ch, nil - } - } - - return channels[rand.IntN(len(channels))], nil -} - -func getChannelWithFallback(cache *dbmodel.ModelCaches, model string, errorRates map[int64]float64, ignoreChannelIDs ...int64) (*dbmodel.Channel, error) { - channel, err := GetRandomChannel(cache, model, errorRates, ignoreChannelIDs...) - if err == nil { - return channel, nil - } - if !errors.Is(err, ErrChannelsExhausted) { - return nil, err - } - return GetRandomChannel(cache, model, errorRates) -} - -func NewRelay(mode relaymode.Mode) func(c *gin.Context) { - relayController, ok := relayController(mode) - if !ok { - log.Fatalf("relay mode %d not implemented", mode) - } - return func(c *gin.Context) { - relay(c, mode, relayController) - } -} - -func relay(c *gin.Context, mode relaymode.Mode, relayController RelayController) { - log := middleware.GetLogger(c) - requestModel := middleware.GetOriginalModel(c) - - // Get initial channel - initialChannel, err := getInitialChannel(c, requestModel, log) - if err != nil || initialChannel == nil || initialChannel.channel == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{ - "error": &model.Error{ - Message: "the upstream load is saturated, please try again later", - Code: "upstream_load_saturated", - Type: middleware.ErrorTypeAIPROXY, - }, - }) - return - } - - // First attempt - meta := middleware.NewMetaByContext(c, initialChannel.channel, requestModel, mode) - bizErr, retry := RelayHelper(meta, c, relayController) - if handleRelayResult(c, bizErr, retry) { - return - } - - // Setup retry state - retryState := initRetryState(initialChannel.channel, - bizErr, - initialChannel.ignoreChannelIDs, - initialChannel.errorRates, - ) - - // Retry loop - retryLoop(c, mode, requestModel, retryState, relayController, log) -} - -type retryState struct { - retryTimes int - lastHasPermissionChannel *dbmodel.Channel - ignoreChannelIDs []int64 - errorRates map[int64]float64 - exhausted bool - bizErr *model.ErrorWithStatusCode - startTime time.Time -} - -type initialChannel struct { - channel *dbmodel.Channel - ignoreChannelIDs []int64 - errorRates map[int64]float64 -} - -func getInitialChannel(c *gin.Context, requestModel string, log *log.Entry) (*initialChannel, error) { - ids, err := monitor.GetBannedChannelsWithModel(c.Request.Context(), requestModel) - if err != nil { - log.Errorf("get %s auto banned channels failed: %+v", requestModel, err) - } - log.Debugf("%s model banned channels: %+v", requestModel, ids) - - errorRates, err := monitor.GetModelChannelErrorRate(c.Request.Context(), requestModel) - if err != nil { - log.Errorf("get channel model error rates failed: %+v", err) - } - - mc := middleware.GetModelCaches(c) - channel, err := getChannelWithFallback(mc, requestModel, errorRates, ids...) - if err != nil { - return nil, err - } - - return &initialChannel{ - channel: channel, - ignoreChannelIDs: ids, - errorRates: errorRates, - }, nil -} - -func handleRelayResult(c *gin.Context, bizErr *model.ErrorWithStatusCode, retry bool) bool { - if bizErr == nil { - return true - } - if !retry { - bizErr.Error.Message = middleware.MessageWithRequestID(c, bizErr.Error.Message) - c.JSON(bizErr.StatusCode, bizErr) - return true - } - return false -} - -func initRetryState(channel *dbmodel.Channel, bizErr *model.ErrorWithStatusCode, ignoreChannelIDs []int64, errorRates map[int64]float64) *retryState { - state := &retryState{ - retryTimes: int(config.GetRetryTimes()), - ignoreChannelIDs: ignoreChannelIDs, - errorRates: errorRates, - bizErr: bizErr, - startTime: time.Now(), - } - - if !channelHasPermission(bizErr.StatusCode) { - state.ignoreChannelIDs = append(state.ignoreChannelIDs, int64(channel.ID)) - } else { - state.lastHasPermissionChannel = channel - } - - return state -} - -func retryLoop(c *gin.Context, mode relaymode.Mode, requestModel string, state *retryState, relayController RelayController, log *log.Entry) { - mc := middleware.GetModelCaches(c) - - for i := 0; i < state.retryTimes; i++ { - ctxErr := c.Request.Context().Err() - if ctxErr != nil { - log.Warnf("retry loop context error: %+v", ctxErr) - break - } - - newChannel, err := getRetryChannel(mc, requestModel, state) - if err != nil { - break - } - - log.Data["retry"] = strconv.Itoa(i + 1) - - log.Warnf("using channel %s (type: %d, id: %d) to retry (remain times %d)", - newChannel.Name, - newChannel.Type, - newChannel.ID, - state.retryTimes-i, - ) - - if !prepareRetry(c, state.bizErr.StatusCode) { - break - } - - meta := middleware.NewMetaByContext(c, - newChannel, - requestModel, - mode, - meta.WithRetryTimes(i+1), - ) - bizErr, retry := RelayHelper(meta, c, relayController) - - done := handleRetryResult(bizErr, retry, newChannel, state) - if done { - break - } - } - - if state.bizErr != nil { - state.bizErr.Error.Message = middleware.MessageWithRequestID(c, state.bizErr.Error.Message) - c.JSON(state.bizErr.StatusCode, state.bizErr) - } -} - -func getRetryChannel(mc *dbmodel.ModelCaches, model string, state *retryState) (*dbmodel.Channel, error) { - if state.exhausted { - return state.lastHasPermissionChannel, nil - } - - newChannel, err := GetRandomChannel(mc, model, state.errorRates, state.ignoreChannelIDs...) - if err != nil { - if !errors.Is(err, ErrChannelsExhausted) || state.lastHasPermissionChannel == nil { - return nil, err - } - state.exhausted = true - return state.lastHasPermissionChannel, nil - } - - return newChannel, nil -} - -func prepareRetry(c *gin.Context, statusCode int) bool { - requestBody, err := common.GetRequestBody(c.Request) - if err != nil { - log.Errorf("get request body failed in prepare retry: %+v", err) - return false - } - c.Request.Body = io.NopCloser(bytes.NewBuffer(requestBody)) - - if shouldDelay(statusCode) { - //nolint:gosec - time.Sleep(time.Duration(rand.Float64()*float64(time.Second)) + time.Second) - } - - return true -} - -func handleRetryResult(bizErr *model.ErrorWithStatusCode, retry bool, newChannel *dbmodel.Channel, state *retryState) (done bool) { - state.bizErr = bizErr - if !retry || bizErr == nil { - return true - } - - if state.exhausted { - if !channelHasPermission(bizErr.StatusCode) { - return true - } - } else { - if !channelHasPermission(bizErr.StatusCode) { - state.ignoreChannelIDs = append(state.ignoreChannelIDs, int64(newChannel.ID)) - state.retryTimes++ - } else { - state.lastHasPermissionChannel = newChannel - } - } - - return false -} - -func shouldRetry(_ *gin.Context, statusCode int) bool { - return statusCode != http.StatusBadRequest -} - -var channelNoPermissionStatusCodesMap = map[int]struct{}{ - http.StatusUnauthorized: {}, - http.StatusPaymentRequired: {}, - http.StatusForbidden: {}, -} - -func channelHasPermission(statusCode int) bool { - _, ok := channelNoPermissionStatusCodesMap[statusCode] - return !ok -} - -func shouldDelay(statusCode int) bool { - return statusCode == http.StatusTooManyRequests -} - -// 仅当是channel错误时，才需要记录，用户请求参数错误时，不需要记录 -func shouldErrorMonitor(statusCode int) bool { - return statusCode != http.StatusBadRequest -} - -func RelayNotImplemented(c *gin.Context) { - c.JSON(http.StatusNotImplemented, gin.H{ - "error": &model.Error{ - Message: "API not implemented", - Type: middleware.ErrorTypeAIPROXY, - Code: "api_not_implemented", - }, - }) -} diff --git a/service/aiproxy/controller/token.go b/service/aiproxy/controller/token.go deleted file mode 100644 index 134750150..000000000 --- a/service/aiproxy/controller/token.go +++ /dev/null @@ -1,465 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "net/http" - "strconv" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/network" - "github.com/labring/sealos/service/aiproxy/common/random" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" -) - -// TokenResponse represents the response structure for token endpoints -type TokenResponse struct { - *model.Token - AccessedAt time.Time `json:"accessed_at"` -} - -func (t *TokenResponse) MarshalJSON() ([]byte, error) { - type Alias TokenResponse - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at"` - ExpiredAt int64 `json:"expired_at"` - AccessedAt int64 `json:"accessed_at"` - }{ - Alias: (*Alias)(t), - CreatedAt: t.CreatedAt.UnixMilli(), - ExpiredAt: t.ExpiredAt.UnixMilli(), - AccessedAt: t.AccessedAt.UnixMilli(), - }) -} - -type ( - AddTokenRequest struct { - Name string `json:"name"` - Subnets []string `json:"subnets"` - Models []string `json:"models"` - ExpiredAt int64 `json:"expiredAt"` - Quota float64 `json:"quota"` - } - - UpdateTokenStatusRequest struct { - Status int `json:"status"` - } - - UpdateTokenNameRequest struct { - Name string `json:"name"` - } -) - -func (at *AddTokenRequest) ToToken() *model.Token { - var expiredAt time.Time - if at.ExpiredAt > 0 { - expiredAt = time.UnixMilli(at.ExpiredAt) - } - return &model.Token{ - Name: model.EmptyNullString(at.Name), - Subnets: at.Subnets, - Models: at.Models, - ExpiredAt: expiredAt, - Quota: at.Quota, - } -} - -func validateToken(token AddTokenRequest) error { - if token.Name == "" { - return errors.New("token name cannot be empty") - } - if len(token.Name) > 30 { - return errors.New("token name is too long") - } - if err := network.IsValidSubnets(token.Subnets); err != nil { - return fmt.Errorf("invalid subnet: %w", err) - } - return nil -} - -func validateTokenUpdate(token AddTokenRequest) error { - if err := network.IsValidSubnets(token.Subnets); err != nil { - return fmt.Errorf("invalid subnet: %w", err) - } - return nil -} - -func buildTokenResponse(token *model.Token) *TokenResponse { - lastRequestAt, _ := model.GetTokenLastRequestTime(token.ID) - return &TokenResponse{ - Token: token, - AccessedAt: lastRequestAt, - } -} - -func buildTokenResponses(tokens []*model.Token) []*TokenResponse { - responses := make([]*TokenResponse, len(tokens)) - for i, token := range tokens { - responses[i] = buildTokenResponse(token) - } - return responses -} - -// Token list handlers -func GetTokens(c *gin.Context) { - page, perPage := parsePageParams(c) - group := c.Query("group") - order := c.Query("order") - status, _ := strconv.Atoi(c.Query("status")) - - tokens, total, err := model.GetTokens(group, page, perPage, order, status) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, gin.H{ - "tokens": buildTokenResponses(tokens), - "total": total, - }) -} - -func GetGroupTokens(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - - page, perPage := parsePageParams(c) - order := c.Query("order") - status, _ := strconv.Atoi(c.Query("status")) - - tokens, total, err := model.GetTokens(group, page, perPage, order, status) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, gin.H{ - "tokens": buildTokenResponses(tokens), - "total": total, - }) -} - -func SearchTokens(c *gin.Context) { - page, perPage := parsePageParams(c) - keyword := c.Query("keyword") - order := c.Query("order") - name := c.Query("name") - key := c.Query("key") - status, _ := strconv.Atoi(c.Query("status")) - group := c.Query("group") - - tokens, total, err := model.SearchTokens(group, keyword, page, perPage, order, status, name, key) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, gin.H{ - "tokens": buildTokenResponses(tokens), - "total": total, - }) -} - -func SearchGroupTokens(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - - page, perPage := parsePageParams(c) - keyword := c.Query("keyword") - order := c.Query("order") - name := c.Query("name") - key := c.Query("key") - status, _ := strconv.Atoi(c.Query("status")) - - tokens, total, err := model.SearchTokens(group, keyword, page, perPage, order, status, name, key) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, gin.H{ - "tokens": buildTokenResponses(tokens), - "total": total, - }) -} - -// Single token handlers -func GetToken(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - token, err := model.GetTokenByID(id) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, buildTokenResponse(token)) -} - -func GetGroupToken(c *gin.Context) { - group := c.Param("group") - if group == "" { - middleware.ErrorResponse(c, http.StatusOK, "group is required") - return - } - - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - token, err := model.GetGroupTokenByID(group, id) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, buildTokenResponse(token)) -} - -func AddGroupToken(c *gin.Context) { - group := c.Param("group") - var req AddTokenRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := validateToken(req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, "parameter error: "+err.Error()) - return - } - - token := req.ToToken() - token.GroupID = group - token.Key = random.GenerateKey() - - if err := model.InsertToken(token, c.Query("auto_create_group") == "true"); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, &TokenResponse{Token: token}) -} - -// Delete handlers -func DeleteToken(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.DeleteTokenByID(id); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -func DeleteTokens(c *gin.Context) { - var ids []int - if err := c.ShouldBindJSON(&ids); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.DeleteTokensByIDs(ids); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -func DeleteGroupToken(c *gin.Context) { - group := c.Param("group") - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.DeleteGroupTokenByID(group, id); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -func DeleteGroupTokens(c *gin.Context) { - group := c.Param("group") - var ids []int - if err := c.ShouldBindJSON(&ids); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.DeleteGroupTokensByIDs(group, ids); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -// Update handlers -func UpdateToken(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - var req AddTokenRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := validateTokenUpdate(req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, "parameter error: "+err.Error()) - return - } - - token := req.ToToken() - - if err := model.UpdateToken(id, token); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, &TokenResponse{Token: token}) -} - -func UpdateGroupToken(c *gin.Context) { - group := c.Param("group") - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - var req AddTokenRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := validateTokenUpdate(req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, "parameter error: "+err.Error()) - return - } - - token := req.ToToken() - - if err := model.UpdateGroupToken(id, group, token); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, &TokenResponse{Token: token}) -} - -func UpdateTokenStatus(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - var req UpdateTokenStatusRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.UpdateTokenStatus(id, req.Status); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -func UpdateGroupTokenStatus(c *gin.Context) { - group := c.Param("group") - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - var req UpdateTokenStatusRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.UpdateGroupTokenStatus(group, id, req.Status); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -func UpdateTokenName(c *gin.Context) { - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - var req UpdateTokenNameRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.UpdateTokenName(id, req.Name); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} - -func UpdateGroupTokenName(c *gin.Context) { - group := c.Param("group") - id, err := strconv.Atoi(c.Param("id")) - if err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - var req UpdateTokenNameRequest - if err := c.ShouldBindJSON(&req); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - if err := model.UpdateGroupTokenName(group, id, req.Name); err != nil { - middleware.ErrorResponse(c, http.StatusOK, err.Error()) - return - } - - middleware.SuccessResponse(c, nil) -} diff --git a/service/aiproxy/controller/utils.go b/service/aiproxy/controller/utils.go deleted file mode 100644 index f24260559..000000000 --- a/service/aiproxy/controller/utils.go +++ /dev/null @@ -1,13 +0,0 @@ -package controller - -import ( - "strconv" - - "github.com/gin-gonic/gin" -) - -func parsePageParams(c *gin.Context) (page, perPage int) { - page, _ = strconv.Atoi(c.Query("p")) - perPage, _ = strconv.Atoi(c.Query("per_page")) - return -} diff --git a/service/aiproxy/deploy/Kubefile b/service/aiproxy/deploy/Kubefile deleted file mode 100644 index fa7a2551d..000000000 --- a/service/aiproxy/deploy/Kubefile +++ /dev/null @@ -1,23 +0,0 @@ -FROM scratch -COPY registry registry -COPY manifests manifests -COPY scripts scripts - -ENV cloudDomain="127.0.0.1.nip.io" -ENV cloudPort="" -ENV certSecretName="wildcard-cert" - -ENV ADMIN_KEY="" -ENV SEALOS_JWT_KEY="" -ENV SQL_DSN="" -ENV LOG_SQL_DSN="" -ENV REDIS_CONN_STRING="" - -ENV BALANCE_SEALOS_CHECK_REAL_NAME_ENABLE="false" -ENV BALANCE_SEALOS_NO_REAL_NAME_USED_AMOUNT_LIMIT="1" - -ENV SAVE_ALL_LOG_DETAIL="false" -ENV LOG_DETAIL_REQUEST_BODY_MAX_SIZE="128" -ENV LOG_DETAIL_RESPONSE_BODY_MAX_SIZE="128" - -CMD ["bash scripts/init.sh"] diff --git a/service/aiproxy/deploy/manifests/aiproxy-config.yaml.tmpl b/service/aiproxy/deploy/manifests/aiproxy-config.yaml.tmpl deleted file mode 100644 index 8171c5272..000000000 --- a/service/aiproxy/deploy/manifests/aiproxy-config.yaml.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aiproxy-env -data: - DEBUG: "false" - DEBUG_SQL: "false" - ADMIN_KEY: "{{ .ADMIN_KEY }}" - SEALOS_JWT_KEY: "{{ .SEALOS_JWT_KEY }}" - SQL_DSN: "{{ .SQL_DSN }}" - LOG_SQL_DSN: "{{ .LOG_SQL_DSN }}" - REDIS_CONN_STRING: "{{ .REDIS_CONN_STRING }}" - BALANCE_SEALOS_CHECK_REAL_NAME_ENABLE: "{{ .BALANCE_SEALOS_CHECK_REAL_NAME_ENABLE }}" - BALANCE_SEALOS_NO_REAL_NAME_USED_AMOUNT_LIMIT: "{{ .BALANCE_SEALOS_NO_REAL_NAME_USED_AMOUNT_LIMIT }}" - SAVE_ALL_LOG_DETAIL: "{{ .SAVE_ALL_LOG_DETAIL }}" - LOG_DETAIL_REQUEST_BODY_MAX_SIZE: "{{ .LOG_DETAIL_REQUEST_BODY_MAX_SIZE }}" - LOG_DETAIL_RESPONSE_BODY_MAX_SIZE: "{{ .LOG_DETAIL_RESPONSE_BODY_MAX_SIZE }}" diff --git a/service/aiproxy/deploy/manifests/deploy.yaml.tmpl b/service/aiproxy/deploy/manifests/deploy.yaml.tmpl deleted file mode 100644 index 52031c24a..000000000 --- a/service/aiproxy/deploy/manifests/deploy.yaml.tmpl +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: aiproxy - namespace: aiproxy-system - labels: - cloud.sealos.io/app-deploy-manager: aiproxy -spec: - ports: - - port: 3000 - targetPort: 3000 - selector: - app: aiproxy ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: aiproxy - namespace: aiproxy-system - annotations: - originImageName: ghcr.io/labring/sealos-aiproxy-service:latest - deploy.cloud.sealos.io/minReplicas: '3' - deploy.cloud.sealos.io/maxReplicas: '3' - labels: - cloud.sealos.io/app-deploy-manager: aiproxy - app: aiproxy -spec: - replicas: 3 - revisionHistoryLimit: 1 - selector: - matchLabels: - app: aiproxy - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 1 - template: - metadata: - labels: - app: aiproxy - spec: - containers: - - name: aiproxy - image: ghcr.io/labring/sealos-aiproxy-service:latest - envFrom: - - configMapRef: - name: aiproxy-env - resources: - requests: - cpu: 50m - memory: 50Mi - limits: - cpu: 500m - memory: 512Mi - ports: - - containerPort: 3000 - imagePullPolicy: Always - startupProbe: - httpGet: - port: 3000 - path: /api/status - initialDelaySeconds: 5 - periodSeconds: 3 - failureThreshold: 30 - successThreshold: 1 - timeoutSeconds: 1 - serviceAccountName: default - automountServiceAccountToken: false diff --git a/service/aiproxy/deploy/manifests/ingress.yaml.tmpl b/service/aiproxy/deploy/manifests/ingress.yaml.tmpl deleted file mode 100644 index 51d900969..000000000 --- a/service/aiproxy/deploy/manifests/ingress.yaml.tmpl +++ /dev/null @@ -1,37 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - kubernetes.io/ingress.class: nginx - nginx.ingress.kubernetes.io/enable-cors: "true" - nginx.ingress.kubernetes.io/cors-allow-methods: "PUT, GET, POST, DELETE, PATCH, OPTIONS" - nginx.ingress.kubernetes.io/cors-allow-origin: "https://{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}, https://*.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}" - nginx.ingress.kubernetes.io/cors-allow-credentials: "true" - nginx.ingress.kubernetes.io/cors-max-age: "600" - nginx.ingress.kubernetes.io/backend-protocol: "HTTP" - nginx.ingress.kubernetes.io/configuration-snippet: | - more_clear_headers "X-Frame-Options:"; - more_set_headers "Content-Security-Policy: default-src * blob: data: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}; img-src * data: blob: resource: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}; connect-src * wss: blob: resource:; style-src 'self' 'unsafe-inline' blob: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} resource:; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} resource: *.baidu.com *.bdstatic.com https://js.stripe.com; frame-src 'self' *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} mailto: tel: weixin: mtt: *.baidu.com https://js.stripe.com; frame-ancestors 'self' https://{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} https://*.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}"; - more_set_headers "X-Xss-Protection: 1; mode=block"; - higress.io/response-header-control-remove: X-Frame-Options - higress.io/response-header-control-update: | - Content-Security-Policy "default-src * blob: data: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}; img-src * data: blob: resource: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}; connect-src * wss: blob: resource:; style-src 'self' 'unsafe-inline' blob: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} resource:; script-src 'self' 'unsafe-inline' 'unsafe-eval' blob: *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} resource: *.baidu.com *.bdstatic.com https://js.stripe.com; frame-src 'self' *.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} {{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} mailto: tel: weixin: mtt: *.baidu.com https://js.stripe.com; frame-ancestors 'self' https://{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }} https://*.{{ .cloudDomain }}{{ if .cloudPort }}:{{ .cloudPort }}{{ end }}" - X-Xss-Protection "1; mode=block" - name: aiproxy - namespace: aiproxy-system -spec: - rules: - - host: aiproxy.{{ .cloudDomain }} - http: - paths: - - pathType: Prefix - path: /v1 - backend: - service: - name: aiproxy - port: - number: 3000 - tls: - - hosts: - - 'aiproxy.{{ .cloudDomain }}' - secretName: {{ .certSecretName }} diff --git a/service/aiproxy/deploy/manifests/pgsql-log.yaml b/service/aiproxy/deploy/manifests/pgsql-log.yaml deleted file mode 100644 index 6f1e1c752..000000000 --- a/service/aiproxy/deploy/manifests/pgsql-log.yaml +++ /dev/null @@ -1,94 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - finalizers: - - cluster.kubeblocks.io/finalizer - labels: - clusterdefinition.kubeblocks.io/name: postgresql - clusterversion.kubeblocks.io/name: postgresql-14.8.0 - sealos-db-provider-cr: aiproxy-log - annotations: {} - name: aiproxy-log - namespace: aiproxy-system -spec: - affinity: - nodeLabels: {} - podAntiAffinity: Preferred - tenancy: SharedNode - topologyKeys: - - kubernetes.io/hostname - clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-14.8.0 - componentSpecs: - - componentDefRef: postgresql - monitor: true - name: postgresql - replicas: 2 - resources: - limits: - cpu: 1000m - memory: 1024Mi - requests: - cpu: 100m - memory: 102Mi - serviceAccountName: aiproxy-log - switchPolicy: - type: Noop - volumeClaimTemplates: - - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - terminationPolicy: Delete - tolerations: [] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - sealos-db-provider-cr: aiproxy-log - app.kubernetes.io/instance: aiproxy-log - app.kubernetes.io/managed-by: kbcli - name: aiproxy-log - namespace: aiproxy-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - sealos-db-provider-cr: aiproxy-log - app.kubernetes.io/instance: aiproxy-log - app.kubernetes.io/managed-by: kbcli - name: aiproxy-log - namespace: aiproxy-system -rules: - - apiGroups: - - '*' - resources: - - '*' - verbs: - - '*' - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - sealos-db-provider-cr: aiproxy-log - app.kubernetes.io/instance: aiproxy-log - app.kubernetes.io/managed-by: kbcli - name: aiproxy-log - namespace: aiproxy-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: aiproxy-log -subjects: - - kind: ServiceAccount - name: aiproxy-log - namespace: aiproxy-system diff --git a/service/aiproxy/deploy/manifests/pgsql.yaml b/service/aiproxy/deploy/manifests/pgsql.yaml deleted file mode 100644 index f3b76b8b9..000000000 --- a/service/aiproxy/deploy/manifests/pgsql.yaml +++ /dev/null @@ -1,94 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - finalizers: - - cluster.kubeblocks.io/finalizer - labels: - clusterdefinition.kubeblocks.io/name: postgresql - clusterversion.kubeblocks.io/name: postgresql-14.8.0 - sealos-db-provider-cr: aiproxy - annotations: {} - name: aiproxy - namespace: aiproxy-system -spec: - affinity: - nodeLabels: {} - podAntiAffinity: Preferred - tenancy: SharedNode - topologyKeys: - - kubernetes.io/hostname - clusterDefinitionRef: postgresql - clusterVersionRef: postgresql-14.8.0 - componentSpecs: - - componentDefRef: postgresql - monitor: true - name: postgresql - replicas: 2 - resources: - limits: - cpu: 1000m - memory: 1024Mi - requests: - cpu: 100m - memory: 102Mi - serviceAccountName: aiproxy - switchPolicy: - type: Noop - volumeClaimTemplates: - - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 3Gi - terminationPolicy: Delete - tolerations: [] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - sealos-db-provider-cr: aiproxy - app.kubernetes.io/instance: aiproxy - app.kubernetes.io/managed-by: kbcli - name: aiproxy - namespace: aiproxy-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - sealos-db-provider-cr: aiproxy - app.kubernetes.io/instance: aiproxy - app.kubernetes.io/managed-by: kbcli - name: aiproxy - namespace: aiproxy-system -rules: - - apiGroups: - - "*" - resources: - - "*" - verbs: - - "*" - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - sealos-db-provider-cr: aiproxy - app.kubernetes.io/instance: aiproxy - app.kubernetes.io/managed-by: kbcli - name: aiproxy - namespace: aiproxy-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: aiproxy -subjects: - - kind: ServiceAccount - name: aiproxy - namespace: aiproxy-system diff --git a/service/aiproxy/deploy/manifests/redis.yaml b/service/aiproxy/deploy/manifests/redis.yaml deleted file mode 100644 index a1148a2f7..000000000 --- a/service/aiproxy/deploy/manifests/redis.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 -kind: Cluster -metadata: - finalizers: - - cluster.kubeblocks.io/finalizer - labels: - clusterdefinition.kubeblocks.io/name: redis - clusterversion.kubeblocks.io/name: redis-7.0.6 - sealos-db-provider-cr: aiproxy-redis - annotations: {} - name: aiproxy-redis - namespace: aiproxy-system -spec: - affinity: - nodeLabels: {} - podAntiAffinity: Preferred - tenancy: SharedNode - topologyKeys: - - kubernetes.io/hostname - clusterDefinitionRef: redis - clusterVersionRef: redis-7.0.6 - componentSpecs: - - componentDefRef: redis - monitor: true - name: redis - replicas: 3 - resources: - limits: - cpu: 1000m - memory: 1024Mi - requests: - cpu: 100m - memory: 102Mi - serviceAccountName: aiproxy-redis - switchPolicy: - type: Noop - volumeClaimTemplates: - - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 3Gi - storageClassName: openebs-backup - - componentDefRef: redis-sentinel - monitor: true - name: redis-sentinel - replicas: 3 - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - serviceAccountName: aiproxy-redis - terminationPolicy: Delete - tolerations: [] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - sealos-db-provider-cr: aiproxy-redis - app.kubernetes.io/instance: aiproxy-redis - app.kubernetes.io/managed-by: kbcli - name: aiproxy-redis - namespace: aiproxy-system - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - sealos-db-provider-cr: aiproxy-redis - app.kubernetes.io/instance: aiproxy-redis - app.kubernetes.io/managed-by: kbcli - name: aiproxy-redis - namespace: aiproxy-system -rules: - - apiGroups: - - '*' - resources: - - '*' - verbs: - - '*' - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - sealos-db-provider-cr: aiproxy-redis - app.kubernetes.io/instance: aiproxy-redis - app.kubernetes.io/managed-by: kbcli - name: aiproxy-redis - namespace: aiproxy-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: aiproxy-redis -subjects: - - kind: ServiceAccount - name: aiproxy-redis - namespace: aiproxy-system diff --git a/service/aiproxy/deploy/scripts/init.sh b/service/aiproxy/deploy/scripts/init.sh deleted file mode 100644 index ed1b3678d..000000000 --- a/service/aiproxy/deploy/scripts/init.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/bash -set -e - -# Create namespace -kubectl create ns aiproxy-system || true - -# Function to wait for secret -wait_for_secret() { - local secret_name=$1 - local retries=0 - while ! kubectl get secret -n aiproxy-system ${secret_name} >/dev/null 2>&1; do - sleep 3 - retries=$((retries + 1)) - if [ $retries -ge 30 ]; then - echo "Timeout waiting for secret ${secret_name}" - exit 1 - fi - done -} - -# Function to get secret value -get_secret_value() { - local secret_name=$1 - local key=$2 - base64_value=$(kubectl get secret -n aiproxy-system ${secret_name} -o jsonpath="{.data.${key}}") || return $? - echo "$base64_value" | base64 -d -} - -# Function to build postgres connection string -build_postgres_dsn() { - local secret_name=$1 - username=$(get_secret_value ${secret_name} "username") || return $? - password=$(get_secret_value ${secret_name} "password") || return $? - host=$(get_secret_value ${secret_name} "host") || return $? - port=$(get_secret_value ${secret_name} "port") || return $? - echo "postgres://${username}:${password}@${host}:${port}/postgres?sslmode=disable" -} - -build_redis_conn() { - local secret_name=$1 - username=$(get_secret_value ${secret_name} "username") || return $? - password=$(get_secret_value ${secret_name} "password") || return $? - host=$(get_secret_value ${secret_name} "host") || return $? - port=$(get_secret_value ${secret_name} "port") || return $? - echo "redis://${username}:${password}@${host}:${port}" -} - -# Handle JWT configuration -if grep "" manifests/aiproxy-config.yaml >/dev/null 2>&1; then - JWT_SECRET=$(kubectl get cm -n account-system account-manager-env -o jsonpath="{.data.ACCOUNT_API_JWT_SECRET}") || exit $? - sed -i "s||${JWT_SECRET}|g" manifests/aiproxy-config.yaml -fi - -# Handle PostgreSQL configuration -if grep "" manifests/aiproxy-config.yaml >/dev/null 2>&1; then - if grep "" manifests/aiproxy-config.yaml >/dev/null 2>&1; then - # Deploy PostgreSQL resources - kubectl apply -f manifests/pgsql.yaml -n aiproxy-system - kubectl apply -f manifests/pgsql-log.yaml -n aiproxy-system - - # Wait for secrets - wait_for_secret "aiproxy-conn-credential" - wait_for_secret "aiproxy-log-conn-credential" - - # Build connection strings - SQL_DSN=$(build_postgres_dsn "aiproxy-conn-credential") || exit $? - LOG_SQL_DSN=$(build_postgres_dsn "aiproxy-log-conn-credential") || exit $? - - # Update config - sed -i "s||${SQL_DSN}|g" manifests/aiproxy-config.yaml - sed -i "s||${LOG_SQL_DSN}|g" manifests/aiproxy-config.yaml - elif grep "LOG_SQL_DSN: \"\"" manifests/aiproxy-config.yaml >/dev/null 2>&1; then - # Deploy PostgreSQL resources - kubectl apply -f manifests/pgsql.yaml -n aiproxy-system - - # Wait for secrets - wait_for_secret "aiproxy-conn-credential" - - # Build connection strings - SQL_DSN=$(build_postgres_dsn "aiproxy-conn-credential") || exit $? - - # Update config - sed -i "s||${SQL_DSN}|g" manifests/aiproxy-config.yaml - else - echo "Error: LOG_SQL_DSN is not allowed to be passed alone, please provide both SQL_DSN and LOG_SQL_DSN or provide SQL_DSN only or neither." - exit 1 - fi -elif grep "" manifests/aiproxy-config.yaml >/dev/null 2>&1; then - sed -i 's///g' manifests/aiproxy-config.yaml -fi - -# Handle Redis configuration -if grep "" manifests/aiproxy-config.yaml >/dev/null 2>&1; then - kubectl apply -f manifests/redis.yaml -n aiproxy-system - - wait_for_secret "aiproxy-redis-conn-credential" - - # Build redis connection string - REDIS_CONN=$(build_redis_conn "aiproxy-redis-conn-credential") || exit $? - - sed -i "s||${REDIS_CONN}|g" manifests/aiproxy-config.yaml -fi - -# Deploy application -kubectl apply -f manifests/aiproxy-config.yaml -n aiproxy-system -kubectl apply -f manifests/deploy.yaml -n aiproxy-system - -# Create ingress if domain is specified -if [[ -n "$cloudDomain" ]]; then - kubectl create -f manifests/ingress.yaml -n aiproxy-system || true -fi diff --git a/service/aiproxy/go.mod b/service/aiproxy/go.mod deleted file mode 100644 index 21af84bca..000000000 --- a/service/aiproxy/go.mod +++ /dev/null @@ -1,115 +0,0 @@ -module github.com/labring/sealos/service/aiproxy - -go 1.22.7 - -replace github.com/labring/sealos/service/aiproxy => ../aiproxy - -require ( - cloud.google.com/go/iam v1.3.1 - github.com/aws/aws-sdk-go-v2 v1.36.1 - github.com/aws/aws-sdk-go-v2/credentials v1.17.59 - github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.24.4 - github.com/bytedance/sonic v1.12.10 - github.com/gin-contrib/cors v1.7.3 - github.com/gin-contrib/gzip v1.2.2 - github.com/gin-gonic/gin v1.10.0 - github.com/glebarez/sqlite v1.11.0 - github.com/golang-jwt/jwt/v5 v5.2.1 - github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.3 - github.com/jinzhu/copier v0.4.0 - github.com/joho/godotenv v1.5.1 - github.com/json-iterator/go v1.1.12 - github.com/maruel/natural v1.1.1 - github.com/mattn/go-isatty v0.0.20 - github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/pkg/errors v0.9.1 - github.com/pkoukk/tiktoken-go v0.1.7 - github.com/redis/go-redis/v9 v9.7.0 - github.com/shopspring/decimal v1.4.0 - github.com/sirupsen/logrus v1.9.3 - github.com/smartystreets/goconvey v1.8.1 - github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c - github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef - github.com/stretchr/testify v1.10.0 - golang.org/x/image v0.24.0 - golang.org/x/sync v0.11.0 - google.golang.org/api v0.220.0 - gorm.io/driver/mysql v1.5.7 - gorm.io/driver/postgres v1.5.11 - gorm.io/gorm v1.25.12 -) - -require ( - cloud.google.com/go/auth v0.14.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.7 // indirect - cloud.google.com/go/compute/metadata v0.6.0 // indirect - filippo.io/edwards25519 v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect - github.com/aws/smithy-go v1.22.2 // indirect - github.com/bytedance/sonic/loader v0.2.3 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudwego/base64x v0.1.5 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/dlclark/regexp2 v1.11.4 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.8 // indirect - github.com/gin-contrib/sse v1.0.0 // indirect - github.com/glebarez/go-sqlite v1.22.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-playground/locales v0.14.1 // indirect - github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.24.0 // indirect - github.com/go-sql-driver/mysql v1.8.1 // indirect - github.com/goccy/go-json v0.10.5 // indirect - github.com/google/s2a-go v0.1.9 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.1 // indirect - github.com/gopherjs/gopherjs v1.17.2 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/pgx/v5 v5.7.2 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.5 // indirect - github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect - github.com/leodido/go-urn v1.4.0 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/smarty/assertions v1.15.0 // indirect - github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.2.12 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect - go.opentelemetry.io/otel v1.34.0 // indirect - go.opentelemetry.io/otel/metric v1.34.0 // indirect - go.opentelemetry.io/otel/trace v1.34.0 // indirect - golang.org/x/arch v0.14.0 // indirect - golang.org/x/crypto v0.33.0 // indirect - golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac // indirect - golang.org/x/net v0.35.0 // indirect - golang.org/x/oauth2 v0.26.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect - golang.org/x/time v0.10.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 // indirect - google.golang.org/grpc v1.70.0 // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - modernc.org/libc v1.61.12 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.8.2 // indirect - modernc.org/sqlite v1.34.5 // indirect -) diff --git a/service/aiproxy/go.sum b/service/aiproxy/go.sum deleted file mode 100644 index ced9e9729..000000000 --- a/service/aiproxy/go.sum +++ /dev/null @@ -1,277 +0,0 @@ -cloud.google.com/go/auth v0.14.1 h1:AwoJbzUdxA/whv1qj3TLKwh3XX5sikny2fc40wUl+h0= -cloud.google.com/go/auth v0.14.1/go.mod h1:4JHUxlGXisL0AW8kXPtUF6ztuOksyfUQNFjfsOCXkPM= -cloud.google.com/go/auth/oauth2adapt v0.2.7 h1:/Lc7xODdqcEw8IrZ9SvwnlLX6j9FHQM74z6cBk9Rw6M= -cloud.google.com/go/auth/oauth2adapt v0.2.7/go.mod h1:NTbTTzfvPl1Y3V1nPpOgl2w6d/FjO7NNUQaWSox6ZMc= -cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= -cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= -cloud.google.com/go/iam v1.3.1 h1:KFf8SaT71yYq+sQtRISn90Gyhyf4X8RGgeAVC8XGf3E= -cloud.google.com/go/iam v1.3.1/go.mod h1:3wMtuyT4NcbnYNPLMBzYRFiEfjKfJlLVLrisE7bwm34= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= -github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= -github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.24.4 h1:NYHDOBe0ZIeQfaPSPRaQym2NePzA+QYM3O/Oh4IznKg= -github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.24.4/go.mod h1:AD+JAcEr9fNzFcfKs3CINKBdWGFK7R+/uZ+VdJRhK2U= -github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= -github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= -github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= -github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= -github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= -github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bytedance/sonic v1.12.10 h1:uVCQr6oS5669E9ZVW0HyksTLfNS7Q/9hV6IVS4nEMsI= -github.com/bytedance/sonic v1.12.10/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0= -github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= -github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= -github.com/gin-contrib/cors v1.7.3 h1:hV+a5xp8hwJoTw7OY+a70FsL8JkVVFTXw9EcfrYUdns= -github.com/gin-contrib/cors v1.7.3/go.mod h1:M3bcKZhxzsvI+rlRSkkxHyljJt1ESd93COUvemZ79j4= -github.com/gin-contrib/gzip v1.2.2 h1:iUU/EYCM8ENfkjmZaVrxbjF/ZC267Iqv5S0MMCMEliI= -github.com/gin-contrib/gzip v1.2.2/go.mod h1:C1a5cacjlDsS20cKnHlZRCPUu57D3qH6B2pV0rl+Y/s= -github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= -github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ= -github.com/glebarez/go-sqlite v1.22.0/go.mod h1:PlBIdHe0+aUEFn+r2/uthrWq4FxbzugL0L8Li6yQJbc= -github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= -github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg= -github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= -github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= -github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= -github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= -github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= -github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= -github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= -github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= -github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= -github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= -github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= -github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= -github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= -github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= -github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkoukk/tiktoken-go v0.1.7 h1:qOBHXX4PHtvIvmOtyg1EeKlwFRiMKAcoMp4Q+bLQDmw= -github.com/pkoukk/tiktoken-go v0.1.7/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= -github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= -github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= -github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= -github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= -github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= -github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= -github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c h1:km8GpoQut05eY3GiYWEedbTT0qnSxrCjsVbb7yKY1KE= -github.com/srwiley/oksvg v0.0.0-20221011165216-be6e8873101c/go.mod h1:cNQ3dwVJtS5Hmnjxy6AgTPd0Inb3pW05ftPSX7NZO7Q= -github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef h1:Ch6Q+AZUxDBCVqdkI8FSpFyZDtCVBc2VmejdNrm5rRQ= -github.com/srwiley/rasterx v0.0.0-20220730225603-2ab79fcdd4ef/go.mod h1:nXTWP6+gD5+LUJ8krVhhoeHjvHTutPxMYl5SvkcnJNE= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= -github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= -golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac h1:l5+whBCLH3iH2ZNHYLbAe58bo7yrN4mVcnkHDYz5vvs= -golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac/go.mod h1:hH+7mtFmImwwcMvScyxUhjuVHR3HGaDPMn9rMSUUbxo= -golang.org/x/image v0.24.0 h1:AN7zRgVsbvmTfNyqIbbOraYL8mSwcKncEj8ofjgzcMQ= -golang.org/x/image v0.24.0/go.mod h1:4b/ITuLfqYq1hqZcjofwctIhi7sZh2WaCjvsBNjjya8= -golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= -golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= -golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= -golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= -golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= -google.golang.org/api v0.220.0 h1:3oMI4gdBgB72WFVwE1nerDD8W3HUOS4kypK6rRLbGns= -google.golang.org/api v0.220.0/go.mod h1:26ZAlY6aN/8WgpCzjPNy18QpYaz7Zgg1h0qe1GkZEmY= -google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6 h1:L9JNMl/plZH9wmzQUHleO/ZZDSN+9Gh41wPczNy+5Fk= -google.golang.org/genproto/googleapis/api v0.0.0-20250207221924-e9438ea467c6/go.mod h1:iYONQfRdizDB8JJBybql13nArx91jcUk7zCXEsOofM4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6 h1:2duwAxN2+k0xLNpjnHTXoMUgnv6VPSp5fiqTuwSxjmI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250207221924-e9438ea467c6/go.mod h1:8BS3B93F/U1juMFq9+EDk+qOT5CO1R9IzXxG3PTqiRk= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= -gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314= -gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= -gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= -modernc.org/cc/v4 v4.24.4 h1:TFkx1s6dCkQpd6dKurBNmpo+G8Zl4Sq/ztJ+2+DEsh0= -modernc.org/cc/v4 v4.24.4/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.23.16 h1:Z2N+kk38b7SfySC1ZkpGLN2vthNJP1+ZzGZIlH7uBxo= -modernc.org/ccgo/v4 v4.23.16/go.mod h1:nNma8goMTY7aQZQNTyN9AIoJfxav4nvTnvKThAeMDdo= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.6.3 h1:aJVhcqAte49LF+mGveZ5KPlsp4tdGdAOT4sipJXADjw= -modernc.org/gc/v2 v2.6.3/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.61.12 h1:Fsnh0A7XLXylYNwIOJmKux9PhnfrIvMaMnjuyJ1t/f4= -modernc.org/libc v1.61.12/go.mod h1:8F/uJWL/3nNil0Lgt1Dpz+GgkApWh04N3el3hxJcA6E= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.8.2 h1:cL9L4bcoAObu4NkxOlKWBWtNHIsnnACGF/TbqQ6sbcI= -modernc.org/memory v1.8.2/go.mod h1:ZbjSvMO5NQ1A2i3bWeDiVMxIorXwdClKE/0SZ+BMotU= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/service/aiproxy/main.go b/service/aiproxy/main.go deleted file mode 100644 index 372cb70aa..000000000 --- a/service/aiproxy/main.go +++ /dev/null @@ -1,250 +0,0 @@ -package main - -import ( - "context" - "errors" - "flag" - "fmt" - stdlog "log" - "net/http" - "os" - "os/signal" - "runtime" - "strconv" - "sync" - "syscall" - "time" - - "github.com/gin-gonic/gin" - _ "github.com/joho/godotenv/autoload" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/balance" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/consume" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/labring/sealos/service/aiproxy/controller" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/router" - log "github.com/sirupsen/logrus" -) - -var port int - -func init() { - flag.IntVar(&port, "port", 3000, "http server port") -} - -func initializeServices() error { - setLog(log.StandardLogger()) - - initializeNotifier() - - if err := initializeBalance(); err != nil { - return err - } - - if err := initializeDatabases(); err != nil { - return err - } - - return initializeCaches() -} - -func initializeBalance() error { - sealosJwtKey := os.Getenv("SEALOS_JWT_KEY") - if sealosJwtKey == "" { - log.Info("SEALOS_JWT_KEY is not set, balance will not be enabled") - return nil - } - - log.Info("SEALOS_JWT_KEY is set, balance will be enabled") - return balance.InitSealos(sealosJwtKey, os.Getenv("SEALOS_ACCOUNT_URL")) -} - -func initializeNotifier() { - feishuWh := os.Getenv("NOTIFY_FEISHU_WEBHOOK") - if feishuWh != "" { - notify.SetDefaultNotifier(notify.NewFeishuNotify(feishuWh)) - log.Info("NOTIFY_FEISHU_WEBHOOK is set, notifier will be use feishu") - } -} - -var logCallerIgnoreFuncs = map[string]struct{}{ - "github.com/labring/sealos/service/aiproxy/middleware.logColor": {}, -} - -func setLog(l *log.Logger) { - gin.ForceConsoleColor() - if config.DebugEnabled { - l.SetLevel(log.DebugLevel) - l.SetReportCaller(true) - gin.SetMode(gin.DebugMode) - } else { - l.SetLevel(log.InfoLevel) - l.SetReportCaller(false) - gin.SetMode(gin.ReleaseMode) - } - l.SetOutput(os.Stdout) - stdlog.SetOutput(l.Writer()) - - l.SetFormatter(&log.TextFormatter{ - ForceColors: true, - DisableColors: false, - ForceQuote: config.DebugEnabled, - DisableQuote: !config.DebugEnabled, - DisableSorting: false, - FullTimestamp: true, - TimestampFormat: time.DateTime, - QuoteEmptyFields: true, - CallerPrettyfier: func(f *runtime.Frame) (function string, file string) { - if _, ok := logCallerIgnoreFuncs[f.Function]; ok { - return "", "" - } - return f.Function, fmt.Sprintf("%s:%d", f.File, f.Line) - }, - }) - - if common.NeedColor() { - gin.ForceConsoleColor() - } -} - -func initializeDatabases() error { - model.InitDB() - model.InitLogDB() - return common.InitRedisClient() -} - -func initializeCaches() error { - if err := model.InitOption2DB(); err != nil { - return err - } - return model.InitModelConfigAndChannelCache() -} - -func startSyncServices(ctx context.Context, wg *sync.WaitGroup) { - wg.Add(2) - go model.SyncOptions(ctx, wg, time.Second*5) - go model.SyncModelConfigAndChannelCache(ctx, wg, time.Second*10) -} - -func setupHTTPServer() (*http.Server, *gin.Engine) { - server := gin.New() - - w := log.StandardLogger().Writer() - server. - Use(gin.RecoveryWithWriter(w)). - Use(middleware.NewLog(log.StandardLogger())). - Use(middleware.RequestID, middleware.CORS()) - router.SetRouter(server) - - p := os.Getenv("PORT") - if p == "" { - p = strconv.Itoa(port) - } - - return &http.Server{ - Addr: ":" + p, - ReadHeaderTimeout: 10 * time.Second, - Handler: server, - }, server -} - -func autoTestBannedModels(ctx context.Context) { - log.Info("auto test banned models start") - ticker := time.NewTicker(time.Second * 30) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - controller.AutoTestBannedModels() - } - } -} - -func cleanLog(ctx context.Context) { - log.Info("clean log start") - // the interval should not be too large to avoid cleaning too much at once - ticker := time.NewTicker(time.Second * 15) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - err := model.CleanLog(1000) - if err != nil { - notify.ErrorThrottle("cleanLog", time.Minute, "clean log failed", err.Error()) - } - } - } -} - -func main() { - flag.Parse() - - if err := initializeServices(); err != nil { - log.Fatal("failed to initialize services: " + err.Error()) - } - - defer func() { - if err := model.CloseDB(); err != nil { - log.Fatal("failed to close database: " + err.Error()) - } - }() - - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - - var wg sync.WaitGroup - startSyncServices(ctx, &wg) - - srv, _ := setupHTTPServer() - - go func() { - log.Infof("server started on http://localhost:%s", srv.Addr[1:]) - if err := srv.ListenAndServe(); err != nil && - !errors.Is(err, http.ErrServerClosed) { - log.Fatal("failed to start HTTP server: " + err.Error()) - } - }() - - go autoTestBannedModels(ctx) - go cleanLog(ctx) - go controller.UpdateChannelsBalance(time.Minute * 10) - - batchProcessorCtx, batchProcessorCancel := context.WithCancel(context.Background()) - wg.Add(1) - go model.StartBatchProcessor(batchProcessorCtx, &wg) - - <-ctx.Done() - - shutdownCtx, cancel := context.WithTimeout(context.Background(), 600*time.Second) - defer cancel() - - log.Info("shutting down http server...") - log.Info("max wait time: 600s") - if err := srv.Shutdown(shutdownCtx); err != nil { - log.Error("server forced to shutdown: " + err.Error()) - } else { - log.Info("server shutdown successfully") - } - - log.Info("shutting down consumer...") - consume.Wait() - - batchProcessorCancel() - - log.Info("shutting down sync services...") - wg.Wait() - - log.Info("shutting down batch processor...") - model.ProcessBatchUpdates() - - log.Info("server exiting") -} diff --git a/service/aiproxy/middleware/auth.go b/service/aiproxy/middleware/auth.go deleted file mode 100644 index 6502088ed..000000000 --- a/service/aiproxy/middleware/auth.go +++ /dev/null @@ -1,241 +0,0 @@ -package middleware - -import ( - "fmt" - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/ctxkey" - "github.com/labring/sealos/service/aiproxy/common/network" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/sirupsen/logrus" -) - -type APIResponse struct { - Data any `json:"data,omitempty"` - Message string `json:"message,omitempty"` - Success bool `json:"success"` -} - -func SuccessResponse(c *gin.Context, data any) { - c.JSON(http.StatusOK, &APIResponse{ - Success: true, - Data: data, - }) -} - -func ErrorResponse(c *gin.Context, code int, message string) { - c.JSON(code, &APIResponse{ - Success: false, - Message: message, - }) -} - -func AdminAuth(c *gin.Context) { - accessToken := c.Request.Header.Get("Authorization") - if config.AdminKey != "" && (accessToken == "" || strings.TrimPrefix(accessToken, "Bearer ") != config.AdminKey) { - ErrorResponse(c, http.StatusUnauthorized, "unauthorized, no access token provided") - c.Abort() - return - } - - group := c.Param("group") - if group != "" { - log := GetLogger(c) - log.Data["gid"] = group - } - - c.Next() -} - -func TokenAuth(c *gin.Context) { - log := GetLogger(c) - key := c.Request.Header.Get("Authorization") - key = strings.TrimPrefix( - strings.TrimPrefix(key, "Bearer "), - "sk-", - ) - - var token *model.TokenCache - var useInternalToken bool - if config.AdminKey != "" && config.AdminKey == key || - config.GetInternalToken() != "" && config.GetInternalToken() == key { - token = &model.TokenCache{} - useInternalToken = true - } else { - var err error - token, err = model.ValidateAndGetToken(key) - if err != nil { - abortLogWithMessage(c, http.StatusUnauthorized, err.Error(), &errorField{ - Code: "invalid_token", - }) - return - } - } - - SetLogTokenFields(log.Data, token, useInternalToken) - - if len(token.Subnets) > 0 { - if ok, err := network.IsIPInSubnets(c.ClientIP(), token.Subnets); err != nil { - abortLogWithMessage(c, http.StatusInternalServerError, err.Error()) - return - } else if !ok { - abortLogWithMessage(c, http.StatusForbidden, - fmt.Sprintf("token (%s[%d]) can only be used in the specified subnets: %v, current ip: %s", - token.Name, - token.ID, - token.Subnets, - c.ClientIP(), - ), - ) - return - } - } - - var group *model.GroupCache - if useInternalToken { - group = &model.GroupCache{ - Status: model.GroupStatusInternal, - } - } else { - var err error - group, err = model.CacheGetGroup(token.Group) - if err != nil { - abortLogWithMessage(c, http.StatusInternalServerError, fmt.Sprintf("failed to get group: %v", err)) - return - } - if group.Status != model.GroupStatusEnabled && group.Status != model.GroupStatusInternal { - abortLogWithMessage(c, http.StatusForbidden, "group is disabled") - return - } - } - - SetLogGroupFields(log.Data, group) - - modelCaches := model.LoadModelCaches() - - storeTokenModels(token, modelCaches) - - c.Set(ctxkey.Group, group) - c.Set(ctxkey.Token, token) - c.Set(ctxkey.ModelCaches, modelCaches) - - c.Next() -} - -func GetGroup(c *gin.Context) *model.GroupCache { - return c.MustGet(ctxkey.Group).(*model.GroupCache) -} - -func GetToken(c *gin.Context) *model.TokenCache { - return c.MustGet(ctxkey.Token).(*model.TokenCache) -} - -func GetModelCaches(c *gin.Context) *model.ModelCaches { - return c.MustGet(ctxkey.ModelCaches).(*model.ModelCaches) -} - -func sliceFilter[T any](s []T, fn func(T) bool) []T { - i := 0 - for _, v := range s { - if fn(v) { - s[i] = v - i++ - } - } - return s[:i] -} - -func storeTokenModels(token *model.TokenCache, modelCaches *model.ModelCaches) { - if len(token.Models) == 0 { - token.Models = modelCaches.EnabledModels - } else { - enabledModelsMap := modelCaches.EnabledModelsMap - token.Models = sliceFilter(token.Models, func(m string) bool { - _, ok := enabledModelsMap[m] - return ok - }) - } -} - -func SetLogFieldsFromMeta(m *meta.Meta, fields logrus.Fields) { - SetLogRequestIDField(fields, m.RequestID) - - SetLogModeField(fields, m.Mode) - SetLogModelFields(fields, m.OriginModel) - SetLogActualModelFields(fields, m.ActualModel) - - if m.IsChannelTest { - SetLogIsChannelTestField(fields, true) - } - - SetLogGroupFields(fields, m.Group) - SetLogTokenFields(fields, m.Token, false) - SetLogChannelFields(fields, m.Channel) -} - -func SetLogModeField(fields logrus.Fields, mode relaymode.Mode) { - fields["mode"] = mode.String() -} - -func SetLogIsChannelTestField(fields logrus.Fields, isChannelTest bool) { - fields["test"] = isChannelTest -} - -func SetLogActualModelFields(fields logrus.Fields, actualModel string) { - fields["actmodel"] = actualModel -} - -func SetLogModelFields(fields logrus.Fields, model string) { - fields["model"] = model -} - -func SetLogChannelFields(fields logrus.Fields, channel *meta.ChannelMeta) { - if channel != nil { - fields["chid"] = channel.ID - fields["chname"] = channel.Name - fields["chtype"] = channel.Type - } -} - -func SetLogRequestIDField(fields logrus.Fields, requestID string) { - fields["reqid"] = requestID -} - -func SetLogGroupFields(fields logrus.Fields, group *model.GroupCache) { - if group == nil { - return - } - if group.ID != "" { - fields["gid"] = group.ID - } -} - -func SetLogTokenFields(fields logrus.Fields, token *model.TokenCache, internal bool) { - if token == nil { - return - } - if token.ID > 0 { - fields["tid"] = token.ID - } - if token.Name != "" { - fields["tname"] = token.Name - } - if token.Key != "" { - fields["key"] = maskTokenKey(token.Key) - } - if internal { - fields["internal"] = "true" - } -} - -func maskTokenKey(key string) string { - if len(key) <= 8 { - return "*****" - } - return key[:4] + "*****" + key[len(key)-4:] -} diff --git a/service/aiproxy/middleware/cors.go b/service/aiproxy/middleware/cors.go deleted file mode 100644 index d2a109abe..000000000 --- a/service/aiproxy/middleware/cors.go +++ /dev/null @@ -1,15 +0,0 @@ -package middleware - -import ( - "github.com/gin-contrib/cors" - "github.com/gin-gonic/gin" -) - -func CORS() gin.HandlerFunc { - config := cors.DefaultConfig() - config.AllowAllOrigins = true - config.AllowCredentials = true - config.AllowMethods = []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"} - config.AllowHeaders = []string{"*"} - return cors.New(config) -} diff --git a/service/aiproxy/middleware/distributor.go b/service/aiproxy/middleware/distributor.go deleted file mode 100644 index f6ab3daed..000000000 --- a/service/aiproxy/middleware/distributor.go +++ /dev/null @@ -1,340 +0,0 @@ -package middleware - -import ( - "errors" - "fmt" - "net/http" - "slices" - "strconv" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/bytedance/sonic/ast" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/balance" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/consume" - "github.com/labring/sealos/service/aiproxy/common/ctxkey" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/labring/sealos/service/aiproxy/common/rpmlimit" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - log "github.com/sirupsen/logrus" -) - -func calculateGroupConsumeLevelRatio(usedAmount float64) float64 { - v := config.GetGroupConsumeLevelRatio() - if len(v) == 0 { - return 1 - } - var maxConsumeLevel float64 = -1 - var groupConsumeLevelRatio float64 - for consumeLevel, ratio := range v { - if usedAmount < consumeLevel { - continue - } - if consumeLevel > maxConsumeLevel { - maxConsumeLevel = consumeLevel - groupConsumeLevelRatio = ratio - } - } - if groupConsumeLevelRatio <= 0 { - groupConsumeLevelRatio = 1 - } - return groupConsumeLevelRatio -} - -func getGroupPMRatio(group *model.GroupCache) (float64, float64) { - groupRPMRatio := group.RPMRatio - if groupRPMRatio <= 0 { - groupRPMRatio = 1 - } - groupTPMRatio := group.TPMRatio - if groupTPMRatio <= 0 { - groupTPMRatio = 1 - } - return groupRPMRatio, groupTPMRatio -} - -func GetGroupAdjustedModelConfig(group *model.GroupCache, mc *model.ModelConfig) *model.ModelConfig { - rpm := mc.RPM - tpm := mc.TPM - if group.RPM != nil && group.RPM[mc.Model] > 0 { - rpm = group.RPM[mc.Model] - } - if group.TPM != nil && group.TPM[mc.Model] > 0 { - tpm = group.TPM[mc.Model] - } - rpmRatio, tpmRatio := getGroupPMRatio(group) - groupConsumeLevelRatio := calculateGroupConsumeLevelRatio(group.UsedAmount) - rpm = int64(float64(rpm) * rpmRatio * groupConsumeLevelRatio) - tpm = int64(float64(tpm) * tpmRatio * groupConsumeLevelRatio) - if rpm != mc.RPM || tpm != mc.TPM { - newMc := *mc - newMc.RPM = rpm - newMc.TPM = tpm - return &newMc - } - return mc -} - -var ( - ErrRequestRateLimitExceeded = errors.New("request rate limit exceeded, please try again later") - ErrRequestTpmLimitExceeded = errors.New("request tpm limit exceeded, please try again later") -) - -func checkGroupModelRPMAndTPM(c *gin.Context, group *model.GroupCache, mc *model.ModelConfig) error { - adjustedModelConfig := GetGroupAdjustedModelConfig(group, mc) - - if adjustedModelConfig.RPM > 0 { - ok := rpmlimit.ForceRateLimit( - c.Request.Context(), - group.ID, - mc.Model, - adjustedModelConfig.RPM, - time.Minute, - ) - if !ok { - return ErrRequestRateLimitExceeded - } - } else if common.RedisEnabled { - _, _, err := rpmlimit.PushRequest(c.Request.Context(), group.ID, mc.Model, 1, time.Minute) - if err != nil { - log.Errorf("push request error: %s", err.Error()) - } - } - - if adjustedModelConfig.TPM > 0 { - tpm, err := model.CacheGetGroupModelTPM(group.ID, mc.Model) - if err != nil { - log.Errorf("get group model tpm (%s:%s) error: %s", group.ID, mc.Model, err.Error()) - // ignore error - return nil - } - - if tpm >= adjustedModelConfig.TPM { - return ErrRequestTpmLimitExceeded - } - } - return nil -} - -type GroupBalanceConsumer struct { - GroupBalance float64 - Consumer balance.PostGroupConsumer -} - -func GetGroupBalanceConsumer(c *gin.Context, group *model.GroupCache) (*GroupBalanceConsumer, error) { - gbcI, ok := c.Get(ctxkey.GroupBalance) - if ok { - groupBalanceConsumer, ok := gbcI.(*GroupBalanceConsumer) - if !ok { - return nil, errors.New("internal error: group balance consumer unavailable") - } - return groupBalanceConsumer, nil - } - - var groupBalance float64 - var consumer balance.PostGroupConsumer - - if group.Status == model.GroupStatusInternal { - groupBalance, consumer, _ = balance.MockGetGroupRemainBalance(c.Request.Context(), *group) - } else { - log := GetLogger(c) - var err error - groupBalance, consumer, err = balance.GetGroupRemainBalance(c.Request.Context(), *group) - if err != nil { - return nil, err - } - log.Data["balance"] = strconv.FormatFloat(groupBalance, 'f', -1, 64) - } - - gbc := &GroupBalanceConsumer{GroupBalance: groupBalance, Consumer: consumer} - c.Set(ctxkey.GroupBalance, gbc) - return gbc, nil -} - -func checkGroupBalance(c *gin.Context, group *model.GroupCache) bool { - gbc, err := GetGroupBalanceConsumer(c, group) - if err != nil { - if errors.Is(err, balance.ErrNoRealNameUsedAmountLimit) { - abortLogWithMessage(c, http.StatusForbidden, err.Error(), &errorField{ - Code: "no_real_name_used_amount_limit", - }) - return false - } - notify.ErrorThrottle("balance", time.Minute, fmt.Sprintf("get group (%s) balance error", group.ID), err.Error()) - abortWithMessage(c, http.StatusInternalServerError, fmt.Sprintf("get group (%s) balance error", group.ID), &errorField{ - Code: "get_group_balance_error", - }) - return false - } - - if gbc.GroupBalance <= 0 { - abortLogWithMessage(c, http.StatusForbidden, fmt.Sprintf("group (%s) balance not enough", group.ID), &errorField{ - Code: "group_balance_not_enough", - }) - return false - } - return true -} - -func NewDistribute(mode relaymode.Mode) gin.HandlerFunc { - return func(c *gin.Context) { - distribute(c, mode) - } -} - -func distribute(c *gin.Context, mode relaymode.Mode) { - if config.GetDisableServe() { - abortLogWithMessage(c, http.StatusServiceUnavailable, "service is under maintenance") - return - } - - log := GetLogger(c) - - group := GetGroup(c) - - if !checkGroupBalance(c, group) { - return - } - - requestModel, err := getRequestModel(c, mode) - if err != nil { - abortLogWithMessage(c, http.StatusInternalServerError, err.Error(), &errorField{ - Type: "invalid_request_error", - Code: "get_request_model_error", - }) - return - } - if requestModel == "" { - abortLogWithMessage(c, http.StatusBadRequest, "no model provided", &errorField{ - Type: "invalid_request_error", - Code: "no_model_provided", - }) - return - } - - c.Set(ctxkey.OriginalModel, requestModel) - - SetLogModelFields(log.Data, requestModel) - - token := GetToken(c) - mc, ok := GetModelCaches(c).ModelConfig.GetModelConfig(requestModel) - if !ok || len(token.Models) == 0 || !slices.Contains(token.Models, requestModel) { - abortLogWithMessage(c, - http.StatusNotFound, - fmt.Sprintf("The model `%s` does not exist or you do not have access to it.", requestModel), - &errorField{ - Type: "invalid_request_error", - Code: "model_not_found", - }, - ) - return - } - c.Set(ctxkey.ModelConfig, mc) - - if err := checkGroupModelRPMAndTPM(c, group, mc); err != nil { - errMsg := err.Error() - consume.AsyncConsume( - nil, - http.StatusTooManyRequests, - nil, - NewMetaByContext(c, nil, mc.Model, mode), - 0, - 0, - errMsg, - c.ClientIP(), - 0, - nil, - ) - abortLogWithMessage(c, http.StatusTooManyRequests, errMsg, &errorField{ - Type: "invalid_request_error", - Code: "request_rate_limit_exceeded", - }) - return - } - - c.Next() -} - -func GetOriginalModel(c *gin.Context) string { - return c.GetString(ctxkey.OriginalModel) -} - -func GetModelConfig(c *gin.Context) *model.ModelConfig { - return c.MustGet(ctxkey.ModelConfig).(*model.ModelConfig) -} - -func NewMetaByContext(c *gin.Context, - channel *model.Channel, - modelName string, - mode relaymode.Mode, - opts ...meta.Option, -) *meta.Meta { - requestID := GetRequestID(c) - group := GetGroup(c) - token := GetToken(c) - - opts = append( - opts, - meta.WithRequestID(requestID), - meta.WithGroup(group), - meta.WithToken(token), - meta.WithEndpoint(c.Request.URL.Path), - ) - - return meta.NewMeta( - channel, - mode, - modelName, - GetModelConfig(c), - opts..., - ) -} - -type ModelRequest struct { - Model string `form:"model" json:"model"` -} - -func getRequestModel(c *gin.Context, mode relaymode.Mode) (string, error) { - path := c.Request.URL.Path - switch { - case mode == relaymode.ParsePdf: - query := c.Request.URL.Query() - model := query.Get("model") - if model != "" { - return model, nil - } - - fallthrough - case mode == relaymode.AudioTranscription, - mode == relaymode.AudioTranslation: - return c.Request.FormValue("model"), nil - - case strings.HasPrefix(path, "/v1/engines") && strings.HasSuffix(path, "/embeddings"): - // /engines/:model/embeddings - return c.Param("model"), nil - - default: - body, err := common.GetRequestBody(c.Request) - if err != nil { - return "", fmt.Errorf("get request model failed: %w", err) - } - return GetModelFromJSON(body) - } -} - -func GetModelFromJSON(body []byte) (string, error) { - node, err := sonic.GetWithOptions(body, ast.SearchOptions{}, "model") - if err != nil { - if errors.Is(err, ast.ErrNotExist) { - return "", nil - } - return "", fmt.Errorf("get request model failed: %w", err) - } - return node.String() -} diff --git a/service/aiproxy/middleware/distributor_test.go b/service/aiproxy/middleware/distributor_test.go deleted file mode 100644 index e7d91154a..000000000 --- a/service/aiproxy/middleware/distributor_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package middleware_test - -import ( - "encoding/json" - "strings" - "testing" - - jsoniter "github.com/json-iterator/go" - "github.com/labring/sealos/service/aiproxy/middleware" -) - -func StdGetModelFromJSON(body []byte) (string, error) { - var modelRequest middleware.ModelRequest - err := json.Unmarshal(body, &modelRequest) - if err != nil { - return "", err - } - return modelRequest.Model, nil -} - -func JSONIterGetModelFromJSON(body []byte) (string, error) { - return jsoniter.Get(body, "model").ToString(), nil -} - -func BenchmarkCompareGetModelFromJSON(b *testing.B) { - tests := []struct { - name string - json string - }{ - { - name: "ValidModel", - json: `{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}`, - }, - { - name: "NoModel", - json: `{"messages": [{"role": "user", "content": "Hello"}]}`, - }, - { - name: "EmptyJSON", - json: `{}`, - }, - { - name: "LargeJSON", - json: `{"model": "gpt-4","messages": [{"role": "user", "content": "` + strings.Repeat("x", 1000) + `"}]}`, - }, - { - name: "LargeJSON2", - json: `{"messages": [{"role": "user", "content": "` + strings.Repeat("x", 1000) + `"}],"model": "gpt-4"}`, - }, - { - name: "VeryLargeJSON", - json: `{"model": "gpt-4","messages": [{"role": "user", "content": "` + strings.Repeat("x", 10000) + `"}]}`, - }, - { - name: "VeryLargeJSON2", - json: `{"messages": [{"role": "user", "content": "` + strings.Repeat("x", 10000) + `"}],"model": "gpt-4"}`, - }, - } - - for _, tt := range tests { - jsonBytes := []byte(tt.json) - - b.Run(tt.name+"/Std", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = StdGetModelFromJSON(jsonBytes) - } - }) - - b.Run(tt.name+"/JSONIter", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = JSONIterGetModelFromJSON(jsonBytes) - } - }) - - b.Run(tt.name+"/Sonic", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = middleware.GetModelFromJSON(jsonBytes) - } - }) - } -} diff --git a/service/aiproxy/middleware/log.go b/service/aiproxy/middleware/log.go deleted file mode 100644 index 2af86d17b..000000000 --- a/service/aiproxy/middleware/log.go +++ /dev/null @@ -1,110 +0,0 @@ -package middleware - -import ( - "fmt" - "net/http" - "sync" - "time" - - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" -) - -var fieldsPool = sync.Pool{ - New: func() interface{} { - return make(logrus.Fields, 6) - }, -} - -func NewLog(l *logrus.Logger) gin.HandlerFunc { - return func(c *gin.Context) { - fields := fieldsPool.Get().(logrus.Fields) - defer func() { - clear(fields) - fieldsPool.Put(fields) - }() - - entry := &logrus.Entry{ - Logger: l, - Data: fields, - } - c.Set("log", entry) - - start := time.Now() - path := c.Request.URL.Path - raw := c.Request.URL.RawQuery - - c.Next() - - param := gin.LogFormatterParams{ - Request: c.Request, - Keys: c.Keys, - } - - // Stop timer - param.Latency = time.Since(start) - - param.ClientIP = c.ClientIP() - param.Method = c.Request.Method - param.StatusCode = c.Writer.Status() - param.ErrorMessage = c.Errors.ByType(gin.ErrorTypePrivate).String() - - param.BodySize = c.Writer.Size() - - if raw != "" { - path = path + "?" + raw - } - - param.Path = path - - logColor(entry, param) - } -} - -func logColor(log *logrus.Entry, p gin.LogFormatterParams) { - str := formatter(p) - code := p.StatusCode - switch { - case code >= http.StatusBadRequest && code < http.StatusInternalServerError: - log.Error(str) - default: - log.Info(str) - } -} - -func formatter(param gin.LogFormatterParams) string { - var statusColor, methodColor, resetColor string - if param.IsOutputColor() { - statusColor = param.StatusCodeColor() - methodColor = param.MethodColor() - resetColor = param.ResetColor() - } - - if param.Latency > time.Minute { - param.Latency = param.Latency.Truncate(time.Second) - } - return fmt.Sprintf("[GIN] |%s %3d %s| %13v | %15s |%s %-7s %s %#v\n%s", - statusColor, param.StatusCode, resetColor, - param.Latency, - param.ClientIP, - methodColor, param.Method, resetColor, - param.Path, - param.ErrorMessage, - ) -} - -func GetLogger(c *gin.Context) *logrus.Entry { - if log, ok := c.Get("log"); ok { - return log.(*logrus.Entry) - } - entry := NewLogger() - c.Set("log", entry) - return entry -} - -func NewLogger() *logrus.Entry { - return &logrus.Entry{ - Logger: logrus.StandardLogger(), - Data: fieldsPool.Get().(logrus.Fields), - } -} diff --git a/service/aiproxy/middleware/request-id.go b/service/aiproxy/middleware/request-id.go deleted file mode 100644 index 17ba4516b..000000000 --- a/service/aiproxy/middleware/request-id.go +++ /dev/null @@ -1,30 +0,0 @@ -package middleware - -import ( - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/ctxkey" - "github.com/labring/sealos/service/aiproxy/common/random" -) - -func GenRequestID() string { - return strconv.FormatInt(time.Now().UnixMilli(), 10) + random.GetRandomNumberString(4) -} - -func SetRequestID(c *gin.Context, id string) { - c.Set(ctxkey.RequestID, id) - c.Header(ctxkey.RequestID, id) - log := GetLogger(c) - SetLogRequestIDField(log.Data, id) -} - -func GetRequestID(c *gin.Context) string { - return c.GetString(ctxkey.RequestID) -} - -func RequestID(c *gin.Context) { - id := GenRequestID() - SetRequestID(c, id) -} diff --git a/service/aiproxy/middleware/utils.go b/service/aiproxy/middleware/utils.go deleted file mode 100644 index fa733999e..000000000 --- a/service/aiproxy/middleware/utils.go +++ /dev/null @@ -1,45 +0,0 @@ -package middleware - -import ( - "fmt" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -const ( - ErrorTypeAIPROXY = "aiproxy_error" -) - -func MessageWithRequestID(c *gin.Context, message string) string { - return fmt.Sprintf("%s (aiproxy: %s)", message, GetRequestID(c)) -} - -func abortLogWithMessage(c *gin.Context, statusCode int, message string, fields ...*errorField) { - GetLogger(c).Error(message) - abortWithMessage(c, statusCode, message, fields...) -} - -type errorField struct { - Type string `json:"type"` - Code any `json:"code"` -} - -func abortWithMessage(c *gin.Context, statusCode int, message string, fields ...*errorField) { - typeName := ErrorTypeAIPROXY - var code any = nil - if len(fields) > 0 { - if fields[0].Type != "" { - typeName = fields[0].Type - } - code = fields[0].Code - } - c.JSON(statusCode, gin.H{ - "error": &model.Error{ - Message: MessageWithRequestID(c, message), - Type: typeName, - Code: code, - }, - }) - c.Abort() -} diff --git a/service/aiproxy/model/cache.go b/service/aiproxy/model/cache.go deleted file mode 100644 index 45855e482..000000000 --- a/service/aiproxy/model/cache.go +++ /dev/null @@ -1,706 +0,0 @@ -package model - -import ( - "context" - "encoding" - "errors" - "fmt" - "math/rand/v2" - "slices" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/maruel/natural" - "github.com/redis/go-redis/v9" - log "github.com/sirupsen/logrus" -) - -const ( - SyncFrequency = time.Minute * 3 - TokenCacheKey = "token:%s" - GroupCacheKey = "group:%s" - GroupModelTPMKey = "group:%s:model_tpm" -) - -var ( - _ encoding.BinaryMarshaler = (*redisStringSlice)(nil) - _ redis.Scanner = (*redisStringSlice)(nil) -) - -type redisStringSlice []string - -func (r *redisStringSlice) ScanRedis(value string) error { - return sonic.Unmarshal(conv.StringToBytes(value), r) -} - -func (r redisStringSlice) MarshalBinary() ([]byte, error) { - return sonic.Marshal(r) -} - -type redisTime time.Time - -var ( - _ redis.Scanner = (*redisTime)(nil) - _ encoding.BinaryMarshaler = (*redisTime)(nil) -) - -func (t *redisTime) ScanRedis(value string) error { - return (*time.Time)(t).UnmarshalBinary(conv.StringToBytes(value)) -} - -func (t redisTime) MarshalBinary() ([]byte, error) { - return time.Time(t).MarshalBinary() -} - -type TokenCache struct { - ExpiredAt redisTime `json:"expired_at" redis:"e"` - Group string `json:"group" redis:"g"` - Key string `json:"-" redis:"-"` - Name string `json:"name" redis:"n"` - Subnets redisStringSlice `json:"subnets" redis:"s"` - Models redisStringSlice `json:"models" redis:"m"` - ID int `json:"id" redis:"i"` - Status int `json:"status" redis:"st"` - Quota float64 `json:"quota" redis:"q"` - UsedAmount float64 `json:"used_amount" redis:"u"` -} - -func (t *Token) ToTokenCache() *TokenCache { - return &TokenCache{ - ID: t.ID, - Group: t.GroupID, - Key: t.Key, - Name: t.Name.String(), - Models: t.Models, - Subnets: t.Subnets, - Status: t.Status, - ExpiredAt: redisTime(t.ExpiredAt), - Quota: t.Quota, - UsedAmount: t.UsedAmount, - } -} - -func CacheDeleteToken(key string) error { - if !common.RedisEnabled { - return nil - } - return common.RedisDel(fmt.Sprintf(TokenCacheKey, key)) -} - -//nolint:gosec -func CacheSetToken(token *TokenCache) error { - if !common.RedisEnabled { - return nil - } - key := fmt.Sprintf(TokenCacheKey, token.Key) - pipe := common.RDB.Pipeline() - pipe.HSet(context.Background(), key, token) - expireTime := SyncFrequency + time.Duration(rand.Int64N(60)-30)*time.Second - pipe.Expire(context.Background(), key, expireTime) - _, err := pipe.Exec(context.Background()) - return err -} - -func CacheGetTokenByKey(key string) (*TokenCache, error) { - if !common.RedisEnabled { - token, err := GetTokenByKey(key) - if err != nil { - return nil, err - } - return token.ToTokenCache(), nil - } - - cacheKey := fmt.Sprintf(TokenCacheKey, key) - tokenCache := &TokenCache{} - err := common.RDB.HGetAll(context.Background(), cacheKey).Scan(tokenCache) - if err == nil && tokenCache.ID != 0 { - tokenCache.Key = key - return tokenCache, nil - } else if err != nil && !errors.Is(err, redis.Nil) { - log.Errorf("get token (%s) from redis error: %s", key, err.Error()) - } - - token, err := GetTokenByKey(key) - if err != nil { - return nil, err - } - - tc := token.ToTokenCache() - - if err := CacheSetToken(tc); err != nil { - log.Error("redis set token error: " + err.Error()) - } - - return tc, nil -} - -var updateTokenUsedAmountOnlyIncreaseScript = redis.NewScript(` - local used_amount = redis.call("HGet", KEYS[1], "ua") - if used_amount == false then - return redis.status_reply("ok") - end - if ARGV[1] < used_amount then - return redis.status_reply("ok") - end - redis.call("HSet", KEYS[1], "ua", ARGV[1]) - return redis.status_reply("ok") -`) - -func CacheUpdateTokenUsedAmountOnlyIncrease(key string, amount float64) error { - if !common.RedisEnabled { - return nil - } - return updateTokenUsedAmountOnlyIncreaseScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(TokenCacheKey, key)}, amount).Err() -} - -var updateTokenNameScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "n") then - redis.call("HSet", KEYS[1], "n", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateTokenName(key string, name string) error { - if !common.RedisEnabled { - return nil - } - return updateTokenNameScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(TokenCacheKey, key)}, name).Err() -} - -var updateTokenStatusScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "st") then - redis.call("HSet", KEYS[1], "st", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateTokenStatus(key string, status int) error { - if !common.RedisEnabled { - return nil - } - return updateTokenStatusScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(TokenCacheKey, key)}, status).Err() -} - -type redisMapStringInt64 map[string]int64 - -var ( - _ redis.Scanner = (*redisMapStringInt64)(nil) - _ encoding.BinaryMarshaler = (*redisMapStringInt64)(nil) -) - -func (r *redisMapStringInt64) ScanRedis(value string) error { - return sonic.Unmarshal(conv.StringToBytes(value), r) -} - -func (r redisMapStringInt64) MarshalBinary() ([]byte, error) { - return sonic.Marshal(r) -} - -type GroupCache struct { - ID string `json:"-" redis:"-"` - Status int `json:"status" redis:"st"` - UsedAmount float64 `json:"used_amount" redis:"ua"` - RPMRatio float64 `json:"rpm_ratio" redis:"rpm_r"` - RPM redisMapStringInt64 `json:"rpm" redis:"rpm"` - TPMRatio float64 `json:"tpm_ratio" redis:"tpm_r"` - TPM redisMapStringInt64 `json:"tpm" redis:"tpm"` -} - -func (g *Group) ToGroupCache() *GroupCache { - return &GroupCache{ - ID: g.ID, - Status: g.Status, - UsedAmount: g.UsedAmount, - RPMRatio: g.RPMRatio, - RPM: g.RPM, - TPMRatio: g.TPMRatio, - TPM: g.TPM, - } -} - -func CacheDeleteGroup(id string) error { - if !common.RedisEnabled { - return nil - } - return common.RedisDel(fmt.Sprintf(GroupCacheKey, id)) -} - -var updateGroupRPMRatioScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "rpm_r") then - redis.call("HSet", KEYS[1], "rpm_r", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateGroupRPMRatio(id string, rpmRatio float64) error { - if !common.RedisEnabled { - return nil - } - return updateGroupRPMRatioScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(GroupCacheKey, id)}, rpmRatio).Err() -} - -var updateGroupRPMScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "rpm") then - redis.call("HSet", KEYS[1], "rpm", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateGroupRPM(id string, rpm map[string]int64) error { - if !common.RedisEnabled { - return nil - } - jsonRPM, err := sonic.Marshal(rpm) - if err != nil { - return err - } - return updateGroupRPMScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(GroupCacheKey, id)}, conv.BytesToString(jsonRPM)).Err() -} - -var updateGroupTPMRatioScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "tpm_r") then - redis.call("HSet", KEYS[1], "tpm_r", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateGroupTPMRatio(id string, tpmRatio float64) error { - if !common.RedisEnabled { - return nil - } - return updateGroupTPMRatioScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(GroupCacheKey, id)}, tpmRatio).Err() -} - -var updateGroupTPMScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "tpm") then - redis.call("HSet", KEYS[1], "tpm", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateGroupTPM(id string, tpm map[string]int64) error { - if !common.RedisEnabled { - return nil - } - jsonTPM, err := sonic.Marshal(tpm) - if err != nil { - return err - } - return updateGroupTPMScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(GroupCacheKey, id)}, conv.BytesToString(jsonTPM)).Err() -} - -var updateGroupStatusScript = redis.NewScript(` - if redis.call("HExists", KEYS[1], "st") then - redis.call("HSet", KEYS[1], "st", ARGV[1]) - end - return redis.status_reply("ok") -`) - -func CacheUpdateGroupStatus(id string, status int) error { - if !common.RedisEnabled { - return nil - } - return updateGroupStatusScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(GroupCacheKey, id)}, status).Err() -} - -//nolint:gosec -func CacheSetGroup(group *GroupCache) error { - if !common.RedisEnabled { - return nil - } - key := fmt.Sprintf(GroupCacheKey, group.ID) - pipe := common.RDB.Pipeline() - pipe.HSet(context.Background(), key, group) - expireTime := SyncFrequency + time.Duration(rand.Int64N(60)-30)*time.Second - pipe.Expire(context.Background(), key, expireTime) - _, err := pipe.Exec(context.Background()) - return err -} - -func CacheGetGroup(id string) (*GroupCache, error) { - if !common.RedisEnabled { - group, err := GetGroupByID(id) - if err != nil { - return nil, err - } - return group.ToGroupCache(), nil - } - - cacheKey := fmt.Sprintf(GroupCacheKey, id) - groupCache := &GroupCache{} - err := common.RDB.HGetAll(context.Background(), cacheKey).Scan(groupCache) - if err == nil && groupCache.Status != 0 { - groupCache.ID = id - return groupCache, nil - } else if err != nil && !errors.Is(err, redis.Nil) { - log.Errorf("get group (%s) from redis error: %s", id, err.Error()) - } - - group, err := GetGroupByID(id) - if err != nil { - return nil, err - } - - gc := group.ToGroupCache() - - if err := CacheSetGroup(gc); err != nil { - log.Error("redis set group error: " + err.Error()) - } - - return gc, nil -} - -var updateGroupUsedAmountOnlyIncreaseScript = redis.NewScript(` - local used_amount = redis.call("HGet", KEYS[1], "ua") - if used_amount == false then - return redis.status_reply("ok") - end - if ARGV[1] < used_amount then - return redis.status_reply("ok") - end - redis.call("HSet", KEYS[1], "ua", ARGV[1]) - return redis.status_reply("ok") -`) - -func CacheUpdateGroupUsedAmountOnlyIncrease(id string, amount float64) error { - if !common.RedisEnabled { - return nil - } - return updateGroupUsedAmountOnlyIncreaseScript.Run(context.Background(), common.RDB, []string{fmt.Sprintf(GroupCacheKey, id)}, amount).Err() -} - -//nolint:gosec -func CacheGetGroupModelTPM(id string, model string) (int64, error) { - if !common.RedisEnabled { - return GetGroupModelTPM(id, model) - } - - cacheKey := fmt.Sprintf(GroupModelTPMKey, id) - tpm, err := common.RDB.HGet(context.Background(), cacheKey, model).Int64() - if err == nil { - return tpm, nil - } else if !errors.Is(err, redis.Nil) { - log.Errorf("get group model tpm (%s:%s) from redis error: %s", id, model, err.Error()) - } - - tpm, err = GetGroupModelTPM(id, model) - if err != nil { - return 0, err - } - - pipe := common.RDB.Pipeline() - pipe.HSet(context.Background(), cacheKey, model, tpm) - // 2-5 seconds - pipe.Expire(context.Background(), cacheKey, 2*time.Second+time.Duration(rand.Int64N(3))*time.Second) - _, err = pipe.Exec(context.Background()) - if err != nil { - log.Errorf("set group model tpm (%s:%s) to redis error: %s", id, model, err.Error()) - } - - return tpm, nil -} - -type ModelConfigCache interface { - GetModelConfig(model string) (*ModelConfig, bool) -} - -// read-only cache -// - -type ModelCaches struct { - ModelConfig ModelConfigCache - EnabledModel2channels map[string][]*Channel - EnabledModels []string - EnabledModelsMap map[string]struct{} - EnabledModelConfigs []*ModelConfig - EnabledModelConfigsMap map[string]*ModelConfig - EnabledChannelType2ModelConfigs map[int][]*ModelConfig - EnabledChannelID2channel map[int]*Channel -} - -var modelCaches atomic.Pointer[ModelCaches] - -func init() { - modelCaches.Store(new(ModelCaches)) -} - -func LoadModelCaches() *ModelCaches { - return modelCaches.Load() -} - -// InitModelConfigAndChannelCache initializes the channel cache from database -func InitModelConfigAndChannelCache() error { - modelConfig, err := initializeModelConfigCache() - if err != nil { - return err - } - - // Load enabled newEnabledChannels from database - newEnabledChannels, err := LoadEnabledChannels() - if err != nil { - return err - } - - // Build channel ID to channel map - newEnabledChannelID2channel := buildChannelIDMap(newEnabledChannels) - - // Build all channel ID to channel map - - // Build model to channels map - newEnabledModel2channels := buildModelToChannelsMap(newEnabledChannels) - - // Sort channels by priority - sortChannelsByPriority(newEnabledModel2channels) - - // Build channel type to model configs map - newEnabledChannelType2ModelConfigs := buildChannelTypeToModelConfigsMap(newEnabledChannels, modelConfig) - - // Build enabled models and configs lists - newEnabledModels, newEnabledModelsMap, newEnabledModelConfigs, newEnabledModelConfigsMap := buildEnabledModelsAndConfigs(newEnabledChannelType2ModelConfigs) - - // Update global cache atomically - modelCaches.Store(&ModelCaches{ - ModelConfig: modelConfig, - EnabledModel2channels: newEnabledModel2channels, - EnabledModels: newEnabledModels, - EnabledModelsMap: newEnabledModelsMap, - EnabledModelConfigs: newEnabledModelConfigs, - EnabledModelConfigsMap: newEnabledModelConfigsMap, - EnabledChannelType2ModelConfigs: newEnabledChannelType2ModelConfigs, - EnabledChannelID2channel: newEnabledChannelID2channel, - }) - - return nil -} - -func LoadEnabledChannels() ([]*Channel, error) { - var channels []*Channel - err := DB.Where("status = ?", ChannelStatusEnabled).Find(&channels).Error - if err != nil { - return nil, err - } - - for _, channel := range channels { - initializeChannelModels(channel) - initializeChannelModelMapping(channel) - } - - return channels, nil -} - -func LoadChannels() ([]*Channel, error) { - var channels []*Channel - err := DB.Find(&channels).Error - if err != nil { - return nil, err - } - - for _, channel := range channels { - initializeChannelModels(channel) - initializeChannelModelMapping(channel) - } - - return channels, nil -} - -func LoadChannelByID(id int) (*Channel, error) { - var channel Channel - err := DB.First(&channel, id).Error - if err != nil { - return nil, err - } - - initializeChannelModels(&channel) - initializeChannelModelMapping(&channel) - - return &channel, nil -} - -var _ ModelConfigCache = (*modelConfigMapCache)(nil) - -type modelConfigMapCache struct { - modelConfigMap map[string]*ModelConfig -} - -func (m *modelConfigMapCache) GetModelConfig(model string) (*ModelConfig, bool) { - config, ok := m.modelConfigMap[model] - return config, ok -} - -var _ ModelConfigCache = (*disabledModelConfigCache)(nil) - -type disabledModelConfigCache struct { - modelConfigs ModelConfigCache -} - -func (d *disabledModelConfigCache) GetModelConfig(model string) (*ModelConfig, bool) { - if config, ok := d.modelConfigs.GetModelConfig(model); ok { - return config, true - } - return NewDefaultModelConfig(model), true -} - -func initializeModelConfigCache() (ModelConfigCache, error) { - modelConfigs, err := GetAllModelConfigs() - if err != nil { - return nil, err - } - newModelConfigMap := make(map[string]*ModelConfig) - for _, modelConfig := range modelConfigs { - newModelConfigMap[modelConfig.Model] = modelConfig - } - - configs := &modelConfigMapCache{modelConfigMap: newModelConfigMap} - if config.GetDisableModelConfig() { - return &disabledModelConfigCache{modelConfigs: configs}, nil - } - return configs, nil -} - -func initializeChannelModels(channel *Channel) { - if len(channel.Models) == 0 { - channel.Models = config.GetDefaultChannelModels()[channel.Type] - return - } - - findedModels, missingModels, err := GetModelConfigWithModels(channel.Models) - if err != nil { - return - } - - if len(missingModels) > 0 { - slices.Sort(missingModels) - log.Errorf("model config not found: %v", missingModels) - } - slices.Sort(findedModels) - channel.Models = findedModels -} - -func initializeChannelModelMapping(channel *Channel) { - if len(channel.ModelMapping) == 0 { - channel.ModelMapping = config.GetDefaultChannelModelMapping()[channel.Type] - } -} - -func buildChannelIDMap(channels []*Channel) map[int]*Channel { - channelMap := make(map[int]*Channel) - for _, channel := range channels { - channelMap[channel.ID] = channel - } - return channelMap -} - -func buildModelToChannelsMap(channels []*Channel) map[string][]*Channel { - modelMap := make(map[string][]*Channel) - for _, channel := range channels { - for _, model := range channel.Models { - modelMap[model] = append(modelMap[model], channel) - } - } - return modelMap -} - -func sortChannelsByPriority(modelMap map[string][]*Channel) { - for _, channels := range modelMap { - sort.Slice(channels, func(i, j int) bool { - return channels[i].GetPriority() > channels[j].GetPriority() - }) - } -} - -func buildChannelTypeToModelConfigsMap(channels []*Channel, modelConfigMap ModelConfigCache) map[int][]*ModelConfig { - typeMap := make(map[int][]*ModelConfig) - - for _, channel := range channels { - if _, ok := typeMap[channel.Type]; !ok { - typeMap[channel.Type] = make([]*ModelConfig, 0, len(channel.Models)) - } - configs := typeMap[channel.Type] - - for _, model := range channel.Models { - if config, ok := modelConfigMap.GetModelConfig(model); ok { - configs = append(configs, config) - } - } - typeMap[channel.Type] = configs - } - - for key, configs := range typeMap { - slices.SortStableFunc(configs, SortModelConfigsFunc) - typeMap[key] = slices.CompactFunc(configs, func(e1, e2 *ModelConfig) bool { - return e1.Model == e2.Model - }) - } - return typeMap -} - -func buildEnabledModelsAndConfigs(typeMap map[int][]*ModelConfig) ([]string, map[string]struct{}, []*ModelConfig, map[string]*ModelConfig) { - models := make([]string, 0) - configs := make([]*ModelConfig, 0) - appended := make(map[string]struct{}) - modelConfigsMap := make(map[string]*ModelConfig) - - for _, modelConfigs := range typeMap { - for _, config := range modelConfigs { - if _, ok := appended[config.Model]; ok { - continue - } - models = append(models, config.Model) - configs = append(configs, config) - appended[config.Model] = struct{}{} - modelConfigsMap[config.Model] = config - } - } - - slices.Sort(models) - slices.SortStableFunc(configs, SortModelConfigsFunc) - - return models, appended, configs, modelConfigsMap -} - -func SortModelConfigsFunc(i, j *ModelConfig) int { - if i.Owner != j.Owner { - if natural.Less(string(i.Owner), string(j.Owner)) { - return -1 - } - return 1 - } - if i.Type != j.Type { - if i.Type < j.Type { - return -1 - } - return 1 - } - if i.Model == j.Model { - return 0 - } - if natural.Less(i.Model, j.Model) { - return -1 - } - return 1 -} - -func SyncModelConfigAndChannelCache(ctx context.Context, wg *sync.WaitGroup, frequency time.Duration) { - defer wg.Done() - - ticker := time.NewTicker(frequency) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - err := InitModelConfigAndChannelCache() - if err != nil { - notify.ErrorThrottle("syncModelChannel", time.Minute, "failed to sync channels", err.Error()) - } - } - } -} diff --git a/service/aiproxy/model/channel.go b/service/aiproxy/model/channel.go deleted file mode 100644 index 7e4ca01e1..000000000 --- a/service/aiproxy/model/channel.go +++ /dev/null @@ -1,422 +0,0 @@ -package model - -import ( - "context" - "fmt" - "slices" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/monitor" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -const ( - ErrChannelNotFound = "channel" -) - -const ( - ChannelStatusUnknown = 0 - ChannelStatusEnabled = 1 -) - -type ChannelConfig struct { - SplitThink bool `json:"split_think"` -} - -type Channel struct { - CreatedAt time.Time `gorm:"index" json:"created_at"` - LastTestErrorAt time.Time `json:"last_test_error_at"` - ChannelTests []*ChannelTest `gorm:"foreignKey:ChannelID;references:ID" json:"channel_tests,omitempty"` - BalanceUpdatedAt time.Time `json:"balance_updated_at"` - ModelMapping map[string]string `gorm:"serializer:fastjson;type:text" json:"model_mapping"` - Key string `gorm:"type:text;index" json:"key"` - Name string `gorm:"index" json:"name"` - BaseURL string `gorm:"index" json:"base_url"` - Models []string `gorm:"serializer:fastjson;type:text" json:"models"` - Balance float64 `json:"balance"` - ID int `gorm:"primaryKey" json:"id"` - UsedAmount float64 `gorm:"index" json:"used_amount"` - RequestCount int `gorm:"index" json:"request_count"` - Status int `gorm:"default:1;index" json:"status"` - Type int `gorm:"default:0;index" json:"type"` - Priority int32 `json:"priority"` - EnabledAutoBalanceCheck bool `json:"enabled_auto_balance_check"` - BalanceThreshold float64 `json:"balance_threshold"` - Config *ChannelConfig `gorm:"serializer:fastjson;type:text" json:"config,omitempty"` -} - -func (c *Channel) BeforeDelete(tx *gorm.DB) (err error) { - return tx.Model(&ChannelTest{}).Where("channel_id = ?", c.ID).Delete(&ChannelTest{}).Error -} - -func (c *Channel) GetBalanceThreshold() float64 { - if c.BalanceThreshold < 0 { - return 0 - } - return c.BalanceThreshold -} - -const ( - DefaultPriority = 10 -) - -func (c *Channel) GetPriority() int32 { - if c.Priority == 0 { - return DefaultPriority - } - return c.Priority -} - -func GetModelConfigWithModels(models []string) ([]string, []string, error) { - if len(models) == 0 || config.GetDisableModelConfig() { - return models, nil, nil - } - - where := DB.Model(&ModelConfig{}).Where("model IN ?", models) - var count int64 - if err := where.Count(&count).Error; err != nil { - return nil, nil, err - } - if count == 0 { - return nil, models, nil - } - if count == int64(len(models)) { - return models, nil, nil - } - - var foundModels []string - if err := where.Pluck("model", &foundModels).Error; err != nil { - return nil, nil, err - } - if len(foundModels) == len(models) { - return models, nil, nil - } - foundModelsMap := make(map[string]struct{}, len(foundModels)) - for _, model := range foundModels { - foundModelsMap[model] = struct{}{} - } - if len(models)-len(foundModels) > 0 { - missingModels := make([]string, 0, len(models)-len(foundModels)) - for _, model := range models { - if _, exists := foundModelsMap[model]; !exists { - missingModels = append(missingModels, model) - } - } - return foundModels, missingModels, nil - } - return foundModels, nil, nil -} - -func CheckModelConfigExist(models []string) error { - _, missingModels, err := GetModelConfigWithModels(models) - if err != nil { - return err - } - if len(missingModels) > 0 { - slices.Sort(missingModels) - return fmt.Errorf("model config not found: %v", missingModels) - } - return nil -} - -func (c *Channel) MarshalJSON() ([]byte, error) { - type Alias Channel - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at"` - BalanceUpdatedAt int64 `json:"balance_updated_at"` - LastTestErrorAt int64 `json:"last_test_error_at"` - }{ - Alias: (*Alias)(c), - CreatedAt: c.CreatedAt.UnixMilli(), - BalanceUpdatedAt: c.BalanceUpdatedAt.UnixMilli(), - LastTestErrorAt: c.LastTestErrorAt.UnixMilli(), - }) -} - -//nolint:goconst -func getChannelOrder(order string) string { - prefix, suffix, _ := strings.Cut(order, "-") - switch prefix { - case "name", "type", "created_at", "status", "test_at", "balance_updated_at", "used_amount", "request_count", "priority", "id": - switch suffix { - case "asc": - return prefix + " asc" - default: - return prefix + " desc" - } - default: - return "id desc" - } -} - -func GetAllChannels() (channels []*Channel, err error) { - tx := DB.Model(&Channel{}) - err = tx.Order("id desc").Find(&channels).Error - return channels, err -} - -func GetChannels(page int, perPage int, id int, name string, key string, channelType int, baseURL string, order string) (channels []*Channel, total int64, err error) { - tx := DB.Model(&Channel{}) - if id != 0 { - tx = tx.Where("id = ?", id) - } - if name != "" { - tx = tx.Where("name = ?", name) - } - if key != "" { - tx = tx.Where("key = ?", key) - } - if channelType != 0 { - tx = tx.Where("type = ?", channelType) - } - if baseURL != "" { - tx = tx.Where("base_url = ?", baseURL) - } - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getChannelOrder(order)).Limit(limit).Offset(offset).Find(&channels).Error - return channels, total, err -} - -func SearchChannels(keyword string, page int, perPage int, id int, name string, key string, channelType int, baseURL string, order string) (channels []*Channel, total int64, err error) { - tx := DB.Model(&Channel{}) - - // Handle exact match conditions for non-zero values - if id != 0 { - tx = tx.Where("id = ?", id) - } - if name != "" { - tx = tx.Where("name = ?", name) - } - if key != "" { - tx = tx.Where("key = ?", key) - } - if channelType != 0 { - tx = tx.Where("type = ?", channelType) - } - if baseURL != "" { - tx = tx.Where("base_url = ?", baseURL) - } - - // Handle keyword search for zero value fields - if keyword != "" { - var conditions []string - var values []interface{} - - if id == 0 { - conditions = append(conditions, "id = ?") - values = append(values, String2Int(keyword)) - } - if channelType == 0 { - conditions = append(conditions, "type = ?") - values = append(values, String2Int(keyword)) - } - if name == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "name ILIKE ?") - } else { - conditions = append(conditions, "name LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - if key == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "key ILIKE ?") - } else { - conditions = append(conditions, "key LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - if baseURL == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "base_url ILIKE ?") - } else { - conditions = append(conditions, "base_url LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - - if common.UsingPostgreSQL { - conditions = append(conditions, "models ILIKE ?") - } else { - conditions = append(conditions, "models LIKE ?") - } - values = append(values, "%"+keyword+"%") - - if len(conditions) > 0 { - tx = tx.Where(fmt.Sprintf("(%s)", strings.Join(conditions, " OR ")), values...) - } - } - - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getChannelOrder(order)).Limit(limit).Offset(offset).Find(&channels).Error - return channels, total, err -} - -func GetChannelByID(id int) (*Channel, error) { - channel := Channel{ID: id} - err := DB.First(&channel, "id = ?", id).Error - return &channel, HandleNotFound(err, ErrChannelNotFound) -} - -func BatchInsertChannels(channels []*Channel) (err error) { - defer func() { - if err == nil { - _ = InitModelConfigAndChannelCache() - } - }() - for _, channel := range channels { - if err := CheckModelConfigExist(channel.Models); err != nil { - return err - } - } - return DB.Transaction(func(tx *gorm.DB) error { - return tx.Create(&channels).Error - }) -} - -func UpdateChannel(channel *Channel) (err error) { - defer func() { - if err == nil { - _ = InitModelConfigAndChannelCache() - _ = monitor.ClearChannelAllModelErrors(context.Background(), channel.ID) - } - }() - if err := CheckModelConfigExist(channel.Models); err != nil { - return err - } - result := DB. - Model(channel). - Select( - "model_mapping", - "key", - "name", - "base_url", - "models", - "type", - "priority", - "config", - "enabled_auto_balance_check", - "balance_threshold"). - Clauses(clause.Returning{}). - Where("id = ?", channel.ID). - Updates(channel) - return HandleUpdateResult(result, ErrChannelNotFound) -} - -func ClearLastTestErrorAt(id int) error { - result := DB.Model(&Channel{}).Where("id = ?", id).Update("last_test_error_at", gorm.Expr("NULL")) - return HandleUpdateResult(result, ErrChannelNotFound) -} - -func (c *Channel) UpdateModelTest(testAt time.Time, model, actualModel string, mode relaymode.Mode, took float64, success bool, response string, code int) (*ChannelTest, error) { - var ct *ChannelTest - err := DB.Transaction(func(tx *gorm.DB) error { - if !success { - result := tx.Model(&Channel{}).Where("id = ?", c.ID).Update("last_test_error_at", testAt) - if err := HandleUpdateResult(result, ErrChannelNotFound); err != nil { - return err - } - } else if !c.LastTestErrorAt.IsZero() && time.Since(c.LastTestErrorAt) > time.Hour { - result := tx.Model(&Channel{}).Where("id = ?", c.ID).Update("last_test_error_at", gorm.Expr("NULL")) - if err := HandleUpdateResult(result, ErrChannelNotFound); err != nil { - return err - } - } - ct = &ChannelTest{ - ChannelID: c.ID, - ChannelType: c.Type, - ChannelName: c.Name, - Model: model, - ActualModel: actualModel, - Mode: int(mode), - TestAt: testAt, - Took: took, - Success: success, - Response: response, - Code: code, - } - result := tx.Save(ct) - return HandleUpdateResult(result, ErrChannelNotFound) - }) - if err != nil { - return nil, err - } - return ct, nil -} - -func (c *Channel) UpdateBalance(balance float64) error { - result := DB.Model(&Channel{}). - Select("balance_updated_at", "balance"). - Where("id = ?", c.ID). - Updates(Channel{ - BalanceUpdatedAt: time.Now(), - Balance: balance, - }) - return HandleUpdateResult(result, ErrChannelNotFound) -} - -func DeleteChannelByID(id int) (err error) { - defer func() { - if err == nil { - _ = InitModelConfigAndChannelCache() - _ = monitor.ClearChannelAllModelErrors(context.Background(), id) - } - }() - result := DB.Delete(&Channel{ID: id}) - return HandleUpdateResult(result, ErrChannelNotFound) -} - -func DeleteChannelsByIDs(ids []int) (err error) { - defer func() { - if err == nil { - _ = InitModelConfigAndChannelCache() - for _, id := range ids { - _ = monitor.ClearChannelAllModelErrors(context.Background(), id) - } - } - }() - return DB.Transaction(func(tx *gorm.DB) error { - return tx. - Where("id IN (?)", ids). - Delete(&Channel{}). - Error - }) -} - -func UpdateChannelStatusByID(id int, status int) error { - result := DB.Model(&Channel{}). - Where("id = ?", id). - Update("status", status) - return HandleUpdateResult(result, ErrChannelNotFound) -} - -func UpdateChannelUsedAmount(id int, amount float64, requestCount int) error { - result := DB.Model(&Channel{}). - Where("id = ?", id). - Updates(map[string]interface{}{ - "used_amount": gorm.Expr("used_amount + ?", amount), - "request_count": gorm.Expr("request_count + ?", requestCount), - }) - return HandleUpdateResult(result, ErrChannelNotFound) -} diff --git a/service/aiproxy/model/channeltest.go b/service/aiproxy/model/channeltest.go deleted file mode 100644 index b1678c9f3..000000000 --- a/service/aiproxy/model/channeltest.go +++ /dev/null @@ -1,32 +0,0 @@ -package model - -import ( - "time" - - "github.com/bytedance/sonic" -) - -type ChannelTest struct { - TestAt time.Time `json:"test_at"` - Model string `gorm:"primaryKey" json:"model"` - ActualModel string `json:"actual_model"` - Response string `gorm:"type:text" json:"response"` - ChannelName string `json:"channel_name"` - ChannelType int `json:"channel_type"` - ChannelID int `gorm:"primaryKey" json:"channel_id"` - Took float64 `json:"took"` - Success bool `json:"success"` - Mode int `json:"mode"` - Code int `json:"code"` -} - -func (ct *ChannelTest) MarshalJSON() ([]byte, error) { - type Alias ChannelTest - return sonic.Marshal(&struct { - *Alias - TestAt int64 `json:"test_at"` - }{ - Alias: (*Alias)(ct), - TestAt: ct.TestAt.UnixMilli(), - }) -} diff --git a/service/aiproxy/model/configkey.go b/service/aiproxy/model/configkey.go deleted file mode 100644 index 2cd4757b4..000000000 --- a/service/aiproxy/model/configkey.go +++ /dev/null @@ -1,138 +0,0 @@ -package model - -import "reflect" - -type ModelConfigKey string - -const ( - ModelConfigMaxContextTokensKey ModelConfigKey = "max_context_tokens" - ModelConfigMaxInputTokensKey ModelConfigKey = "max_input_tokens" - ModelConfigMaxOutputTokensKey ModelConfigKey = "max_output_tokens" - ModelConfigVisionKey ModelConfigKey = "vision" - ModelConfigToolChoiceKey ModelConfigKey = "tool_choice" - ModelConfigSupportFormatsKey ModelConfigKey = "support_formats" - ModelConfigSupportVoicesKey ModelConfigKey = "support_voices" -) - -type ModelConfigOption func(config map[ModelConfigKey]any) - -func WithModelConfigMaxContextTokens(maxContextTokens int) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigMaxContextTokensKey] = maxContextTokens - } -} - -func WithModelConfigMaxInputTokens(maxInputTokens int) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigMaxInputTokensKey] = maxInputTokens - } -} - -func WithModelConfigMaxOutputTokens(maxOutputTokens int) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigMaxOutputTokensKey] = maxOutputTokens - } -} - -func WithModelConfigVision(vision bool) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigVisionKey] = vision - } -} - -func WithModelConfigToolChoice(toolChoice bool) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigToolChoiceKey] = toolChoice - } -} - -func WithModelConfigSupportFormats(supportFormats []string) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigSupportFormatsKey] = supportFormats - } -} - -func WithModelConfigSupportVoices(supportVoices []string) ModelConfigOption { - return func(config map[ModelConfigKey]any) { - config[ModelConfigSupportVoicesKey] = supportVoices - } -} - -func NewModelConfig(opts ...ModelConfigOption) map[ModelConfigKey]any { - config := make(map[ModelConfigKey]any) - for _, opt := range opts { - opt(config) - } - return config -} - -func GetModelConfigInt(config map[ModelConfigKey]any, key ModelConfigKey) (int, bool) { - if v, ok := config[key]; ok { - value := reflect.ValueOf(v) - if value.CanInt() { - return int(value.Int()), true - } - if value.CanFloat() { - return int(value.Float()), true - } - } - return 0, false -} - -func GetModelConfigUint(config map[ModelConfigKey]any, key ModelConfigKey) (uint64, bool) { - if v, ok := config[key]; ok { - value := reflect.ValueOf(v) - if value.CanUint() { - return value.Uint(), true - } - if value.CanFloat() { - return uint64(value.Float()), true - } - } - return 0, false -} - -func GetModelConfigFloat(config map[ModelConfigKey]any, key ModelConfigKey) (float64, bool) { - if v, ok := config[key]; ok { - value := reflect.ValueOf(v) - if value.CanFloat() { - return value.Float(), true - } - if value.CanInt() { - return float64(value.Int()), true - } - if value.CanUint() { - return float64(value.Uint()), true - } - } - return 0, false -} - -func GetModelConfigStringSlice(config map[ModelConfigKey]any, key ModelConfigKey) ([]string, bool) { - v, ok := config[key] - if !ok { - return nil, false - } - if slice, ok := v.([]string); ok { - return slice, true - } - if slice, ok := v.([]any); ok { - result := make([]string, len(slice)) - for i, v := range slice { - if s, ok := v.(string); ok { - result[i] = s - continue - } - return nil, false - } - return result, true - } - return nil, false -} - -func GetModelConfigBool(config map[ModelConfigKey]any, key ModelConfigKey) (bool, bool) { - if v, ok := config[key].(bool); ok { - return v, true - } - return false, false -} diff --git a/service/aiproxy/model/consumeerr.go b/service/aiproxy/model/consumeerr.go deleted file mode 100644 index 3df5f4b31..000000000 --- a/service/aiproxy/model/consumeerr.go +++ /dev/null @@ -1,145 +0,0 @@ -package model - -import ( - "fmt" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" -) - -type ConsumeError struct { - RequestAt time.Time `gorm:"index;index:idx_consume_error_group_reqat,priority:2" json:"request_at"` - CreatedAt time.Time `json:"created_at"` - GroupID string `gorm:"index;index:idx_consume_error_group_reqat,priority:1" json:"group_id"` - RequestID string `gorm:"index" json:"request_id"` - TokenName EmptyNullString `gorm:"not null" json:"token_name"` - Model string `json:"model"` - Content string `gorm:"type:text" json:"content"` - ID int `gorm:"primaryKey" json:"id"` - UsedAmount float64 `json:"used_amount"` - TokenID int `json:"token_id"` -} - -func (c *ConsumeError) MarshalJSON() ([]byte, error) { - type Alias ConsumeError - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at"` - RequestAt int64 `json:"request_at"` - }{ - Alias: (*Alias)(c), - CreatedAt: c.CreatedAt.UnixMilli(), - RequestAt: c.RequestAt.UnixMilli(), - }) -} - -func CreateConsumeError(requestID string, requestAt time.Time, group string, tokenName string, model string, content string, usedAmount float64, tokenID int) error { - return LogDB.Create(&ConsumeError{ - RequestID: requestID, - RequestAt: requestAt, - GroupID: group, - TokenName: EmptyNullString(tokenName), - Model: model, - Content: content, - UsedAmount: usedAmount, - TokenID: tokenID, - }).Error -} - -func SearchConsumeError(keyword string, requestID string, group string, tokenName string, model string, content string, usedAmount float64, tokenID int, page int, perPage int, order string) ([]*ConsumeError, int64, error) { - tx := LogDB.Model(&ConsumeError{}) - - // Handle exact match conditions for non-zero values - if group != "" { - tx = tx.Where("group_id = ?", group) - } - if requestID != "" { - tx = tx.Where("request_id = ?", requestID) - } - if tokenName != "" { - tx = tx.Where("token_name = ?", tokenName) - } - if model != "" { - tx = tx.Where("model = ?", model) - } - if content != "" { - tx = tx.Where("content = ?", content) - } - if usedAmount > 0 { - tx = tx.Where("used_amount = ?", usedAmount) - } - if tokenID != 0 { - tx = tx.Where("token_id = ?", tokenID) - } - - // Handle keyword search for zero value fields - if keyword != "" { - var conditions []string - var values []interface{} - - if tokenID == 0 { - conditions = append(conditions, "token_id = ?") - values = append(values, String2Int(keyword)) - } - if requestID == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "request_id ILIKE ?") - } else { - conditions = append(conditions, "request_id LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - if group == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "group_id ILIKE ?") - } else { - conditions = append(conditions, "group_id LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - if tokenName == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "token_name ILIKE ?") - } else { - conditions = append(conditions, "token_name LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - if model == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "model ILIKE ?") - } else { - conditions = append(conditions, "model LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - if content == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "content ILIKE ?") - } else { - conditions = append(conditions, "content LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - - if len(conditions) > 0 { - tx = tx.Where(fmt.Sprintf("(%s)", strings.Join(conditions, " OR ")), values...) - } - } - - var total int64 - err := tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - - var errors []*ConsumeError - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getLogOrder(order)).Limit(limit).Offset(offset).Find(&errors).Error - return errors, total, err -} diff --git a/service/aiproxy/model/group.go b/service/aiproxy/model/group.go deleted file mode 100644 index b983374c5..000000000 --- a/service/aiproxy/model/group.go +++ /dev/null @@ -1,268 +0,0 @@ -package model - -import ( - "errors" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - log "github.com/sirupsen/logrus" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -const ( - ErrGroupNotFound = "group" -) - -const ( - GroupStatusEnabled = 1 - GroupStatusDisabled = 2 - GroupStatusInternal = 3 -) - -type Group struct { - CreatedAt time.Time `json:"created_at"` - ID string `gorm:"primaryKey" json:"id"` - Tokens []*Token `gorm:"foreignKey:GroupID" json:"-"` - Status int `gorm:"default:1;index" json:"status"` - UsedAmount float64 `gorm:"index" json:"used_amount"` - RPMRatio float64 `gorm:"index" json:"rpm_ratio"` - RPM map[string]int64 `gorm:"serializer:fastjson" json:"rpm"` - TPMRatio float64 `gorm:"index" json:"tpm_ratio"` - TPM map[string]int64 `gorm:"serializer:fastjson" json:"tpm"` - RequestCount int `gorm:"index" json:"request_count"` -} - -func (g *Group) BeforeDelete(tx *gorm.DB) (err error) { - return tx.Model(&Token{}).Where("group_id = ?", g.ID).Delete(&Token{}).Error -} - -func getGroupOrder(order string) string { - prefix, suffix, _ := strings.Cut(order, "-") - switch prefix { - case "id", "request_count", "status", "created_at", "used_amount": - switch suffix { - case "asc": - return prefix + " asc" - default: - return prefix + " desc" - } - default: - return "id desc" - } -} - -func GetGroups(page int, perPage int, order string, onlyDisabled bool) (groups []*Group, total int64, err error) { - tx := DB.Model(&Group{}) - if onlyDisabled { - tx = tx.Where("status = ?", GroupStatusDisabled) - } - - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getGroupOrder(order)).Limit(limit).Offset(offset).Find(&groups).Error - return groups, total, err -} - -func GetGroupByID(id string) (*Group, error) { - if id == "" { - return nil, errors.New("group id is empty") - } - group := Group{ID: id} - err := DB.First(&group, "id = ?", id).Error - return &group, HandleNotFound(err, ErrGroupNotFound) -} - -func DeleteGroupByID(id string) (err error) { - if id == "" { - return errors.New("group id is empty") - } - defer func() { - if err == nil { - if err := CacheDeleteGroup(id); err != nil { - log.Error("cache delete group failed: " + err.Error()) - } - if _, err := DeleteGroupLogs(id); err != nil { - log.Error("delete group logs failed: " + err.Error()) - } - } - }() - result := DB.Delete(&Group{ID: id}) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func DeleteGroupsByIDs(ids []string) (err error) { - if len(ids) == 0 { - return nil - } - groups := make([]Group, len(ids)) - defer func() { - if err == nil { - for _, group := range groups { - if err := CacheDeleteGroup(group.ID); err != nil { - log.Error("cache delete group failed: " + err.Error()) - } - if _, err := DeleteGroupLogs(group.ID); err != nil { - log.Error("delete group logs failed: " + err.Error()) - } - } - } - }() - return DB.Transaction(func(tx *gorm.DB) error { - return tx. - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "id"}, - }, - }). - Where("id IN (?)", ids). - Delete(&groups). - Error - }) -} - -func UpdateGroup(id string, group *Group) (err error) { - if id == "" { - return errors.New("group id is empty") - } - defer func() { - if err == nil { - if err := CacheDeleteGroup(id); err != nil { - log.Error("cache delete group failed: " + err.Error()) - } - } - }() - result := DB. - Clauses(clause.Returning{}). - Where("id = ?", id). - Select("rpm_ratio", "rpm", "tpm_ratio", "tpm"). - Updates(group) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func UpdateGroupUsedAmountAndRequestCount(id string, amount float64, count int) (err error) { - group := &Group{} - defer func() { - if amount > 0 && err == nil { - if err := CacheUpdateGroupUsedAmountOnlyIncrease(group.ID, group.UsedAmount); err != nil { - log.Error("update group used amount in cache failed: " + err.Error()) - } - } - }() - result := DB. - Model(group). - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "used_amount"}, - }, - }). - Where("id = ?", id). - Updates(map[string]interface{}{ - "used_amount": gorm.Expr("used_amount + ?", amount), - "request_count": gorm.Expr("request_count + ?", count), - }) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func UpdateGroupRPMRatio(id string, rpmRatio float64) (err error) { - defer func() { - if err == nil { - if err := CacheUpdateGroupRPMRatio(id, rpmRatio); err != nil { - log.Error("cache update group rpm failed: " + err.Error()) - } - } - }() - result := DB.Model(&Group{}).Where("id = ?", id).Update("rpm_ratio", rpmRatio) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func UpdateGroupRPM(id string, rpm map[string]int64) (err error) { - defer func() { - if err == nil { - if err := CacheUpdateGroupRPM(id, rpm); err != nil { - log.Error("cache update group rpm failed: " + err.Error()) - } - } - }() - jsonRpm, err := sonic.Marshal(rpm) - if err != nil { - return err - } - result := DB.Model(&Group{}).Where("id = ?", id).Update("rpm", jsonRpm) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func UpdateGroupTPMRatio(id string, tpmRatio float64) (err error) { - defer func() { - if err == nil { - if err := CacheUpdateGroupTPMRatio(id, tpmRatio); err != nil { - log.Error("cache update group tpm ratio failed: " + err.Error()) - } - } - }() - result := DB.Model(&Group{}).Where("id = ?", id).Update("tpm_ratio", tpmRatio) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func UpdateGroupTPM(id string, tpm map[string]int64) (err error) { - defer func() { - if err == nil { - if err := CacheUpdateGroupTPM(id, tpm); err != nil { - log.Error("cache update group tpm failed: " + err.Error()) - } - } - }() - jsonTpm, err := sonic.Marshal(tpm) - if err != nil { - return err - } - result := DB.Model(&Group{}).Where("id = ?", id).Update("tpm", jsonTpm) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func UpdateGroupStatus(id string, status int) (err error) { - defer func() { - if err == nil { - if err := CacheUpdateGroupStatus(id, status); err != nil { - log.Error("cache update group status failed: " + err.Error()) - } - } - }() - result := DB.Model(&Group{}).Where("id = ?", id).Update("status", status) - return HandleUpdateResult(result, ErrGroupNotFound) -} - -func SearchGroup(keyword string, page int, perPage int, order string, status int) (groups []*Group, total int64, err error) { - tx := DB.Model(&Group{}) - if status != 0 { - tx = tx.Where("status = ?", status) - } - if common.UsingPostgreSQL { - tx = tx.Where("id ILIKE ?", "%"+keyword+"%") - } else { - tx = tx.Where("id LIKE ?", "%"+keyword+"%") - } - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getGroupOrder(order)).Limit(limit).Offset(offset).Find(&groups).Error - return groups, total, err -} - -func CreateGroup(group *Group) error { - return DB.Create(group).Error -} diff --git a/service/aiproxy/model/log.go b/service/aiproxy/model/log.go deleted file mode 100644 index e987a8beb..000000000 --- a/service/aiproxy/model/log.go +++ /dev/null @@ -1,1300 +0,0 @@ -package model - -import ( - "cmp" - "errors" - "fmt" - "slices" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/shopspring/decimal" - "golang.org/x/sync/errgroup" - "gorm.io/gorm" -) - -type RequestDetail struct { - CreatedAt time.Time `gorm:"autoCreateTime;index" json:"-"` - RequestBody string `gorm:"type:text" json:"request_body,omitempty"` - ResponseBody string `gorm:"type:text" json:"response_body,omitempty"` - RequestBodyTruncated bool `json:"request_body_truncated,omitempty"` - ResponseBodyTruncated bool `json:"response_body_truncated,omitempty"` - ID int `gorm:"primaryKey" json:"id"` - LogID int `gorm:"index" json:"log_id"` -} - -func (d *RequestDetail) BeforeSave(_ *gorm.DB) (err error) { - if reqMax := config.GetLogDetailRequestBodyMaxSize(); reqMax > 0 && int64(len(d.RequestBody)) > reqMax { - d.RequestBody = common.TruncateByRune(d.RequestBody, int(reqMax)) + "..." - d.RequestBodyTruncated = true - } - if respMax := config.GetLogDetailResponseBodyMaxSize(); respMax > 0 && int64(len(d.ResponseBody)) > respMax { - d.ResponseBody = common.TruncateByRune(d.ResponseBody, int(respMax)) + "..." - d.ResponseBodyTruncated = true - } - return -} - -type Log struct { - RequestDetail *RequestDetail `gorm:"foreignKey:LogID;constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" json:"request_detail,omitempty"` - RequestAt time.Time `gorm:"index" json:"request_at"` - TimestampTruncByDay int64 `json:"timestamp_trunc_by_day"` - TimestampTruncByHour int64 `json:"timestamp_trunc_by_hour"` - CreatedAt time.Time `gorm:"autoCreateTime;index" json:"created_at"` - TokenName string `json:"token_name,omitempty"` - Endpoint string `json:"endpoint"` - Content string `gorm:"type:text" json:"content,omitempty"` - GroupID string `gorm:"index" json:"group,omitempty"` - Model string `gorm:"index" json:"model"` - RequestID string `gorm:"index" json:"request_id"` - Price float64 `json:"price,omitempty"` - ID int `gorm:"primaryKey" json:"id"` - CompletionPrice float64 `json:"completion_price,omitempty"` - TokenID int `gorm:"index" json:"token_id,omitempty"` - UsedAmount float64 `json:"used_amount,omitempty"` - PromptTokens int `json:"prompt_tokens,omitempty"` - CompletionTokens int `json:"completion_tokens,omitempty"` - TotalTokens int `json:"total_tokens,omitempty"` - ChannelID int `gorm:"index" json:"channel,omitempty"` - Code int `gorm:"index" json:"code,omitempty"` - Mode int `json:"mode,omitempty"` - IP string `gorm:"index" json:"ip,omitempty"` - RetryTimes int `json:"retry_times,omitempty"` -} - -func CreateLogIndexes(db *gorm.DB) error { - var indexes []string - if common.UsingSQLite { - // not support INCLUDE - indexes = []string{ - // used by global search logs - "CREATE INDEX IF NOT EXISTS idx_model_reqat ON logs (model, request_at)", - // used by global search logs - "CREATE INDEX IF NOT EXISTS idx_channel_reqat ON logs (channel_id, request_at)", - // used by global search logs - "CREATE INDEX IF NOT EXISTS idx_channel_model_reqat ON logs (channel_id, model, request_at)", - - // global day indexes, used by global dashboard - "CREATE INDEX IF NOT EXISTS idx_model_reqat_truncday ON logs (model, request_at, timestamp_trunc_by_day)", - // global hour indexes, used by global dashboard - "CREATE INDEX IF NOT EXISTS idx_model_reqat_trunchour ON logs (model, request_at, timestamp_trunc_by_hour)", - - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat ON logs (group_id, token_name, request_at)", - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_model_reqat ON logs (group_id, model, request_at)", - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_token_model_reqat ON logs (group_id, token_name, model, request_at)", - - // day indexes, used by dashboard - "CREATE INDEX IF NOT EXISTS idx_group_reqat_truncday ON logs (group_id, request_at, timestamp_trunc_by_day)", - "CREATE INDEX IF NOT EXISTS idx_group_model_reqat_truncday ON logs (group_id, model, request_at, timestamp_trunc_by_day)", - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat_truncday ON logs (group_id, token_name, request_at, timestamp_trunc_by_day)", - "CREATE INDEX IF NOT EXISTS idx_group_model_token_reqat_truncday ON logs (group_id, model, token_name, request_at, timestamp_trunc_by_day)", - // hour indexes, used by dashboard - "CREATE INDEX IF NOT EXISTS idx_group_reqat_trunchour ON logs (group_id, request_at, timestamp_trunc_by_hour)", - "CREATE INDEX IF NOT EXISTS idx_group_model_reqat_trunchour ON logs (group_id, model, request_at, timestamp_trunc_by_hour)", - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat_trunchour ON logs (group_id, token_name, request_at, timestamp_trunc_by_hour)", - "CREATE INDEX IF NOT EXISTS idx_group_model_token_reqat_trunchour ON logs (group_id, model, token_name, request_at, timestamp_trunc_by_hour)", - } - } else { - indexes = []string{ - // used by global search logs - "CREATE INDEX IF NOT EXISTS idx_model_reqat ON logs (model, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - // used by global search logs - "CREATE INDEX IF NOT EXISTS idx_channel_reqat ON logs (channel_id, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - // used by global search logs - "CREATE INDEX IF NOT EXISTS idx_channel_model_reqat ON logs (channel_id, model, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - - // global day indexes, used by global dashboard - "CREATE INDEX IF NOT EXISTS idx_model_reqat_truncday ON logs (model, request_at, timestamp_trunc_by_day) INCLUDE (code, used_amount, total_tokens)", - // global hour indexes, used by global dashboard - "CREATE INDEX IF NOT EXISTS idx_model_reqat_trunchour ON logs (model, request_at, timestamp_trunc_by_hour) INCLUDE (code, used_amount, total_tokens)", - - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat ON logs (group_id, token_name, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat ON logs (group_id, token_name, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_model_reqat ON logs (group_id, model, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - // used by search group logs - "CREATE INDEX IF NOT EXISTS idx_group_token_model_reqat ON logs (group_id, token_name, model, request_at) INCLUDE (code, used_amount, total_tokens, request_id)", - - // day indexes, used by dashboard - "CREATE INDEX IF NOT EXISTS idx_group_reqat_truncday ON logs (group_id, request_at, timestamp_trunc_by_day) INCLUDE (code, used_amount, total_tokens)", - "CREATE INDEX IF NOT EXISTS idx_group_model_reqat_truncday ON logs (group_id, model, request_at, timestamp_trunc_by_day) INCLUDE (code, used_amount, total_tokens)", - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat_truncday ON logs (group_id, token_name, request_at, timestamp_trunc_by_day) INCLUDE (code, used_amount, total_tokens)", - "CREATE INDEX IF NOT EXISTS idx_group_model_token_reqat_truncday ON logs (group_id, model, token_name, request_at, timestamp_trunc_by_day) INCLUDE (code, used_amount, total_tokens)", - // hour indexes, used by dashboard - "CREATE INDEX IF NOT EXISTS idx_group_reqat_trunchour ON logs (group_id, request_at, timestamp_trunc_by_hour) INCLUDE (code, used_amount, total_tokens)", - "CREATE INDEX IF NOT EXISTS idx_group_model_reqat_trunchour ON logs (group_id, model, request_at, timestamp_trunc_by_hour) INCLUDE (code, used_amount, total_tokens)", - "CREATE INDEX IF NOT EXISTS idx_group_token_reqat_trunchour ON logs (group_id, token_name, request_at, timestamp_trunc_by_hour) INCLUDE (code, used_amount, total_tokens)", - "CREATE INDEX IF NOT EXISTS idx_group_model_token_reqat_trunchour ON logs (group_id, model, token_name, request_at, timestamp_trunc_by_hour) INCLUDE (code, used_amount, total_tokens)", - } - } - - for _, index := range indexes { - if err := db.Exec(index).Error; err != nil { - return err - } - } - - return nil -} - -const ( - contentMaxSize = 2 * 1024 // 2KB -) - -func (l *Log) BeforeSave(_ *gorm.DB) (err error) { - if len(l.Content) > contentMaxSize { - l.Content = common.TruncateByRune(l.Content, contentMaxSize) + "..." - } - if l.TimestampTruncByDay == 0 { - l.TimestampTruncByDay = l.RequestAt.Truncate(24 * time.Hour).Unix() - } - if l.TimestampTruncByHour == 0 { - l.TimestampTruncByHour = l.RequestAt.Truncate(time.Hour).Unix() - } - return -} - -func (l *Log) MarshalJSON() ([]byte, error) { - type Alias Log - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at"` - RequestAt int64 `json:"request_at"` - }{ - Alias: (*Alias)(l), - CreatedAt: l.CreatedAt.UnixMilli(), - RequestAt: l.RequestAt.UnixMilli(), - }) -} - -func GetLogDetail(logID int) (*RequestDetail, error) { - var detail RequestDetail - err := LogDB. - Model(&RequestDetail{}). - Where("log_id = ?", logID). - First(&detail).Error - if err != nil { - return nil, err - } - return &detail, nil -} - -func GetGroupLogDetail(logID int, group string) (*RequestDetail, error) { - if group == "" { - return nil, errors.New("group is required") - } - var detail RequestDetail - err := LogDB. - Model(&RequestDetail{}). - Joins("JOIN logs ON logs.id = request_details.log_id"). - Where("logs.group_id = ?", group). - Where("log_id = ?", logID). - First(&detail).Error - if err != nil { - return nil, err - } - return &detail, nil -} - -const defaultCleanLogBatchSize = 1000 - -func CleanLog(batchSize int) error { - err := cleanLog(batchSize) - if err != nil { - return err - } - return cleanLogDetail(batchSize) -} - -func cleanLog(batchSize int) error { - logStorageHours := config.GetLogStorageHours() - if logStorageHours <= 0 { - return nil - } - if batchSize <= 0 { - batchSize = defaultCleanLogBatchSize - } - return LogDB. - Session(&gorm.Session{SkipDefaultTransaction: true}). - Where( - "created_at < ?", - time.Now().Add(-time.Duration(logStorageHours)*time.Hour), - ). - Limit(batchSize). - Delete(&Log{}).Error -} - -func cleanLogDetail(batchSize int) error { - detailStorageHours := config.GetLogDetailStorageHours() - if detailStorageHours <= 0 { - return nil - } - if batchSize <= 0 { - batchSize = defaultCleanLogBatchSize - } - return LogDB. - Session(&gorm.Session{SkipDefaultTransaction: true}). - Where( - "created_at < ?", - time.Now().Add(-time.Duration(detailStorageHours)*time.Hour), - ). - Limit(batchSize). - Delete(&RequestDetail{}).Error -} - -func RecordConsumeLog( - requestID string, - requestAt time.Time, - group string, - code int, - channelID int, - promptTokens int, - completionTokens int, - modelName string, - tokenID int, - tokenName string, - amount float64, - price float64, - completionPrice float64, - endpoint string, - content string, - mode int, - ip string, - retryTimes int, - requestDetail *RequestDetail, -) error { - log := &Log{ - RequestID: requestID, - RequestAt: requestAt, - GroupID: group, - CreatedAt: time.Now(), - Code: code, - PromptTokens: promptTokens, - CompletionTokens: completionTokens, - TotalTokens: promptTokens + completionTokens, - TokenID: tokenID, - TokenName: tokenName, - Model: modelName, - Mode: mode, - IP: ip, - UsedAmount: amount, - Price: price, - CompletionPrice: completionPrice, - ChannelID: channelID, - Endpoint: endpoint, - Content: content, - RetryTimes: retryTimes, - RequestDetail: requestDetail, - } - return LogDB.Create(log).Error -} - -func getLogOrder(order string) string { - prefix, suffix, _ := strings.Cut(order, "-") - switch prefix { - case "request_at", "id", "created_at": - switch suffix { - case "asc": - return prefix + " asc" - default: - return prefix + " desc" - } - default: - return "request_at desc" - } -} - -type CodeType string - -const ( - CodeTypeAll CodeType = "all" - CodeTypeSuccess CodeType = "success" - CodeTypeError CodeType = "error" -) - -type GetLogsResult struct { - Logs []*Log `json:"logs"` - Total int64 `json:"total"` -} - -type GetGroupLogsResult struct { - GetLogsResult - Models []string `json:"models"` - TokenNames []string `json:"token_names"` -} - -func buildGetLogsQuery( - group string, - startTimestamp time.Time, - endTimestamp time.Time, - modelName string, - requestID string, - tokenID int, - tokenName string, - channelID int, - endpoint string, - mode int, - codeType CodeType, - ip string, -) *gorm.DB { - tx := LogDB.Model(&Log{}) - if group != "" { - tx = tx.Where("group_id = ?", group) - } - if !startTimestamp.IsZero() && !endTimestamp.IsZero() { - tx = tx.Where("request_at BETWEEN ? AND ?", startTimestamp, endTimestamp) - } else if !startTimestamp.IsZero() { - tx = tx.Where("request_at >= ?", startTimestamp) - } else if !endTimestamp.IsZero() { - tx = tx.Where("request_at <= ?", endTimestamp) - } - if tokenName != "" { - tx = tx.Where("token_name = ?", tokenName) - } - if modelName != "" { - tx = tx.Where("model = ?", modelName) - } - if mode != 0 { - tx = tx.Where("mode = ?", mode) - } - if requestID != "" { - tx = tx.Where("request_id = ?", requestID) - } - if tokenID != 0 { - tx = tx.Where("token_id = ?", tokenID) - } - if channelID != 0 { - tx = tx.Where("channel_id = ?", channelID) - } - if endpoint != "" { - tx = tx.Where("endpoint = ?", endpoint) - } - if ip != "" { - tx = tx.Where("ip = ?", ip) - } - switch codeType { - case CodeTypeSuccess: - tx = tx.Where("code = 200") - case CodeTypeError: - tx = tx.Where("code != 200") - } - return tx -} - -func getLogs( - group string, - startTimestamp time.Time, - endTimestamp time.Time, - modelName string, - requestID string, - tokenID int, - tokenName string, - channelID int, - endpoint string, - order string, - mode int, - codeType CodeType, - withBody bool, - ip string, - page int, - perPage int, -) (int64, []*Log, error) { - var total int64 - var logs []*Log - - g := new(errgroup.Group) - - g.Go(func() error { - return buildGetLogsQuery( - group, - startTimestamp, - endTimestamp, - modelName, - requestID, - tokenID, - tokenName, - channelID, - endpoint, - mode, - codeType, - ip, - ).Count(&total).Error - }) - - g.Go(func() error { - query := buildGetLogsQuery( - group, - startTimestamp, - endTimestamp, - modelName, - requestID, - tokenID, - tokenName, - channelID, - endpoint, - mode, - codeType, - ip, - ) - if withBody { - query = query.Preload("RequestDetail") - } else { - query = query.Preload("RequestDetail", func(db *gorm.DB) *gorm.DB { - return db.Select("id", "log_id") - }) - } - - limit, offset := toLimitOffset(page, perPage) - return query. - Order(getLogOrder(order)). - Limit(limit). - Offset(offset). - Find(&logs).Error - }) - - if err := g.Wait(); err != nil { - return 0, nil, err - } - - return total, logs, nil -} - -func GetLogs( - group string, - startTimestamp time.Time, - endTimestamp time.Time, - modelName string, - requestID string, - tokenID int, - tokenName string, - channelID int, - endpoint string, - order string, - mode int, - codeType CodeType, - withBody bool, - ip string, - page int, - perPage int, -) (*GetLogsResult, error) { - var ( - total int64 - logs []*Log - ) - - g := new(errgroup.Group) - - g.Go(func() error { - var err error - total, logs, err = getLogs(group, startTimestamp, endTimestamp, modelName, requestID, tokenID, tokenName, channelID, endpoint, order, mode, codeType, withBody, ip, page, perPage) - return err - }) - - if err := g.Wait(); err != nil { - return nil, err - } - - result := &GetLogsResult{ - Logs: logs, - Total: total, - } - - return result, nil -} - -func GetGroupLogs( - group string, - startTimestamp time.Time, - endTimestamp time.Time, - modelName string, - requestID string, - tokenID int, - tokenName string, - channelID int, - endpoint string, - order string, - mode int, - codeType CodeType, - withBody bool, - ip string, - page int, - perPage int, -) (*GetGroupLogsResult, error) { - if group == "" { - return nil, errors.New("group is required") - } - - var ( - total int64 - logs []*Log - tokenNames []string - models []string - ) - - g := new(errgroup.Group) - - g.Go(func() error { - var err error - total, logs, err = getLogs(group, startTimestamp, endTimestamp, modelName, requestID, tokenID, tokenName, channelID, endpoint, order, mode, codeType, withBody, ip, page, perPage) - return err - }) - - g.Go(func() error { - var err error - tokenNames, err = GetUsedTokenNames(group, startTimestamp, endTimestamp) - return err - }) - - g.Go(func() error { - var err error - models, err = GetUsedModels(group, startTimestamp, endTimestamp) - return err - }) - - if err := g.Wait(); err != nil { - return nil, err - } - - return &GetGroupLogsResult{ - GetLogsResult: GetLogsResult{ - Logs: logs, - Total: total, - }, - Models: models, - TokenNames: tokenNames, - }, nil -} - -func buildSearchLogsQuery( - group string, - keyword string, - endpoint string, - requestID string, - tokenID int, - tokenName string, - modelName string, - startTimestamp time.Time, - endTimestamp time.Time, - channelID int, - mode int, - codeType CodeType, - ip string, -) *gorm.DB { - tx := LogDB.Model(&Log{}) - if group != "" { - tx = tx.Where("group_id = ?", group) - } - - if tokenName != "" { - tx = tx.Where("token_name = ?", tokenName) - } - - if modelName != "" { - tx = tx.Where("model = ?", modelName) - } - - if !startTimestamp.IsZero() && !endTimestamp.IsZero() { - tx = tx.Where("request_at BETWEEN ? AND ?", startTimestamp, endTimestamp) - } else if !startTimestamp.IsZero() { - tx = tx.Where("request_at >= ?", startTimestamp) - } else if !endTimestamp.IsZero() { - tx = tx.Where("request_at <= ?", endTimestamp) - } - - if requestID != "" { - tx = tx.Where("request_id = ?", requestID) - } - if tokenID != 0 { - tx = tx.Where("token_id = ?", tokenID) - } - if channelID != 0 { - tx = tx.Where("channel_id = ?", channelID) - } - - switch codeType { - case CodeTypeSuccess: - tx = tx.Where("code = 200") - case CodeTypeError: - tx = tx.Where("code != 200") - } - - if ip != "" { - tx = tx.Where("ip = ?", ip) - } - - if mode != 0 { - tx = tx.Where("mode = ?", mode) - } - if endpoint != "" { - tx = tx.Where("endpoint = ?", endpoint) - } - - // Handle keyword search for zero value fields - if keyword != "" { - var conditions []string - var values []interface{} - - if group == "" { - conditions = append(conditions, "group_id = ?") - values = append(values, keyword) - } - if tokenName == "" { - conditions = append(conditions, "token_name = ?") - values = append(values, keyword) - } - if modelName == "" { - conditions = append(conditions, "model = ?") - values = append(values, keyword) - } - if requestID == "" { - conditions = append(conditions, "request_id = ?") - values = append(values, keyword) - } - - // if num := String2Int(keyword); num != 0 { - // if channelID == 0 { - // conditions = append(conditions, "channel_id = ?") - // values = append(values, num) - // } - // if mode != 0 { - // conditions = append(conditions, "mode = ?") - // values = append(values, num) - // } - // } - - // if ip != "" { - // conditions = append(conditions, "ip = ?") - // values = append(values, ip) - // } - - // if endpoint == "" { - // if common.UsingPostgreSQL { - // conditions = append(conditions, "endpoint ILIKE ?") - // } else { - // conditions = append(conditions, "endpoint LIKE ?") - // } - // values = append(values, "%"+keyword+"%") - // } - - // slow query - // if common.UsingPostgreSQL { - // conditions = append(conditions, "content ILIKE ?") - // } else { - // conditions = append(conditions, "content LIKE ?") - // } - // values = append(values, "%"+keyword+"%") - - if len(conditions) > 0 { - tx = tx.Where(fmt.Sprintf("(%s)", strings.Join(conditions, " OR ")), values...) - } - } - - return tx -} - -func searchLogs( - group string, - keyword string, - endpoint string, - requestID string, - tokenID int, - tokenName string, - modelName string, - startTimestamp time.Time, - endTimestamp time.Time, - channelID int, - order string, - mode int, - codeType CodeType, - withBody bool, - ip string, - page int, - perPage int, -) (int64, []*Log, error) { - var total int64 - var logs []*Log - - g := new(errgroup.Group) - - g.Go(func() error { - return buildSearchLogsQuery( - group, - keyword, - endpoint, - requestID, - tokenID, - tokenName, - modelName, - startTimestamp, - endTimestamp, - channelID, - mode, - codeType, - ip, - ).Count(&total).Error - }) - - g.Go(func() error { - query := buildSearchLogsQuery( - group, - keyword, - endpoint, - requestID, - tokenID, - tokenName, - modelName, - startTimestamp, - endTimestamp, - channelID, - mode, - codeType, - ip, - ) - - if withBody { - query = query.Preload("RequestDetail") - } else { - query = query.Preload("RequestDetail", func(db *gorm.DB) *gorm.DB { - return db.Select("id", "log_id") - }) - } - - limit, offset := toLimitOffset(page, perPage) - return query. - Order(getLogOrder(order)). - Limit(limit). - Offset(offset). - Find(&logs).Error - }) - - if err := g.Wait(); err != nil { - return 0, nil, err - } - - return total, logs, nil -} - -func SearchLogs( - group string, - keyword string, - endpoint string, - requestID string, - tokenID int, - tokenName string, - modelName string, - startTimestamp time.Time, - endTimestamp time.Time, - channelID int, - order string, - mode int, - codeType CodeType, - withBody bool, - ip string, - page int, - perPage int, -) (*GetLogsResult, error) { - var ( - total int64 - logs []*Log - ) - - g := new(errgroup.Group) - - g.Go(func() error { - var err error - total, logs, err = searchLogs(group, keyword, endpoint, requestID, tokenID, tokenName, modelName, startTimestamp, endTimestamp, channelID, order, mode, codeType, withBody, ip, page, perPage) - return err - }) - - if err := g.Wait(); err != nil { - return nil, err - } - - result := &GetLogsResult{ - Logs: logs, - Total: total, - } - - return result, nil -} - -func SearchGroupLogs( - group string, - keyword string, - endpoint string, - requestID string, - tokenID int, - tokenName string, - modelName string, - startTimestamp time.Time, - endTimestamp time.Time, - channelID int, - order string, - mode int, - codeType CodeType, - withBody bool, - ip string, - page int, - perPage int, -) (*GetGroupLogsResult, error) { - if group == "" { - return nil, errors.New("group is required") - } - - var ( - total int64 - logs []*Log - tokenNames []string - models []string - ) - - g := new(errgroup.Group) - - g.Go(func() error { - var err error - total, logs, err = searchLogs(group, keyword, endpoint, requestID, tokenID, tokenName, modelName, startTimestamp, endTimestamp, channelID, order, mode, codeType, withBody, ip, page, perPage) - return err - }) - - g.Go(func() error { - var err error - tokenNames, err = GetUsedTokenNames(group, startTimestamp, endTimestamp) - return err - }) - - g.Go(func() error { - var err error - models, err = GetUsedModels(group, startTimestamp, endTimestamp) - return err - }) - - if err := g.Wait(); err != nil { - return nil, err - } - - result := &GetGroupLogsResult{ - GetLogsResult: GetLogsResult{ - Logs: logs, - Total: total, - }, - Models: models, - TokenNames: tokenNames, - } - - return result, nil -} - -func DeleteOldLog(timestamp time.Time) (int64, error) { - result := LogDB.Where("request_at < ?", timestamp).Delete(&Log{}) - return result.RowsAffected, result.Error -} - -func DeleteGroupLogs(groupID string) (int64, error) { - if groupID == "" { - return 0, errors.New("group is required") - } - result := LogDB.Where("group_id = ?", groupID).Delete(&Log{}) - return result.RowsAffected, result.Error -} - -type ChartData struct { - Timestamp int64 `json:"timestamp"` - RequestCount int64 `json:"request_count"` - UsedAmount float64 `json:"used_amount"` - ExceptionCount int64 `json:"exception_count"` -} - -type DashboardResponse struct { - ChartData []*ChartData `json:"chart_data"` - TotalCount int64 `json:"total_count"` - ExceptionCount int64 `json:"exception_count"` - UsedAmount float64 `json:"used_amount"` - RPM int64 `json:"rpm"` - TPM int64 `json:"tpm"` -} - -type GroupDashboardResponse struct { - DashboardResponse - Models []string `json:"models"` - TokenNames []string `json:"token_names"` -} - -type TimeSpanType string - -const ( - TimeSpanDay TimeSpanType = "day" - TimeSpanHour TimeSpanType = "hour" -) - -func getTimeSpanFormat(t TimeSpanType) string { - switch t { - case TimeSpanDay: - return "timestamp_trunc_by_day" - case TimeSpanHour: - return "timestamp_trunc_by_hour" - default: - return "" - } -} - -func getChartData(group string, start, end time.Time, tokenName, modelName string, timeSpan TimeSpanType) ([]*ChartData, error) { - var chartData []*ChartData - - timeSpanFormat := getTimeSpanFormat(timeSpan) - if timeSpanFormat == "" { - return nil, errors.New("unsupported time format") - } - - query := LogDB.Table("logs"). - Select(timeSpanFormat + " as timestamp, count(*) as request_count, sum(used_amount) as used_amount, sum(case when code != 200 then 1 else 0 end) as exception_count"). - Group("timestamp"). - Order("timestamp ASC") - - if group != "" { - query = query.Where("group_id = ?", group) - } - - if !start.IsZero() && !end.IsZero() { - query = query.Where("request_at BETWEEN ? AND ?", start, end) - } else if !start.IsZero() { - query = query.Where("request_at >= ?", start) - } else if !end.IsZero() { - query = query.Where("request_at <= ?", end) - } - - if tokenName != "" { - query = query.Where("token_name = ?", tokenName) - } - if modelName != "" { - query = query.Where("model = ?", modelName) - } - - err := query.Scan(&chartData).Error - - return chartData, err -} - -func GetUsedModels(group string, start, end time.Time) ([]string, error) { - return getLogGroupByValues[string]("model", group, start, end) -} - -func GetUsedTokenNames(group string, start, end time.Time) ([]string, error) { - if group == "" { - return nil, errors.New("group is required") - } - return getLogGroupByValues[string]("token_name", group, start, end) -} - -//nolint:unused -func getLogDistinctValues[T cmp.Ordered](field string, group string, start, end time.Time) ([]T, error) { - var values []T - query := LogDB. - Model(&Log{}) - - if group != "" { - query = query.Where("group_id = ?", group) - } - - if !start.IsZero() && !end.IsZero() { - query = query.Where("request_at BETWEEN ? AND ?", start, end) - } else if !start.IsZero() { - query = query.Where("request_at >= ?", start) - } else if !end.IsZero() { - query = query.Where("request_at <= ?", end) - } - - err := query. - Distinct(field). - Pluck(field, &values).Error - if err != nil { - return nil, err - } - slices.Sort(values) - return values, nil -} - -func getLogGroupByValues[T cmp.Ordered](field string, group string, start, end time.Time) ([]T, error) { - var values []T - query := LogDB. - Model(&Log{}) - - if group != "" { - query = query.Where("group_id = ?", group) - } - - if !start.IsZero() && !end.IsZero() { - query = query.Where("request_at BETWEEN ? AND ?", start, end) - } else if !start.IsZero() { - query = query.Where("request_at >= ?", start) - } else if !end.IsZero() { - query = query.Where("request_at <= ?", end) - } - - err := query. - Select(field). - Group(field). - Pluck(field, &values).Error - if err != nil { - return nil, err - } - slices.Sort(values) - return values, nil -} - -func sumTotalCount(chartData []*ChartData) int64 { - var count int64 - for _, data := range chartData { - count += data.RequestCount - } - return count -} - -func sumExceptionCount(chartData []*ChartData) int64 { - var count int64 - for _, data := range chartData { - count += data.ExceptionCount - } - return count -} - -func sumUsedAmount(chartData []*ChartData) float64 { - var amount decimal.Decimal - for _, data := range chartData { - amount = amount.Add(decimal.NewFromFloat(data.UsedAmount)) - } - return amount.InexactFloat64() -} - -func getRPM(group string, end time.Time, tokenName, modelName string) (int64, error) { - query := LogDB.Model(&Log{}) - - if group != "" { - query = query.Where("group_id = ?", group) - } - if tokenName != "" { - query = query.Where("token_name = ?", tokenName) - } - if modelName != "" { - query = query.Where("model = ?", modelName) - } - - var count int64 - err := query. - Where("request_at BETWEEN ? AND ?", end.Add(-time.Minute), end). - Count(&count).Error - return count, err -} - -func getTPM(group string, end time.Time, tokenName, modelName string) (int64, error) { - query := LogDB.Model(&Log{}). - Select("COALESCE(SUM(total_tokens), 0)"). - Where("request_at >= ? AND request_at <= ?", end.Add(-time.Minute), end) - - if group != "" { - query = query.Where("group_id = ?", group) - } - if tokenName != "" { - query = query.Where("token_name = ?", tokenName) - } - if modelName != "" { - query = query.Where("model = ?", modelName) - } - - var tpm int64 - err := query.Scan(&tpm).Error - return tpm, err -} - -func GetDashboardData(start, end time.Time, modelName string, timeSpan TimeSpanType) (*DashboardResponse, error) { - if end.IsZero() { - end = time.Now() - } else if end.Before(start) { - return nil, errors.New("end time is before start time") - } - - var ( - chartData []*ChartData - rpm int64 - tpm int64 - ) - - g := new(errgroup.Group) - - g.Go(func() error { - var err error - chartData, err = getChartData("", start, end, "", modelName, timeSpan) - return err - }) - - g.Go(func() error { - var err error - rpm, err = getRPM("", end, "", modelName) - return err - }) - - g.Go(func() error { - var err error - tpm, err = getTPM("", end, "", modelName) - return err - }) - - if err := g.Wait(); err != nil { - return nil, err - } - - totalCount := sumTotalCount(chartData) - exceptionCount := sumExceptionCount(chartData) - usedAmount := sumUsedAmount(chartData) - - return &DashboardResponse{ - ChartData: chartData, - TotalCount: totalCount, - ExceptionCount: exceptionCount, - UsedAmount: usedAmount, - RPM: rpm, - TPM: tpm, - }, nil -} - -func GetGroupDashboardData(group string, start, end time.Time, tokenName string, modelName string, timeSpan TimeSpanType) (*GroupDashboardResponse, error) { - if group == "" { - return nil, errors.New("group is required") - } - - if end.IsZero() { - end = time.Now() - } else if end.Before(start) { - return nil, errors.New("end time is before start time") - } - - var ( - chartData []*ChartData - tokenNames []string - models []string - rpm int64 - tpm int64 - ) - - g := new(errgroup.Group) - - g.Go(func() error { - var err error - chartData, err = getChartData(group, start, end, tokenName, modelName, timeSpan) - return err - }) - - g.Go(func() error { - var err error - tokenNames, err = GetUsedTokenNames(group, start, end) - return err - }) - - g.Go(func() error { - var err error - models, err = GetUsedModels(group, start, end) - return err - }) - - g.Go(func() error { - var err error - rpm, err = getRPM(group, end, tokenName, modelName) - return err - }) - - g.Go(func() error { - var err error - tpm, err = getTPM(group, end, tokenName, modelName) - return err - }) - - if err := g.Wait(); err != nil { - return nil, err - } - - totalCount := sumTotalCount(chartData) - exceptionCount := sumExceptionCount(chartData) - usedAmount := sumUsedAmount(chartData) - - return &GroupDashboardResponse{ - DashboardResponse: DashboardResponse{ - ChartData: chartData, - TotalCount: totalCount, - ExceptionCount: exceptionCount, - UsedAmount: usedAmount, - RPM: rpm, - TPM: tpm, - }, - Models: models, - TokenNames: tokenNames, - }, nil -} - -func GetGroupLastRequestTime(group string) (time.Time, error) { - if group == "" { - return time.Time{}, errors.New("group is required") - } - var log Log - err := LogDB.Model(&Log{}).Where("group_id = ?", group).Order("request_at desc").First(&log).Error - return log.RequestAt, err -} - -func GetTokenLastRequestTime(id int) (time.Time, error) { - var log Log - tx := LogDB.Model(&Log{}) - err := tx.Where("token_id = ?", id).Order("request_at desc").First(&log).Error - return log.RequestAt, err -} - -func GetGroupModelTPM(group string, model string) (int64, error) { - end := time.Now() - start := end.Add(-time.Minute) - var tpm int64 - err := LogDB. - Model(&Log{}). - Where("group_id = ? AND request_at >= ? AND request_at <= ? AND model = ?", group, start, end, model). - Select("COALESCE(SUM(total_tokens), 0)"). - Scan(&tpm).Error - return tpm, err -} - -type ModelCostRank struct { - Model string `json:"model"` - UsedAmount float64 `json:"used_amount"` - Total int64 `json:"total"` -} - -func GetModelCostRank(group string, start, end time.Time) ([]*ModelCostRank, error) { - var ranks []*ModelCostRank - - query := LogDB.Model(&Log{}). - Select("model, SUM(used_amount) as used_amount, COUNT(*) as total"). - Group("model"). - Order("used_amount DESC") - - if group != "" { - query = query.Where("group_id = ?", group) - } - - if !start.IsZero() && !end.IsZero() { - query = query.Where("request_at BETWEEN ? AND ?", start, end) - } else if !start.IsZero() { - query = query.Where("request_at >= ?", start) - } else if !end.IsZero() { - query = query.Where("request_at <= ?", end) - } - - err := query.Scan(&ranks).Error - if err != nil { - return nil, err - } - - return ranks, nil -} diff --git a/service/aiproxy/model/main.go b/service/aiproxy/model/main.go deleted file mode 100644 index 66dd8d3c0..000000000 --- a/service/aiproxy/model/main.go +++ /dev/null @@ -1,234 +0,0 @@ -package model - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "time" - - "github.com/glebarez/sqlite" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/env" - - // import fastjson serializer - _ "github.com/labring/sealos/service/aiproxy/common/fastJSONSerializer" - log "github.com/sirupsen/logrus" - "gorm.io/driver/mysql" - "gorm.io/driver/postgres" - "gorm.io/gorm" - gormLogger "gorm.io/gorm/logger" -) - -var ( - DB *gorm.DB - LogDB *gorm.DB -) - -func chooseDB(envName string) (*gorm.DB, error) { - dsn := os.Getenv(envName) - - switch { - case strings.HasPrefix(dsn, "postgres"): - // Use PostgreSQL - log.Info("using PostgreSQL as database") - common.UsingPostgreSQL = true - return OpenPostgreSQL(dsn) - case strings.HasPrefix(dsn, "mysql"): - // Use MySQL - log.Info("using MySQL as database") - common.UsingMySQL = true - return OpenMySQL(dsn) - default: - // Use SQLite - absPath, err := filepath.Abs(common.SQLitePath) - if err != nil { - return nil, fmt.Errorf("failed to get absolute path of SQLite database: %w", err) - } - log.Info("SQL_DSN not set, using SQLite as database: ", absPath) - common.UsingSQLite = true - return OpenSQLite(absPath) - } -} - -func newDBLogger() gormLogger.Interface { - var logLevel gormLogger.LogLevel - if config.DebugSQLEnabled { - logLevel = gormLogger.Info - } else { - logLevel = gormLogger.Warn - } - return gormLogger.New( - log.StandardLogger(), - gormLogger.Config{ - SlowThreshold: time.Second, - LogLevel: logLevel, - IgnoreRecordNotFoundError: true, - ParameterizedQueries: !config.DebugSQLEnabled, - Colorful: common.NeedColor(), - }, - ) -} - -func OpenPostgreSQL(dsn string) (*gorm.DB, error) { - return gorm.Open(postgres.New(postgres.Config{ - DSN: dsn, - PreferSimpleProtocol: true, // disables implicit prepared statement usage - }), &gorm.Config{ - PrepareStmt: true, // precompile SQL - TranslateError: true, - Logger: newDBLogger(), - DisableForeignKeyConstraintWhenMigrating: false, - IgnoreRelationshipsWhenMigrating: false, - }) -} - -func OpenMySQL(dsn string) (*gorm.DB, error) { - return gorm.Open(mysql.Open(strings.TrimPrefix(dsn, "mysql://")), &gorm.Config{ - PrepareStmt: true, // precompile SQL - TranslateError: true, - Logger: newDBLogger(), - DisableForeignKeyConstraintWhenMigrating: false, - IgnoreRelationshipsWhenMigrating: false, - }) -} - -func OpenSQLite(sqlitePath string) (*gorm.DB, error) { - baseDir := filepath.Dir(sqlitePath) - if err := os.MkdirAll(baseDir, 0o755); err != nil { - return nil, fmt.Errorf("failed to create base directory: %w", err) - } - - dsn := fmt.Sprintf("%s?_busy_timeout=%d", sqlitePath, common.SQLiteBusyTimeout) - return gorm.Open(sqlite.Open(dsn), &gorm.Config{ - PrepareStmt: true, // precompile SQL - TranslateError: true, - Logger: newDBLogger(), - DisableForeignKeyConstraintWhenMigrating: false, - IgnoreRelationshipsWhenMigrating: false, - }) -} - -func InitDB() { - var err error - DB, err = chooseDB("SQL_DSN") - if err != nil { - log.Fatal("failed to initialize database: " + err.Error()) - return - } - - setDBConns(DB) - - if config.DisableAutoMigrateDB { - return - } - - log.Info("database migration started") - if err = migrateDB(); err != nil { - log.Fatal("failed to migrate database: " + err.Error()) - return - } - log.Info("database migrated") -} - -func migrateDB() error { - err := DB.AutoMigrate( - &Channel{}, - &ChannelTest{}, - &Token{}, - &Group{}, - &Option{}, - &ModelConfig{}, - ) - if err != nil { - return err - } - return nil -} - -func InitLogDB() { - if os.Getenv("LOG_SQL_DSN") == "" { - LogDB = DB - if config.DisableAutoMigrateDB { - return - } - err := migrateLOGDB() - if err != nil { - log.Fatal("failed to migrate secondary database: " + err.Error()) - return - } - log.Info("secondary database migrated") - return - } - - log.Info("using secondary database for table logs") - var err error - LogDB, err = chooseDB("LOG_SQL_DSN") - if err != nil { - log.Fatal("failed to initialize secondary database: " + err.Error()) - return - } - - setDBConns(LogDB) - - if config.DisableAutoMigrateDB { - return - } - - log.Info("secondary database migration started") - err = migrateLOGDB() - if err != nil { - log.Fatal("failed to migrate secondary database: " + err.Error()) - return - } - log.Info("secondary database migrated") -} - -func migrateLOGDB() error { - err := LogDB.AutoMigrate( - &Log{}, - &RequestDetail{}, - &ConsumeError{}, - ) - if err != nil { - return err - } - - return CreateLogIndexes(LogDB) -} - -func setDBConns(db *gorm.DB) { - if config.DebugSQLEnabled { - db = db.Debug() - } - - sqlDB, err := db.DB() - if err != nil { - log.Fatal("failed to connect database: " + err.Error()) - return - } - - sqlDB.SetMaxIdleConns(int(env.Int64("SQL_MAX_IDLE_CONNS", 100))) - sqlDB.SetMaxOpenConns(int(env.Int64("SQL_MAX_OPEN_CONNS", 1000))) - sqlDB.SetConnMaxLifetime(time.Second * time.Duration(env.Int64("SQL_MAX_LIFETIME", 60))) -} - -func closeDB(db *gorm.DB) error { - sqlDB, err := db.DB() - if err != nil { - return err - } - err = sqlDB.Close() - return err -} - -func CloseDB() error { - if LogDB != DB { - err := closeDB(LogDB) - if err != nil { - return err - } - } - return closeDB(DB) -} diff --git a/service/aiproxy/model/modelconfig.go b/service/aiproxy/model/modelconfig.go deleted file mode 100644 index e4a95a7db..000000000 --- a/service/aiproxy/model/modelconfig.go +++ /dev/null @@ -1,201 +0,0 @@ -package model - -import ( - "fmt" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "gorm.io/gorm" -) - -const ( - // /1K tokens - PriceUnit = 1000 -) - -type ModelConfig struct { - CreatedAt time.Time `gorm:"index;autoCreateTime" json:"created_at"` - UpdatedAt time.Time `gorm:"index;autoUpdateTime" json:"updated_at"` - Config map[ModelConfigKey]any `gorm:"serializer:fastjson;type:text" json:"config,omitempty"` - ImagePrices map[string]float64 `gorm:"serializer:fastjson" json:"image_prices,omitempty"` - Model string `gorm:"primaryKey" json:"model"` - Owner ModelOwner `gorm:"type:varchar(255);index" json:"owner"` - ImageMaxBatchSize int `json:"image_batch_size,omitempty"` - Type relaymode.Mode `json:"type"` // relaymode/define.go - InputPrice float64 `json:"input_price,omitempty"` - OutputPrice float64 `json:"output_price,omitempty"` - RPM int64 `json:"rpm,omitempty"` - TPM int64 `json:"tpm,omitempty"` -} - -func NewDefaultModelConfig(model string) *ModelConfig { - return &ModelConfig{ - Model: model, - } -} - -func (c *ModelConfig) MarshalJSON() ([]byte, error) { - type Alias ModelConfig - return sonic.Marshal(&struct { - *Alias - CreatedAt int64 `json:"created_at,omitempty"` - UpdatedAt int64 `json:"updated_at,omitempty"` - }{ - Alias: (*Alias)(c), - CreatedAt: c.CreatedAt.UnixMilli(), - UpdatedAt: c.UpdatedAt.UnixMilli(), - }) -} - -func (c *ModelConfig) MaxContextTokens() (int, bool) { - return GetModelConfigInt(c.Config, ModelConfigMaxContextTokensKey) -} - -func (c *ModelConfig) MaxInputTokens() (int, bool) { - return GetModelConfigInt(c.Config, ModelConfigMaxInputTokensKey) -} - -func (c *ModelConfig) MaxOutputTokens() (int, bool) { - return GetModelConfigInt(c.Config, ModelConfigMaxOutputTokensKey) -} - -func (c *ModelConfig) SupportVision() (bool, bool) { - return GetModelConfigBool(c.Config, ModelConfigVisionKey) -} - -func (c *ModelConfig) SupportVoices() ([]string, bool) { - return GetModelConfigStringSlice(c.Config, ModelConfigSupportVoicesKey) -} - -func (c *ModelConfig) SupportToolChoice() (bool, bool) { - return GetModelConfigBool(c.Config, ModelConfigToolChoiceKey) -} - -func (c *ModelConfig) SupportFormats() ([]string, bool) { - return GetModelConfigStringSlice(c.Config, ModelConfigSupportFormatsKey) -} - -func GetModelConfigs(page int, perPage int, model string) (configs []*ModelConfig, total int64, err error) { - tx := DB.Model(&ModelConfig{}) - if model != "" { - tx = tx.Where("model = ?", model) - } - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order("created_at desc").Limit(limit).Offset(offset).Find(&configs).Error - return configs, total, err -} - -func GetAllModelConfigs() (configs []*ModelConfig, err error) { - tx := DB.Model(&ModelConfig{}) - err = tx.Order("created_at desc").Find(&configs).Error - return configs, err -} - -func GetModelConfigsByModels(models []string) (configs []*ModelConfig, err error) { - tx := DB.Model(&ModelConfig{}).Where("model IN (?)", models) - err = tx.Order("created_at desc").Find(&configs).Error - return configs, err -} - -func GetModelConfig(model string) (*ModelConfig, error) { - config := &ModelConfig{} - err := DB.Model(&ModelConfig{}).Where("model = ?", model).First(config).Error - return config, HandleNotFound(err, ErrModelConfigNotFound) -} - -func SearchModelConfigs(keyword string, page int, perPage int, model string, owner ModelOwner) (configs []*ModelConfig, total int64, err error) { - tx := DB.Model(&ModelConfig{}).Where("model LIKE ?", "%"+keyword+"%") - if model != "" { - tx = tx.Where("model = ?", model) - } - if owner != "" { - tx = tx.Where("owner = ?", owner) - } - if keyword != "" { - var conditions []string - var values []interface{} - - if model == "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "model ILIKE ?") - } else { - conditions = append(conditions, "model LIKE ?") - } - values = append(values, "%"+keyword+"%") - } - - if owner != "" { - if common.UsingPostgreSQL { - conditions = append(conditions, "owner ILIKE ?") - } else { - conditions = append(conditions, "owner LIKE ?") - } - values = append(values, "%"+string(owner)+"%") - } - - if len(conditions) > 0 { - tx = tx.Where(fmt.Sprintf("(%s)", strings.Join(conditions, " OR ")), values...) - } - } - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order("created_at desc").Limit(limit).Offset(offset).Find(&configs).Error - return configs, total, err -} - -func SaveModelConfig(config *ModelConfig) (err error) { - defer func() { - if err == nil { - _ = InitModelConfigAndChannelCache() - } - }() - return DB.Save(config).Error -} - -func SaveModelConfigs(configs []*ModelConfig) (err error) { - defer func() { - if err == nil { - _ = InitModelConfigAndChannelCache() - } - }() - return DB.Transaction(func(tx *gorm.DB) error { - for _, config := range configs { - if err := tx.Save(config).Error; err != nil { - return err - } - } - return nil - }) -} - -const ErrModelConfigNotFound = "model config" - -func DeleteModelConfig(model string) error { - result := DB.Where("model = ?", model).Delete(&ModelConfig{}) - return HandleUpdateResult(result, ErrModelConfigNotFound) -} - -func DeleteModelConfigsByModels(models []string) error { - return DB.Transaction(func(tx *gorm.DB) error { - return tx. - Where("model IN (?)", models). - Delete(&ModelConfig{}). - Error - }) -} diff --git a/service/aiproxy/model/option.go b/service/aiproxy/model/option.go deleted file mode 100644 index 2868f14f5..000000000 --- a/service/aiproxy/model/option.go +++ /dev/null @@ -1,340 +0,0 @@ -package model - -import ( - "context" - "errors" - "fmt" - "slices" - "sort" - "strconv" - "sync" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/notify" - log "github.com/sirupsen/logrus" -) - -type Option struct { - Key string `gorm:"primaryKey" json:"key"` - Value string `json:"value"` -} - -func GetAllOption() ([]*Option, error) { - var options []*Option - err := DB.Where("key IN (?)", optionKeys).Find(&options).Error - return options, err -} - -func GetOption(key string) (*Option, error) { - if !slices.Contains(optionKeys, key) { - return nil, ErrUnknownOptionKey - } - var option Option - err := DB.Where("key = ?", key).First(&option).Error - return &option, err -} - -var ( - optionMap = make(map[string]string) - // allowed option keys - optionKeys []string -) - -func InitOption2DB() error { - err := initOptionMap() - if err != nil { - return err - } - - err = loadOptionsFromDatabase(true) - if err != nil { - return err - } - return storeOptionMap() -} - -func initOptionMap() error { - optionMap["LogStorageHours"] = strconv.FormatInt(config.GetLogStorageHours(), 10) - optionMap["LogDetailStorageHours"] = strconv.FormatInt(config.GetLogDetailStorageHours(), 10) - optionMap["SaveAllLogDetail"] = strconv.FormatBool(config.GetSaveAllLogDetail()) - optionMap["LogDetailRequestBodyMaxSize"] = strconv.FormatInt(config.GetLogDetailRequestBodyMaxSize(), 10) - optionMap["LogDetailResponseBodyMaxSize"] = strconv.FormatInt(config.GetLogDetailResponseBodyMaxSize(), 10) - optionMap["DisableServe"] = strconv.FormatBool(config.GetDisableServe()) - optionMap["BillingEnabled"] = strconv.FormatBool(config.GetBillingEnabled()) - optionMap["RetryTimes"] = strconv.FormatInt(config.GetRetryTimes(), 10) - optionMap["ModelErrorAutoBanRate"] = strconv.FormatFloat(config.GetModelErrorAutoBanRate(), 'f', -1, 64) - optionMap["EnableModelErrorAutoBan"] = strconv.FormatBool(config.GetEnableModelErrorAutoBan()) - timeoutWithModelTypeJSON, err := sonic.Marshal(config.GetTimeoutWithModelType()) - if err != nil { - return err - } - optionMap["TimeoutWithModelType"] = conv.BytesToString(timeoutWithModelTypeJSON) - defaultChannelModelsJSON, err := sonic.Marshal(config.GetDefaultChannelModels()) - if err != nil { - return err - } - optionMap["DefaultChannelModels"] = conv.BytesToString(defaultChannelModelsJSON) - defaultChannelModelMappingJSON, err := sonic.Marshal(config.GetDefaultChannelModelMapping()) - if err != nil { - return err - } - optionMap["DefaultChannelModelMapping"] = conv.BytesToString(defaultChannelModelMappingJSON) - optionMap["GeminiSafetySetting"] = config.GetGeminiSafetySetting() - optionMap["GroupMaxTokenNum"] = strconv.FormatInt(config.GetGroupMaxTokenNum(), 10) - groupConsumeLevelRatioJSON, err := sonic.Marshal(config.GetGroupConsumeLevelRatioStringKeyMap()) - if err != nil { - return err - } - optionMap["GroupConsumeLevelRatio"] = conv.BytesToString(groupConsumeLevelRatioJSON) - optionMap["InternalToken"] = config.GetInternalToken() - optionMap["NotifyNote"] = config.GetNotifyNote() - - optionKeys = make([]string, 0, len(optionMap)) - for key := range optionMap { - optionKeys = append(optionKeys, key) - } - return nil -} - -func storeOptionMap() error { - for key, value := range optionMap { - err := saveOption(key, value) - if err != nil { - return err - } - } - return nil -} - -func loadOptionsFromDatabase(isInit bool) error { - options, err := GetAllOption() - if err != nil { - return err - } - for _, option := range options { - err := updateOption(option.Key, option.Value, isInit) - if err != nil { - if !errors.Is(err, ErrUnknownOptionKey) { - return fmt.Errorf("failed to update option: %s, value: %s, error: %w", option.Key, option.Value, err) - } - if isInit { - log.Warnf("unknown option: %s, value: %s", option.Key, option.Value) - } - continue - } - if isInit { - delete(optionMap, option.Key) - } - } - return nil -} - -func SyncOptions(ctx context.Context, wg *sync.WaitGroup, frequency time.Duration) { - defer wg.Done() - - ticker := time.NewTicker(frequency) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - if err := loadOptionsFromDatabase(false); err != nil { - notify.ErrorThrottle("syncOptions", time.Minute, "failed to sync options", err.Error()) - } - } - } -} - -func saveOption(key string, value string) error { - option := Option{ - Key: key, - Value: value, - } - result := DB.Save(&option) - return HandleUpdateResult(result, "option:"+key) -} - -func UpdateOption(key string, value string) error { - err := updateOption(key, value, false) - if err != nil { - return err - } - return saveOption(key, value) -} - -func UpdateOptions(options map[string]string) error { - errs := make([]error, 0) - for key, value := range options { - err := UpdateOption(key, value) - if err != nil && !errors.Is(err, ErrUnknownOptionKey) { - errs = append(errs, err) - } - } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil -} - -var ErrUnknownOptionKey = errors.New("unknown option key") - -func toBool(value string) bool { - result, _ := strconv.ParseBool(value) - return result -} - -//nolint:gocyclo -func updateOption(key string, value string, isInit bool) (err error) { - switch key { - case "InternalToken": - config.SetInternalToken(value) - case "LogStorageHours": - logStorageHours, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return err - } - config.SetLogStorageHours(logStorageHours) - case "LogDetailStorageHours": - logDetailStorageHours, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return err - } - config.SetLogDetailStorageHours(logDetailStorageHours) - case "SaveAllLogDetail": - config.SetSaveAllLogDetail(toBool(value)) - case "LogDetailRequestBodyMaxSize": - logDetailRequestBodyMaxSize, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return err - } - config.SetLogDetailRequestBodyMaxSize(logDetailRequestBodyMaxSize) - case "LogDetailResponseBodyMaxSize": - logDetailResponseBodyMaxSize, err := strconv.ParseInt(value, 10, 64) - if err != nil { - return err - } - config.SetLogDetailResponseBodyMaxSize(logDetailResponseBodyMaxSize) - case "DisableServe": - config.SetDisableServe(toBool(value)) - case "BillingEnabled": - config.SetBillingEnabled(toBool(value)) - case "GroupMaxTokenNum": - groupMaxTokenNum, err := strconv.ParseInt(value, 10, 32) - if err != nil { - return err - } - if groupMaxTokenNum < 0 { - return errors.New("group max token num must be greater than 0") - } - config.SetGroupMaxTokenNum(groupMaxTokenNum) - case "GeminiSafetySetting": - config.SetGeminiSafetySetting(value) - case "DefaultChannelModels": - var newModels map[int][]string - err := sonic.Unmarshal(conv.StringToBytes(value), &newModels) - if err != nil { - return err - } - // check model config exist - allModelsMap := make(map[string]struct{}) - for _, models := range newModels { - for _, model := range models { - allModelsMap[model] = struct{}{} - } - } - allModels := make([]string, 0, len(allModelsMap)) - for model := range allModelsMap { - allModels = append(allModels, model) - } - foundModels, missingModels, err := GetModelConfigWithModels(allModels) - if err != nil { - return err - } - if !isInit && len(missingModels) > 0 { - sort.Strings(missingModels) - return fmt.Errorf("model config not found: %v", missingModels) - } - if len(missingModels) > 0 { - sort.Strings(missingModels) - log.Errorf("model config not found: %v", missingModels) - } - allowedNewModels := make(map[int][]string) - for t, ms := range newModels { - for _, m := range ms { - if slices.Contains(foundModels, m) { - allowedNewModels[t] = append(allowedNewModels[t], m) - } - } - } - config.SetDefaultChannelModels(allowedNewModels) - case "DefaultChannelModelMapping": - var newMapping map[int]map[string]string - err := sonic.Unmarshal(conv.StringToBytes(value), &newMapping) - if err != nil { - return err - } - config.SetDefaultChannelModelMapping(newMapping) - case "RetryTimes": - retryTimes, err := strconv.ParseInt(value, 10, 32) - if err != nil { - return err - } - if retryTimes < 0 { - return errors.New("retry times must be greater than 0") - } - config.SetRetryTimes(retryTimes) - case "EnableModelErrorAutoBan": - config.SetEnableModelErrorAutoBan(toBool(value)) - case "ModelErrorAutoBanRate": - modelErrorAutoBanRate, err := strconv.ParseFloat(value, 64) - if err != nil { - return err - } - if modelErrorAutoBanRate < 0 || modelErrorAutoBanRate > 1 { - return errors.New("model error auto ban rate must be between 0 and 1") - } - config.SetModelErrorAutoBanRate(modelErrorAutoBanRate) - case "TimeoutWithModelType": - var newTimeoutWithModelType map[int]int64 - err := sonic.Unmarshal(conv.StringToBytes(value), &newTimeoutWithModelType) - if err != nil { - return err - } - for _, v := range newTimeoutWithModelType { - if v < 0 { - return errors.New("timeout must be greater than 0") - } - } - config.SetTimeoutWithModelType(newTimeoutWithModelType) - case "GroupConsumeLevelRatio": - var newGroupRpmRatio map[string]float64 - err := sonic.Unmarshal(conv.StringToBytes(value), &newGroupRpmRatio) - if err != nil { - return err - } - newGroupRpmRatioMap := make(map[float64]float64) - for k, v := range newGroupRpmRatio { - consumeLevel, err := strconv.ParseFloat(k, 64) - if err != nil { - return err - } - if consumeLevel < 0 { - return errors.New("consume level must be greater than 0") - } - if v < 0 { - return errors.New("rpm ratio must be greater than 0") - } - newGroupRpmRatioMap[consumeLevel] = v - } - config.SetGroupConsumeLevelRatio(newGroupRpmRatioMap) - case "NotifyNote": - config.SetNotifyNote(value) - default: - return ErrUnknownOptionKey - } - return err -} diff --git a/service/aiproxy/model/owner.go b/service/aiproxy/model/owner.go deleted file mode 100644 index a44c1a1f5..000000000 --- a/service/aiproxy/model/owner.go +++ /dev/null @@ -1,37 +0,0 @@ -package model - -type ModelOwner string - -const ( - ModelOwnerOpenAI ModelOwner = "openai" - ModelOwnerAlibaba ModelOwner = "alibaba" - ModelOwnerTencent ModelOwner = "tencent" - ModelOwnerXunfei ModelOwner = "xunfei" - ModelOwnerDeepSeek ModelOwner = "deepseek" - ModelOwnerMoonshot ModelOwner = "moonshot" - ModelOwnerMiniMax ModelOwner = "minimax" - ModelOwnerBaidu ModelOwner = "baidu" - ModelOwnerGoogle ModelOwner = "google" - ModelOwnerBAAI ModelOwner = "baai" - ModelOwnerFunAudioLLM ModelOwner = "funaudiollm" - ModelOwnerDoubao ModelOwner = "doubao" - ModelOwnerFishAudio ModelOwner = "fishaudio" - ModelOwnerChatGLM ModelOwner = "chatglm" - ModelOwnerStabilityAI ModelOwner = "stabilityai" - ModelOwnerNetease ModelOwner = "netease" - ModelOwnerAI360 ModelOwner = "ai360" - ModelOwnerAnthropic ModelOwner = "anthropic" - ModelOwnerMeta ModelOwner = "meta" - ModelOwnerBaichuan ModelOwner = "baichuan" - ModelOwnerMistral ModelOwner = "mistral" - ModelOwnerOpenChat ModelOwner = "openchat" - ModelOwnerMicrosoft ModelOwner = "microsoft" - ModelOwnerDefog ModelOwner = "defog" - ModelOwnerNexusFlow ModelOwner = "nexusflow" - ModelOwnerCohere ModelOwner = "cohere" - ModelOwnerHuggingFace ModelOwner = "huggingface" - ModelOwnerLingyiWanwu ModelOwner = "lingyiwanwu" - ModelOwnerStepFun ModelOwner = "stepfun" - ModelOwnerXAI ModelOwner = "xai" - ModelOwnerDoc2x ModelOwner = "doc2x" -) diff --git a/service/aiproxy/model/token.go b/service/aiproxy/model/token.go deleted file mode 100644 index 6db1ec12f..000000000 --- a/service/aiproxy/model/token.go +++ /dev/null @@ -1,504 +0,0 @@ -package model - -import ( - "errors" - "fmt" - "strings" - "time" - - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - log "github.com/sirupsen/logrus" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -const ( - ErrTokenNotFound = "token" -) - -const ( - TokenStatusEnabled = 1 - TokenStatusDisabled = 2 -) - -type Token struct { - CreatedAt time.Time `json:"created_at"` - ExpiredAt time.Time `json:"expired_at"` - Group *Group `gorm:"foreignKey:GroupID" json:"-"` - Key string `gorm:"type:char(48);uniqueIndex" json:"key"` - Name EmptyNullString `gorm:"index;uniqueIndex:idx_group_name;not null" json:"name"` - GroupID string `gorm:"index;uniqueIndex:idx_group_name" json:"group"` - Subnets []string `gorm:"serializer:fastjson;type:text" json:"subnets"` - Models []string `gorm:"serializer:fastjson;type:text" json:"models"` - Status int `gorm:"default:1;index" json:"status"` - ID int `gorm:"primaryKey" json:"id"` - Quota float64 `json:"quota"` - UsedAmount float64 `gorm:"index" json:"used_amount"` - RequestCount int `gorm:"index" json:"request_count"` -} - -func getTokenOrder(order string) string { - prefix, suffix, _ := strings.Cut(order, "-") - switch prefix { - case "name", "expired_at", "group", "used_amount", "request_count", "id", "created_at": - switch suffix { - case "asc": - return prefix + " asc" - default: - return prefix + " desc" - } - default: - return "id desc" - } -} - -func InsertToken(token *Token, autoCreateGroup bool) error { - if autoCreateGroup { - group := &Group{ - ID: token.GroupID, - } - if err := OnConflictDoNothing().Create(group).Error; err != nil { - return err - } - } - maxTokenNum := config.GetGroupMaxTokenNum() - err := DB.Transaction(func(tx *gorm.DB) error { - if maxTokenNum > 0 { - var count int64 - err := tx.Model(&Token{}).Where("group_id = ?", token.GroupID).Count(&count).Error - if err != nil { - return err - } - if count >= maxTokenNum { - return errors.New("group max token num reached") - } - } - return tx.Create(token).Error - }) - if err != nil { - if errors.Is(err, gorm.ErrDuplicatedKey) { - return errors.New("token name already exists in this group") - } - return err - } - return nil -} - -func GetTokens(group string, page int, perPage int, order string, status int) (tokens []*Token, total int64, err error) { - tx := DB.Model(&Token{}) - if group != "" { - tx = tx.Where("group_id = ?", group) - } - - if status != 0 { - tx = tx.Where("status = ?", status) - } - - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getTokenOrder(order)).Limit(limit).Offset(offset).Find(&tokens).Error - return tokens, total, err -} - -func SearchTokens(group string, keyword string, page int, perPage int, order string, status int, name string, key string) (tokens []*Token, total int64, err error) { - tx := DB.Model(&Token{}) - if group != "" { - tx = tx.Where("group_id = ?", group) - } - if status != 0 { - tx = tx.Where("status = ?", status) - } - if name != "" { - tx = tx.Where("name = ?", name) - } - if key != "" { - tx = tx.Where("key = ?", key) - } - - if keyword != "" { - var conditions []string - var values []interface{} - - if group == "" { - conditions = append(conditions, "group_id = ?") - values = append(values, keyword) - } - if status == 0 { - conditions = append(conditions, "status = ?") - values = append(values, String2Int(keyword)) - } - if name == "" { - conditions = append(conditions, "name = ?") - values = append(values, keyword) - } - if key == "" { - conditions = append(conditions, "key = ?") - values = append(values, keyword) - } - - if common.UsingPostgreSQL { - conditions = append(conditions, "models ILIKE ?") - } else { - conditions = append(conditions, "models LIKE ?") - } - values = append(values, "%"+keyword+"%") - - if len(conditions) > 0 { - tx = tx.Where(fmt.Sprintf("(%s)", strings.Join(conditions, " OR ")), values...) - } - } - - err = tx.Count(&total).Error - if err != nil { - return nil, 0, err - } - if total <= 0 { - return nil, 0, nil - } - limit, offset := toLimitOffset(page, perPage) - err = tx.Order(getTokenOrder(order)).Limit(limit).Offset(offset).Find(&tokens).Error - return tokens, total, err -} - -func GetTokenByKey(key string) (*Token, error) { - if key == "" { - return nil, errors.New("key is empty") - } - var token Token - err := DB.Where("key = ?", key).First(&token).Error - return &token, HandleNotFound(err, ErrTokenNotFound) -} - -func ValidateAndGetToken(key string) (token *TokenCache, err error) { - if key == "" { - return nil, errors.New("no token provided") - } - token, err = CacheGetTokenByKey(key) - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, errors.New("invalid token") - } - log.Error("get token from cache failed: " + err.Error()) - return nil, errors.New("token validation failed") - } - if token.Status == TokenStatusDisabled { - return nil, fmt.Errorf("token (%s[%d]) is disabled", token.Name, token.ID) - } - if !time.Time(token.ExpiredAt).IsZero() && time.Time(token.ExpiredAt).Before(time.Now()) { - return nil, fmt.Errorf("token (%s[%d]) is expired", token.Name, token.ID) - } - if token.Quota > 0 && token.UsedAmount >= token.Quota { - return nil, fmt.Errorf("token (%s[%d]) quota is exhausted", token.Name, token.ID) - } - return token, nil -} - -func GetGroupTokenByID(group string, id int) (*Token, error) { - if id == 0 || group == "" { - return nil, errors.New("id or group is empty") - } - token := Token{} - err := DB. - Where("id = ? and group_id = ?", id, group). - First(&token).Error - return &token, HandleNotFound(err, ErrTokenNotFound) -} - -func GetTokenByID(id int) (*Token, error) { - if id == 0 { - return nil, errors.New("id is empty") - } - token := Token{ID: id} - err := DB.First(&token, "id = ?", id).Error - return &token, HandleNotFound(err, ErrTokenNotFound) -} - -func UpdateTokenStatus(id int, status int) (err error) { - token := Token{ID: id} - defer func() { - if err == nil { - if err := CacheUpdateTokenStatus(token.Key, status); err != nil { - log.Error("update token status in cache failed: " + err.Error()) - } - } - }() - result := DB. - Model(&token). - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where("id = ?", id). - Updates( - map[string]interface{}{ - "status": status, - }, - ) - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func UpdateGroupTokenStatus(group string, id int, status int) (err error) { - if id == 0 || group == "" { - return errors.New("id or group is empty") - } - token := Token{} - defer func() { - if err == nil { - if err := CacheUpdateTokenStatus(token.Key, status); err != nil { - log.Error("update token status in cache failed: " + err.Error()) - } - } - }() - result := DB. - Model(&token). - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where("id = ? and group_id = ?", id, group). - Updates( - map[string]interface{}{ - "status": status, - }, - ) - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func DeleteGroupTokenByID(groupID string, id int) (err error) { - if id == 0 || groupID == "" { - return errors.New("id or group is empty") - } - token := Token{ID: id, GroupID: groupID} - defer func() { - if err == nil { - if err := CacheDeleteToken(token.Key); err != nil { - log.Error("delete token from cache failed: " + err.Error()) - } - } - }() - result := DB. - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where(token). - Delete(&token) - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func DeleteGroupTokensByIDs(group string, ids []int) (err error) { - if group == "" { - return errors.New("group is empty") - } - if len(ids) == 0 { - return nil - } - tokens := make([]Token, len(ids)) - defer func() { - if err == nil { - for _, token := range tokens { - if err := CacheDeleteToken(token.Key); err != nil { - log.Error("delete token from cache failed: " + err.Error()) - } - } - } - }() - return DB.Transaction(func(tx *gorm.DB) error { - return tx. - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where("group_id = ?", group). - Where("id IN (?)", ids). - Delete(&tokens). - Error - }) -} - -func DeleteTokenByID(id int) (err error) { - if id == 0 { - return errors.New("id is empty") - } - token := Token{ID: id} - defer func() { - if err == nil { - if err := CacheDeleteToken(token.Key); err != nil { - log.Error("delete token from cache failed: " + err.Error()) - } - } - }() - result := DB. - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where(token). - Delete(&token) - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func DeleteTokensByIDs(ids []int) (err error) { - if len(ids) == 0 { - return nil - } - tokens := make([]Token, len(ids)) - defer func() { - if err == nil { - for _, token := range tokens { - if err := CacheDeleteToken(token.Key); err != nil { - log.Error("delete token from cache failed: " + err.Error()) - } - } - } - }() - return DB.Transaction(func(tx *gorm.DB) error { - return tx. - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where("id IN (?)", ids). - Delete(&tokens). - Error - }) -} - -func UpdateToken(id int, token *Token) (err error) { - if id == 0 { - return errors.New("id is empty") - } - defer func() { - if err == nil { - if err := CacheDeleteToken(token.Key); err != nil { - log.Error("delete token from cache failed: " + err.Error()) - } - } - }() - result := DB. - Select("subnets", "quota", "models", "expired_at"). - Where("id = ?", id). - Clauses(clause.Returning{}). - Updates(token) - if result.Error != nil { - if errors.Is(result.Error, gorm.ErrDuplicatedKey) { - return errors.New("token name already exists in this group") - } - } - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func UpdateGroupToken(id int, group string, token *Token) (err error) { - if id == 0 || group == "" { - return errors.New("id or group is empty") - } - - defer func() { - if err == nil { - if err := CacheDeleteToken(token.Key); err != nil { - log.Error("delete token from cache failed: " + err.Error()) - } - } - }() - result := DB. - Select("subnets", "quota", "models", "expired_at"). - Where("id = ? and group_id = ?", id, group). - Clauses(clause.Returning{}). - Updates(token) - if result.Error != nil { - if errors.Is(result.Error, gorm.ErrDuplicatedKey) { - return errors.New("token name already exists in this group") - } - } - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func UpdateTokenUsedAmount(id int, amount float64, requestCount int) (err error) { - token := &Token{} - defer func() { - if amount > 0 && err == nil && token.Quota > 0 { - if err := CacheUpdateTokenUsedAmountOnlyIncrease(token.Key, token.UsedAmount); err != nil { - log.Error("update token used amount in cache failed: " + err.Error()) - } - } - }() - result := DB. - Model(token). - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - {Name: "quota"}, - {Name: "used_amount"}, - }, - }). - Where("id = ?", id). - Updates( - map[string]interface{}{ - "used_amount": gorm.Expr("used_amount + ?", amount), - "request_count": gorm.Expr("request_count + ?", requestCount), - }, - ) - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func UpdateTokenName(id int, name string) (err error) { - token := &Token{ID: id} - defer func() { - if err == nil { - if err := CacheUpdateTokenName(token.Key, name); err != nil { - log.Error("update token name in cache failed: " + err.Error()) - } - } - }() - result := DB. - Model(token). - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where("id = ?", id). - Update("name", name) - if result.Error != nil && errors.Is(result.Error, gorm.ErrDuplicatedKey) { - return errors.New("token name already exists in this group") - } - return HandleUpdateResult(result, ErrTokenNotFound) -} - -func UpdateGroupTokenName(group string, id int, name string) (err error) { - token := &Token{ID: id, GroupID: group} - defer func() { - if err == nil { - if err := CacheUpdateTokenName(token.Key, name); err != nil { - log.Error("update token name in cache failed: " + err.Error()) - } - } - }() - result := DB. - Model(token). - Clauses(clause.Returning{ - Columns: []clause.Column{ - {Name: "key"}, - }, - }). - Where("id = ? and group_id = ?", id, group). - Update("name", name) - if result.Error != nil && errors.Is(result.Error, gorm.ErrDuplicatedKey) { - return errors.New("token name already exists in this group") - } - return HandleUpdateResult(result, ErrTokenNotFound) -} diff --git a/service/aiproxy/model/utils.go b/service/aiproxy/model/utils.go deleted file mode 100644 index 7f6b6eeef..000000000 --- a/service/aiproxy/model/utils.go +++ /dev/null @@ -1,299 +0,0 @@ -package model - -import ( - "context" - "database/sql/driver" - "errors" - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/labring/sealos/service/aiproxy/common/notify" - "github.com/shopspring/decimal" - "gorm.io/gorm" - "gorm.io/gorm/clause" -) - -func NotFoundError(errMsg ...string) error { - return fmt.Errorf("%s %w", strings.Join(errMsg, " "), gorm.ErrRecordNotFound) -} - -func HandleNotFound(err error, errMsg ...string) error { - if err != nil && errors.Is(err, gorm.ErrRecordNotFound) { - return NotFoundError(strings.Join(errMsg, " ")) - } - return err -} - -// Helper function to handle update results -func HandleUpdateResult(result *gorm.DB, entityName string) error { - if result.Error != nil { - return HandleNotFound(result.Error, entityName) - } - if result.RowsAffected == 0 { - return NotFoundError(entityName) - } - return nil -} - -func OnConflictDoNothing() *gorm.DB { - return DB.Clauses(clause.OnConflict{ - DoNothing: true, - }) -} - -func IgnoreNotFound(err error) error { - if err != nil && errors.Is(err, gorm.ErrRecordNotFound) { - return nil - } - return err -} - -type BatchUpdateData struct { - Groups map[string]*GroupUpdate - Tokens map[int]*TokenUpdate - Channels map[int]*ChannelUpdate - sync.Mutex -} - -type GroupUpdate struct { - Amount float64 - Count int -} - -type TokenUpdate struct { - Amount float64 - Count int -} - -type ChannelUpdate struct { - Amount float64 - Count int -} - -var batchData BatchUpdateData - -func init() { - batchData = BatchUpdateData{ - Groups: make(map[string]*GroupUpdate), - Tokens: make(map[int]*TokenUpdate), - Channels: make(map[int]*ChannelUpdate), - } -} - -func StartBatchProcessor(ctx context.Context, wg *sync.WaitGroup) { - defer wg.Done() - - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - ProcessBatchUpdates() - return - case <-ticker.C: - ProcessBatchUpdates() - } - } -} - -func ProcessBatchUpdates() { - batchData.Lock() - defer batchData.Unlock() - - if len(batchData.Groups) > 0 { - for groupID, data := range batchData.Groups { - err := UpdateGroupUsedAmountAndRequestCount(groupID, data.Amount, data.Count) - if IgnoreNotFound(err) != nil { - notify.ErrorThrottle( - "batchUpdateGroupUsedAmountAndRequestCount", - time.Minute, - "failed to batch update group", - err.Error(), - ) - } else { - delete(batchData.Groups, groupID) - } - } - } - - if len(batchData.Tokens) > 0 { - for tokenID, data := range batchData.Tokens { - err := UpdateTokenUsedAmount(tokenID, data.Amount, data.Count) - if IgnoreNotFound(err) != nil { - notify.ErrorThrottle( - "batchUpdateTokenUsedAmount", - time.Minute, - "failed to batch update token", - err.Error(), - ) - } else { - delete(batchData.Tokens, tokenID) - } - } - } - - if len(batchData.Channels) > 0 { - for channelID, data := range batchData.Channels { - err := UpdateChannelUsedAmount(channelID, data.Amount, data.Count) - if IgnoreNotFound(err) != nil { - notify.ErrorThrottle( - "batchUpdateChannelUsedAmount", - time.Minute, - "failed to batch update channel", - err.Error(), - ) - } else { - delete(batchData.Channels, channelID) - } - } - } -} - -func BatchRecordConsume( - requestID string, - requestAt time.Time, - group string, - code int, - channelID int, - promptTokens int, - completionTokens int, - modelName string, - tokenID int, - tokenName string, - amount float64, - price float64, - completionPrice float64, - endpoint string, - content string, - mode int, - ip string, - retryTimes int, - requestDetail *RequestDetail, -) error { - err := RecordConsumeLog( - requestID, - requestAt, - group, - code, - channelID, - promptTokens, - completionTokens, - modelName, - tokenID, - tokenName, - amount, - price, - completionPrice, - endpoint, - content, - mode, - ip, - retryTimes, - requestDetail, - ) - - amountDecimal := decimal.NewFromFloat(amount) - - batchData.Lock() - defer batchData.Unlock() - - if group != "" { - if _, ok := batchData.Groups[group]; !ok { - batchData.Groups[group] = &GroupUpdate{} - } - - if amount > 0 { - batchData.Groups[group].Amount = amountDecimal. - Add(decimal.NewFromFloat(batchData.Groups[group].Amount)). - InexactFloat64() - } - batchData.Groups[group].Count += 1 - } - - if tokenID > 0 { - if _, ok := batchData.Tokens[tokenID]; !ok { - batchData.Tokens[tokenID] = &TokenUpdate{} - } - - if amount > 0 { - batchData.Tokens[tokenID].Amount = amountDecimal. - Add(decimal.NewFromFloat(batchData.Tokens[tokenID].Amount)). - InexactFloat64() - } - batchData.Tokens[tokenID].Count += 1 - } - - if channelID > 0 { - if _, ok := batchData.Channels[channelID]; !ok { - batchData.Channels[channelID] = &ChannelUpdate{} - } - - if amount > 0 { - batchData.Channels[channelID].Amount = amountDecimal. - Add(decimal.NewFromFloat(batchData.Channels[channelID].Amount)). - InexactFloat64() - } - batchData.Channels[channelID].Count += 1 - } - - return err -} - -type EmptyNullString string - -func (ns EmptyNullString) String() string { - return string(ns) -} - -// Scan implements the [Scanner] interface. -func (ns *EmptyNullString) Scan(value any) error { - if value == nil { - *ns = "" - return nil - } - switch v := value.(type) { - case []byte: - *ns = EmptyNullString(v) - case string: - *ns = EmptyNullString(v) - default: - return fmt.Errorf("unsupported type: %T", v) - } - return nil -} - -// Value implements the [driver.Valuer] interface. -func (ns EmptyNullString) Value() (driver.Value, error) { - if ns == "" { - return nil, nil - } - return string(ns), nil -} - -func String2Int(keyword string) int { - if keyword == "" { - return 0 - } - i, err := strconv.Atoi(keyword) - if err != nil { - return 0 - } - return i -} - -func toLimitOffset(page int, perPage int) (limit int, offset int) { - page-- - if page < 0 { - page = 0 - } - if perPage <= 0 { - perPage = 10 - } else if perPage > 100 { - perPage = 100 - } - return perPage, page * perPage -} diff --git a/service/aiproxy/monitor/memmodel.go b/service/aiproxy/monitor/memmodel.go deleted file mode 100644 index aaa2357f2..000000000 --- a/service/aiproxy/monitor/memmodel.go +++ /dev/null @@ -1,337 +0,0 @@ -package monitor - -import ( - "context" - "sync" - "time" - - "github.com/labring/sealos/service/aiproxy/common/config" -) - -var memModelMonitor *MemModelMonitor - -func init() { - memModelMonitor = NewMemModelMonitor() -} - -const ( - timeWindow = 10 * time.Second - maxSliceCount = 12 - banDuration = 5 * time.Minute - minRequestCount = 20 - cleanupInterval = time.Minute -) - -type MemModelMonitor struct { - mu sync.RWMutex - models map[string]*ModelData -} - -type ModelData struct { - channels map[int64]*ChannelStats - totalStats *TimeWindowStats -} - -type ChannelStats struct { - timeWindows *TimeWindowStats - bannedUntil time.Time -} - -type TimeWindowStats struct { - slices []*timeSlice - mu sync.Mutex -} - -type timeSlice struct { - windowStart time.Time - requests int - errors int -} - -func NewTimeWindowStats() *TimeWindowStats { - return &TimeWindowStats{ - slices: make([]*timeSlice, 0, maxSliceCount), - } -} - -func NewMemModelMonitor() *MemModelMonitor { - mm := &MemModelMonitor{ - models: make(map[string]*ModelData), - } - - go mm.periodicCleanup() - - return mm -} - -func (m *MemModelMonitor) periodicCleanup() { - ticker := time.NewTicker(cleanupInterval) - defer ticker.Stop() - - for range ticker.C { - m.cleanupExpiredData() - } -} - -func (m *MemModelMonitor) cleanupExpiredData() { - m.mu.Lock() - defer m.mu.Unlock() - - now := time.Now() - - for modelName, modelData := range m.models { - for channelID, channelStats := range modelData.channels { - hasValidSlices := channelStats.timeWindows.HasValidSlices() - if !hasValidSlices && !channelStats.bannedUntil.After(now) { - delete(modelData.channels, channelID) - } - } - - hasValidSlices := modelData.totalStats.HasValidSlices() - if !hasValidSlices && len(modelData.channels) == 0 { - delete(m.models, modelName) - } - } -} - -func (m *MemModelMonitor) AddRequest(model string, channelID int64, isError, tryBan bool) (beyondThreshold, banExecution bool) { - m.mu.Lock() - defer m.mu.Unlock() - - now := time.Now() - - var modelData *ModelData - var exists bool - if modelData, exists = m.models[model]; !exists { - modelData = &ModelData{ - channels: make(map[int64]*ChannelStats), - totalStats: NewTimeWindowStats(), - } - m.models[model] = modelData - } - - var channel *ChannelStats - if channel, exists = modelData.channels[channelID]; !exists { - channel = &ChannelStats{ - timeWindows: NewTimeWindowStats(), - } - modelData.channels[channelID] = channel - } - - modelData.totalStats.AddRequest(now, isError) - channel.timeWindows.AddRequest(now, isError) - - return m.checkAndBan(now, channel, tryBan) -} - -func (m *MemModelMonitor) checkAndBan(now time.Time, channel *ChannelStats, tryBan bool) (beyondThreshold, banExecution bool) { - canBan := config.GetEnableModelErrorAutoBan() - if tryBan && canBan { - if channel.bannedUntil.After(now) { - return false, false - } - channel.bannedUntil = now.Add(banDuration) - return false, true - } - - req, err := channel.timeWindows.GetStats(maxSliceCount) - if req < minRequestCount { - return false, false - } - - if float64(err)/float64(req) >= config.GetModelErrorAutoBanRate() { - if !canBan || channel.bannedUntil.After(now) { - return true, false - } - channel.bannedUntil = now.Add(banDuration) - return false, true - } - return false, false -} - -func getErrorRateFromStats(stats *TimeWindowStats) float64 { - req, err := stats.GetStats(maxSliceCount) - if req < minRequestCount { - return 0 - } - return float64(err) / float64(req) -} - -func (m *MemModelMonitor) GetModelsErrorRate(ctx context.Context) (map[string]float64, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - result := make(map[string]float64) - for model, data := range m.models { - result[model] = getErrorRateFromStats(data.totalStats) - } - return result, nil -} - -func (m *MemModelMonitor) GetModelChannelErrorRate(ctx context.Context, model string) (map[int64]float64, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - result := make(map[int64]float64) - if data, exists := m.models[model]; exists { - for channelID, channel := range data.channels { - result[channelID] = getErrorRateFromStats(channel.timeWindows) - } - } - return result, nil -} - -func (m *MemModelMonitor) GetChannelModelErrorRates(ctx context.Context, channelID int64) (map[string]float64, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - result := make(map[string]float64) - for model, data := range m.models { - if channel, exists := data.channels[channelID]; exists { - result[model] = getErrorRateFromStats(channel.timeWindows) - } - } - return result, nil -} - -func (m *MemModelMonitor) GetAllChannelModelErrorRates(ctx context.Context) (map[int64]map[string]float64, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - result := make(map[int64]map[string]float64) - for model, data := range m.models { - for channelID, channel := range data.channels { - if _, exists := result[channelID]; !exists { - result[channelID] = make(map[string]float64) - } - result[channelID][model] = getErrorRateFromStats(channel.timeWindows) - } - } - return result, nil -} - -func (m *MemModelMonitor) GetBannedChannelsWithModel(ctx context.Context, model string) ([]int64, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - var banned []int64 - if data, exists := m.models[model]; exists { - now := time.Now() - for channelID, channel := range data.channels { - if channel.bannedUntil.After(now) { - banned = append(banned, channelID) - } else { - channel.bannedUntil = time.Time{} - } - } - } - return banned, nil -} - -func (m *MemModelMonitor) GetAllBannedModelChannels(ctx context.Context) (map[string][]int64, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - result := make(map[string][]int64) - now := time.Now() - - for model, data := range m.models { - for channelID, channel := range data.channels { - if channel.bannedUntil.After(now) { - if _, exists := result[model]; !exists { - result[model] = []int64{} - } - result[model] = append(result[model], channelID) - } else { - channel.bannedUntil = time.Time{} - } - } - } - return result, nil -} - -func (m *MemModelMonitor) ClearChannelModelErrors(ctx context.Context, model string, channelID int) error { - m.mu.Lock() - defer m.mu.Unlock() - - if data, exists := m.models[model]; exists { - delete(data.channels, int64(channelID)) - } - return nil -} - -func (m *MemModelMonitor) ClearChannelAllModelErrors(ctx context.Context, channelID int) error { - m.mu.Lock() - defer m.mu.Unlock() - - for _, data := range m.models { - delete(data.channels, int64(channelID)) - } - return nil -} - -func (m *MemModelMonitor) ClearAllModelErrors(ctx context.Context) error { - m.mu.Lock() - defer m.mu.Unlock() - - m.models = make(map[string]*ModelData) - return nil -} - -func (t *TimeWindowStats) cleanupLocked(callback func(slice *timeSlice)) { - cutoff := time.Now().Add(-timeWindow * time.Duration(maxSliceCount)) - validSlices := t.slices[:0] - for _, s := range t.slices { - if s.windowStart.After(cutoff) || s.windowStart.Equal(cutoff) { - validSlices = append(validSlices, s) - if callback != nil { - callback(s) - } - } - } - t.slices = validSlices -} - -func (t *TimeWindowStats) AddRequest(now time.Time, isError bool) { - t.mu.Lock() - defer t.mu.Unlock() - - t.cleanupLocked(nil) - - currentWindow := now.Truncate(timeWindow) - var slice *timeSlice - for i := range t.slices { - if t.slices[i].windowStart.Equal(currentWindow) { - slice = t.slices[i] - break - } - } - if slice == nil { - slice = &timeSlice{windowStart: currentWindow} - t.slices = append(t.slices, slice) - } - - slice.requests++ - if isError { - slice.errors++ - } -} - -func (t *TimeWindowStats) GetStats(maxSlice int) (totalReq, totalErr int) { - t.mu.Lock() - defer t.mu.Unlock() - - t.cleanupLocked(func(slice *timeSlice) { - totalReq += slice.requests - totalErr += slice.errors - }) - return -} - -func (t *TimeWindowStats) HasValidSlices() bool { - t.mu.Lock() - defer t.mu.Unlock() - - t.cleanupLocked(nil) - return len(t.slices) > 0 -} diff --git a/service/aiproxy/monitor/model.go b/service/aiproxy/monitor/model.go deleted file mode 100644 index 767655bbb..000000000 --- a/service/aiproxy/monitor/model.go +++ /dev/null @@ -1,514 +0,0 @@ -package monitor - -import ( - "context" - "fmt" - "strconv" - "strings" - "time" - - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/redis/go-redis/v9" -) - -// Redis key prefixes and patterns -const ( - modelKeyPrefix = "model:" - bannedKeySuffix = ":banned" - statsKeySuffix = ":stats" - modelTotalStatsSuffix = ":total_stats" - channelKeyPart = ":channel:" -) - -// Redis scripts -var ( - addRequestScript = redis.NewScript(addRequestLuaScript) - getErrorRateScript = redis.NewScript(getErrorRateLuaScript) - clearChannelModelErrorsScript = redis.NewScript(clearChannelModelErrorsLuaScript) - clearChannelAllModelErrorsScript = redis.NewScript(clearChannelAllModelErrorsLuaScript) - clearAllModelErrorsScript = redis.NewScript(clearAllModelErrorsLuaScript) -) - -// GetModelErrorRate gets error rate for a specific model across all channels -func GetModelsErrorRate(ctx context.Context) (map[string]float64, error) { - if !common.RedisEnabled { - return memModelMonitor.GetModelsErrorRate(ctx) - } - - result := make(map[string]float64) - pattern := modelKeyPrefix + "*" + modelTotalStatsSuffix - - now := time.Now().UnixMilli() - - iter := common.RDB.Scan(ctx, 0, pattern, 0).Iterator() - for iter.Next(ctx) { - key := iter.Val() - model := strings.TrimPrefix(key, modelKeyPrefix) - model = strings.TrimSuffix(model, modelTotalStatsSuffix) - - rate, err := getErrorRateScript.Run( - ctx, - common.RDB, - []string{key}, - now, - ).Float64() - if err != nil { - return nil, err - } - - result[model] = rate - } - - if err := iter.Err(); err != nil { - return nil, err - } - - return result, nil -} - -func canBan() int { - if config.GetEnableModelErrorAutoBan() { - return 1 - } - return 0 -} - -// AddRequest adds a request record and checks if channel should be banned -func AddRequest(ctx context.Context, model string, channelID int64, isError, tryBan bool) (beyondThreshold bool, banExecution bool, err error) { - if !common.RedisEnabled { - beyondThreshold, banExecution = memModelMonitor.AddRequest(model, channelID, isError, tryBan) - return beyondThreshold, banExecution, nil - } - - errorFlag := 0 - if isError { - errorFlag = 1 - } else { - tryBan = false - } - - now := time.Now().UnixMilli() - val, err := addRequestScript.Run( - ctx, - common.RDB, - []string{model}, - channelID, - errorFlag, - now, - config.GetModelErrorAutoBanRate(), - canBan(), - tryBan, - ).Int64() - if err != nil { - return false, false, err - } - return val == 3, val == 1, nil -} - -func buildStatsKey(model string, channelID string) string { - return fmt.Sprintf("%s%s%s%v%s", modelKeyPrefix, model, channelKeyPart, channelID, statsKeySuffix) -} - -func getModelChannelID(key string) (string, int64, bool) { - content := strings.TrimPrefix(key, modelKeyPrefix) - content = strings.TrimSuffix(content, statsKeySuffix) - model, channelIDStr, ok := strings.Cut(content, channelKeyPart) - if !ok { - return "", 0, false - } - channelID, err := strconv.ParseInt(channelIDStr, 10, 64) - if err != nil { - return "", 0, false - } - return model, channelID, true -} - -// GetChannelModelErrorRates gets error rates for a specific channel -func GetChannelModelErrorRates(ctx context.Context, channelID int64) (map[string]float64, error) { - if !common.RedisEnabled { - return memModelMonitor.GetChannelModelErrorRates(ctx, channelID) - } - - result := make(map[string]float64) - pattern := buildStatsKey("*", strconv.FormatInt(channelID, 10)) - now := time.Now().UnixMilli() - - iter := common.RDB.Scan(ctx, 0, pattern, 0).Iterator() - for iter.Next(ctx) { - key := iter.Val() - - model, _, ok := getModelChannelID(key) - if !ok { - continue - } - - rate, err := getErrorRateScript.Run( - ctx, - common.RDB, - []string{key}, - now, - ).Float64() - if err != nil { - return nil, err - } - - result[model] = rate - } - - if err := iter.Err(); err != nil { - return nil, err - } - - return result, nil -} - -func GetModelChannelErrorRate(ctx context.Context, model string) (map[int64]float64, error) { - if !common.RedisEnabled { - return memModelMonitor.GetModelChannelErrorRate(ctx, model) - } - - result := make(map[int64]float64) - pattern := buildStatsKey(model, "*") - now := time.Now().UnixMilli() - - iter := common.RDB.Scan(ctx, 0, pattern, 0).Iterator() - for iter.Next(ctx) { - key := iter.Val() - - _, channelID, ok := getModelChannelID(key) - if !ok { - continue - } - - rate, err := getErrorRateScript.Run( - ctx, - common.RDB, - []string{key}, - now, - ).Float64() - if err != nil { - return nil, err - } - - result[channelID] = rate - } - - if err := iter.Err(); err != nil { - return nil, err - } - - return result, nil -} - -// GetBannedChannelsWithModel gets banned channels for a specific model -func GetBannedChannelsWithModel(ctx context.Context, model string) ([]int64, error) { - if !config.GetEnableModelErrorAutoBan() { - return []int64{}, nil - } - - if !common.RedisEnabled { - return memModelMonitor.GetBannedChannelsWithModel(ctx, model) - } - - result := []int64{} - prefix := modelKeyPrefix + model + channelKeyPart - pattern := prefix + "*" + bannedKeySuffix - iter := common.RDB.Scan(ctx, 0, pattern, 0).Iterator() - - for iter.Next(ctx) { - key := iter.Val() - channelIDStr := strings.TrimSuffix(strings.TrimPrefix(key, prefix), bannedKeySuffix) - - channelID, err := strconv.ParseInt(channelIDStr, 10, 64) - if err != nil { - continue - } - - result = append(result, channelID) - } - - if err := iter.Err(); err != nil { - return nil, err - } - - return result, nil -} - -// ClearChannelModelErrors clears errors for a specific channel and model -func ClearChannelModelErrors(ctx context.Context, model string, channelID int) error { - if !common.RedisEnabled { - return memModelMonitor.ClearChannelModelErrors(ctx, model, channelID) - } - return clearChannelModelErrorsScript.Run( - ctx, - common.RDB, - []string{model}, - strconv.Itoa(channelID), - ).Err() -} - -// ClearChannelAllModelErrors clears all errors for a specific channel -func ClearChannelAllModelErrors(ctx context.Context, channelID int) error { - if !common.RedisEnabled { - return memModelMonitor.ClearChannelAllModelErrors(ctx, channelID) - } - return clearChannelAllModelErrorsScript.Run( - ctx, - common.RDB, - []string{}, - strconv.Itoa(channelID), - ).Err() -} - -// ClearAllModelErrors clears all error records -func ClearAllModelErrors(ctx context.Context) error { - if !common.RedisEnabled { - return memModelMonitor.ClearAllModelErrors(ctx) - } - return clearAllModelErrorsScript.Run(ctx, common.RDB, []string{}).Err() -} - -// GetAllBannedModelChannels gets all banned channels for all models -func GetAllBannedModelChannels(ctx context.Context) (map[string][]int64, error) { - if !config.GetEnableModelErrorAutoBan() { - return map[string][]int64{}, nil - } - - if !common.RedisEnabled { - return memModelMonitor.GetAllBannedModelChannels(ctx) - } - - result := make(map[string][]int64) - pattern := modelKeyPrefix + "*" + channelKeyPart + "*" + bannedKeySuffix - iter := common.RDB.Scan(ctx, 0, pattern, 0).Iterator() - - for iter.Next(ctx) { - key := iter.Val() - parts := strings.TrimPrefix(key, modelKeyPrefix) - parts = strings.TrimSuffix(parts, bannedKeySuffix) - - model, channelIDStr, ok := strings.Cut(parts, channelKeyPart) - if !ok { - continue - } - - channelID, err := strconv.ParseInt(channelIDStr, 10, 64) - if err != nil { - continue - } - - if _, exists := result[model]; !exists { - result[model] = []int64{} - } - result[model] = append(result[model], channelID) - } - - if err := iter.Err(); err != nil { - return nil, err - } - - return result, nil -} - -// GetAllChannelModelErrorRates gets error rates for all channels and models -func GetAllChannelModelErrorRates(ctx context.Context) (map[int64]map[string]float64, error) { - if !common.RedisEnabled { - return memModelMonitor.GetAllChannelModelErrorRates(ctx) - } - - result := make(map[int64]map[string]float64) - pattern := buildStatsKey("*", "*") - now := time.Now().UnixMilli() - - iter := common.RDB.Scan(ctx, 0, pattern, 0).Iterator() - for iter.Next(ctx) { - key := iter.Val() - - model, channelID, ok := getModelChannelID(key) - if !ok { - continue - } - - rate, err := getErrorRateScript.Run( - ctx, - common.RDB, - []string{key}, - now, - ).Float64() - if err != nil { - return nil, err - } - - if _, exists := result[channelID]; !exists { - result[channelID] = make(map[string]float64) - } - result[channelID][model] = rate - } - - if err := iter.Err(); err != nil { - return nil, err - } - - return result, nil -} - -// Lua scripts -const ( - addRequestLuaScript = ` -local model = KEYS[1] -local channel_id = ARGV[1] -local is_error = tonumber(ARGV[2]) -local now_ts = tonumber(ARGV[3]) -local max_error_rate = tonumber(ARGV[4]) -local can_ban = tonumber(ARGV[5]) -local try_ban = tonumber(ARGV[6]) - -local banned_key = "model:" .. model .. ":channel:" .. channel_id .. ":banned" -local stats_key = "model:" .. model .. ":channel:" .. channel_id .. ":stats" -local model_stats_key = "model:" .. model .. ":total_stats" -local maxSliceCount = 12 -local statsExpiry = maxSliceCount * 10 * 1000 -local banExpiry = 5 * 60 * 1000 -local current_slice = math.floor(now_ts / 10 / 1000) - -local function parse_req_err(value) - if not value then return 0, 0 end - local r, e = value:match("^(%d+):(%d+)$") - return tonumber(r) or 0, tonumber(e) or 0 -end - -local function update_stats(key) - local req, err = parse_req_err(redis.call("HGET", key, current_slice)) - req = req + 1 - err = err + (is_error == 1 and 1 or 0) - redis.call("HSET", key, current_slice, req .. ":" .. err) - redis.call("PEXPIRE", key, statsExpiry) - return req, err -end - -local function get_clean_req_err(key) - local total_req, total_err = 0, 0 - local min_valid_slice = current_slice - maxSliceCount - local all_slices = redis.call("HGETALL", key) - for i = 1, #all_slices, 2 do - local slice = tonumber(all_slices[i]) - if slice < min_valid_slice then - redis.call("HDEL", key, all_slices[i]) - else - local req, err = parse_req_err(all_slices[i+1]) - total_req = total_req + req - total_err = total_err + err - end - end - return total_req, total_err -end - -update_stats(stats_key) -update_stats(model_stats_key) - -local function check_channel_error() - local already_banned = redis.call("EXISTS", banned_key) == 1 - - if try_ban == 1 and can_ban == 1 then - if already_banned then - return 2 - end - redis.call("SET", banned_key, 1) - redis.call("PEXPIRE", banned_key, banExpiry) - return 1 - end - - local total_req, total_err = get_clean_req_err(stats_key) - if total_req < 20 then - return 0 - end - - if (total_err / total_req) < max_error_rate then - return 0 - else - if can_ban == 0 or already_banned then - return 3 - end - redis.call("SET", banned_key, 1) - redis.call("PEXPIRE", banned_key, banExpiry) - return 1 - end -end - -return check_channel_error() -` - - getErrorRateLuaScript = ` -local stats_key = KEYS[1] -local now_ts = tonumber(ARGV[1]) -local maxSliceCount = 12 -local current_slice = math.floor(now_ts / 10 / 1000) - -local function parse_req_err(value) - if not value then return 0, 0 end - local r, e = value:match("^(%d+):(%d+)$") - return tonumber(r) or 0, tonumber(e) or 0 -end - -local function get_clean_req_err(key) - local total_req, total_err = 0, 0 - local min_valid_slice = current_slice - maxSliceCount - local all_slices = redis.call("HGETALL", key) - for i = 1, #all_slices, 2 do - local slice = tonumber(all_slices[i]) - if slice < min_valid_slice then - redis.call("HDEL", key, all_slices[i]) - else - local req, err = parse_req_err(all_slices[i+1]) - total_req = total_req + req - total_err = total_err + err - end - end - return total_req, total_err -end - -local total_req, total_err = get_clean_req_err(stats_key) -if total_req < 20 then return 0 end -return string.format("%.2f", total_err / total_req) -` - - clearChannelModelErrorsLuaScript = ` -local model = KEYS[1] -local channel_id = ARGV[1] -local stats_key = "model:" .. model .. ":channel:" .. channel_id .. ":stats" -local banned_key = "model:" .. model .. ":channel:" .. channel_id .. ":banned" - -redis.call("DEL", stats_key) -redis.call("DEL", banned_key) -return redis.status_reply("ok") -` - - clearChannelAllModelErrorsLuaScript = ` -local function del_keys(pattern) - local keys = redis.call("KEYS", pattern) - if #keys > 0 then redis.call("DEL", unpack(keys)) end -end - -local channel_id = ARGV[1] -local stats_pattern = "model:*:channel:" .. channel_id .. ":stats" -local banned_pattern = "model:*:channel:" .. channel_id .. ":banned" - -del_keys(stats_pattern) -del_keys(banned_pattern) - -return redis.status_reply("ok") -` - - clearAllModelErrorsLuaScript = ` -local function del_keys(pattern) - local keys = redis.call("KEYS", pattern) - if #keys > 0 then redis.call("DEL", unpack(keys)) end -end - -del_keys("model:*:channel:*:stats") -del_keys("model:*:channel:*:banned") - -return redis.status_reply("ok") -` -) diff --git a/service/aiproxy/relay/adaptor/ai360/adaptor.go b/service/aiproxy/relay/adaptor/ai360/adaptor.go deleted file mode 100644 index 5ce25e6e0..000000000 --- a/service/aiproxy/relay/adaptor/ai360/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package ai360 - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://ai.360.cn/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "ai360" -} diff --git a/service/aiproxy/relay/adaptor/ai360/constants.go b/service/aiproxy/relay/adaptor/ai360/constants.go deleted file mode 100644 index 6160a450c..000000000 --- a/service/aiproxy/relay/adaptor/ai360/constants.go +++ /dev/null @@ -1,29 +0,0 @@ -package ai360 - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "360GPT_S2_V9", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAI360, - }, - { - Model: "embedding-bert-512-v1", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerAI360, - }, - { - Model: "embedding_s1_v1", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerAI360, - }, - { - Model: "semantic_similarity_s1_v1", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerAI360, - }, -} diff --git a/service/aiproxy/relay/adaptor/ali/adaptor.go b/service/aiproxy/relay/adaptor/ali/adaptor.go deleted file mode 100644 index 2a4398a2a..000000000 --- a/service/aiproxy/relay/adaptor/ali/adaptor.go +++ /dev/null @@ -1,128 +0,0 @@ -package ali - -import ( - "bytes" - "fmt" - "io" - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -// https://help.aliyun.com/zh/dashscope/developer-reference/api-details - -type Adaptor struct{} - -const baseURL = "https://dashscope.aliyuncs.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - u := meta.Channel.BaseURL - if u == "" { - u = baseURL - } - switch meta.Mode { - case relaymode.ImagesGenerations: - return u + "/api/v1/services/aigc/text2image/image-synthesis", nil - case relaymode.ChatCompletions: - return u + "/compatible-mode/v1/chat/completions", nil - case relaymode.Completions: - return u + "/compatible-mode/v1/completions", nil - case relaymode.Embeddings: - return u + "/compatible-mode/v1/embeddings", nil - case relaymode.AudioSpeech, relaymode.AudioTranscription: - return u + "/api-ws/v1/inference", nil - case relaymode.Rerank: - return u + "/api/v1/services/rerank/text-rerank/text-rerank", nil - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - - // req.Header.Set("X-Dashscope-Plugin", meta.Channel.Config.Plugin) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.ImagesGenerations: - return ConvertImageRequest(meta, req) - case relaymode.Rerank: - return ConvertRerankRequest(meta, req) - case relaymode.ChatCompletions, relaymode.Completions, relaymode.Embeddings: - return openai.ConvertRequest(meta, req) - case relaymode.AudioSpeech: - return ConvertTTSRequest(meta, req) - case relaymode.AudioTranscription: - return ConvertSTTRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func ignoreTest(meta *meta.Meta) bool { - return meta.IsChannelTest && - (strings.Contains(meta.ActualModel, "-ocr") || - strings.HasPrefix(meta.ActualModel, "qwen-mt-")) -} - -func (a *Adaptor) DoRequest(meta *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - if ignoreTest(meta) { - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(bytes.NewReader(nil)), - }, nil - } - switch meta.Mode { - case relaymode.AudioSpeech: - return TTSDoRequest(meta, req) - case relaymode.AudioTranscription: - return STTDoRequest(meta, req) - case relaymode.ChatCompletions: - fallthrough - default: - return utils.DoRequest(req) - } -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - if ignoreTest(meta) { - return &relaymodel.Usage{}, nil - } - switch meta.Mode { - case relaymode.ImagesGenerations: - usage, err = ImageHandler(meta, c, resp) - case relaymode.ChatCompletions, relaymode.Completions, relaymode.Embeddings: - usage, err = openai.DoResponse(meta, c, resp) - case relaymode.Rerank: - usage, err = RerankHandler(meta, c, resp) - case relaymode.AudioSpeech: - usage, err = TTSDoResponse(meta, c, resp) - case relaymode.AudioTranscription: - usage, err = STTDoResponse(meta, c, resp) - default: - return nil, openai.ErrorWrapperWithMessage(fmt.Sprintf("unsupported mode: %s", meta.Mode), "unsupported_mode", http.StatusBadRequest) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "ali" -} diff --git a/service/aiproxy/relay/adaptor/ali/constants.go b/service/aiproxy/relay/adaptor/ali/constants.go deleted file mode 100644 index d68f360e1..000000000 --- a/service/aiproxy/relay/adaptor/ali/constants.go +++ /dev/null @@ -1,892 +0,0 @@ -package ali - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://help.aliyun.com/zh/model-studio/getting-started/models?spm=a2c4g.11186623.0.i12#ced16cb6cdfsy - -var ModelList = []*model.ModelConfig{ - // 通义千问-Max - { - Model: "qwen-max", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.02, - OutputPrice: 0.06, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(30720), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-max-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.02, - OutputPrice: 0.06, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(30720), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问-Plus - { - Model: "qwen-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0008, - OutputPrice: 0.002, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-plus-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0008, - OutputPrice: 0.002, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(8000), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问-Turbo - { - Model: "qwen-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-turbo-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1000000), - model.WithModelConfigMaxInputTokens(1000000), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - - // Qwen-Long - { - Model: "qwen-long", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0005, - OutputPrice: 0.002, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1000000), - model.WithModelConfigMaxInputTokens(1000000), - model.WithModelConfigMaxOutputTokens(6000), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问VL - { - Model: "qwen-vl-max", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.02, - OutputPrice: 0.02, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-vl-max-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.02, - OutputPrice: 0.02, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-vl-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.008, - OutputPrice: 0.008, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigMaxInputTokens(6000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-vl-plus-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.008, - OutputPrice: 0.008, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问OCR - { - Model: "qwen-vl-ocr", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.005, - OutputPrice: 0.005, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(34096), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigVision(true), - ), - }, - { - Model: "qwen-vl-ocr-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.005, - OutputPrice: 0.005, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(34096), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigVision(true), - ), - }, - - // 通义千问Math - { - Model: "qwen-math-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-math-plus-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-math-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-math-turbo-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问Coder - { - Model: "qwen-coder-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-coder-plus-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-coder-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-coder-turbo-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问2.5 - { - Model: "qwen2.5-72b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-32b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-14b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-vl-72b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.016, - OutputPrice: 0.048, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-vl-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.005, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-vl-3b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0012, - OutputPrice: 0.0036, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问2 - { - Model: "qwen2-72b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(128000), - model.WithModelConfigMaxOutputTokens(6144), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2-57b-a14b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(65536), - model.WithModelConfigMaxInputTokens(63488), - model.WithModelConfigMaxOutputTokens(6144), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(128000), - model.WithModelConfigMaxOutputTokens(6144), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2-vl-72b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.016, - OutputPrice: 0.048, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(30720), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigVision(true), - ), - }, - - // 通义千问1.5 - { - Model: "qwen1.5-110b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.007, - OutputPrice: 0.014, - RPM: 10, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(8000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen1.5-72b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.005, - OutputPrice: 0.01, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(8000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen1.5-32b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 10, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(8000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen1.5-14b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.004, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigMaxInputTokens(6000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen1.5-7b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigMaxInputTokens(6000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问 - { - Model: "qwen-72b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.02, - OutputPrice: 0.02, - RPM: 80, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigMaxInputTokens(30000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-14b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.008, - OutputPrice: 0.008, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigMaxInputTokens(6000), - model.WithModelConfigMaxOutputTokens(2000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen-7b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.006, - OutputPrice: 0.006, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(7500), - model.WithModelConfigMaxInputTokens(6000), - model.WithModelConfigMaxOutputTokens(1500), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问数学模型 - { - Model: "qwen2.5-math-72b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-math-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2-math-72b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 10, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2-math-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 10, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxInputTokens(3072), - model.WithModelConfigMaxOutputTokens(3072), - model.WithModelConfigToolChoice(true), - ), - }, - - // 通义千问Coder - { - Model: "qwen2.5-coder-32b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-coder-14b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.002, - OutputPrice: 0.006, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "qwen2.5-coder-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxInputTokens(129024), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "qwq-32b-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0035, - OutputPrice: 0.007, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(30720), - model.WithModelConfigMaxOutputTokens(16384), - ), - }, - { - Model: "qvq-72b-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.012, - OutputPrice: 0.036, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(16384), - model.WithModelConfigMaxOutputTokens(16384), - ), - }, - - { - Model: "qwen-mt-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.015, - OutputPrice: 0.045, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(2048), - model.WithModelConfigMaxInputTokens(1024), - model.WithModelConfigMaxOutputTokens(1024), - ), - }, - { - Model: "qwen-mt-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.001, - OutputPrice: 0.003, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(2048), - model.WithModelConfigMaxInputTokens(1024), - model.WithModelConfigMaxOutputTokens(1024), - ), - }, - - // stable-diffusion - { - Model: "stable-diffusion-xl", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - RPM: 2, - }, - { - Model: "stable-diffusion-v1.5", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - RPM: 2, - }, - { - Model: "stable-diffusion-3.5-large", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - RPM: 2, - }, - { - Model: "stable-diffusion-3.5-large-turbo", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - RPM: 2, - }, - - { - Model: "sambert-v1", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.1, - RPM: 20, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(10000), - model.WithModelConfigSupportFormats([]string{"mp3", "wav", "pcm"}), - model.WithModelConfigSupportVoices([]string{ - "zhinan", - "zhiqi", - "zhichu", - "zhide", - "zhijia", - "zhiru", - "zhiqian", - "zhixiang", - "zhiwei", - "zhihao", - "zhijing", - "zhiming", - "zhimo", - "zhina", - "zhishu", - "zhistella", - "zhiting", - "zhixiao", - "zhiya", - "zhiye", - "zhiying", - "zhiyuan", - "zhiyue", - "zhigui", - "zhishuo", - "zhimiao-emo", - "zhimao", - "zhilun", - "zhifei", - "zhida", - "indah", - "clara", - "hanna", - "beth", - "betty", - "cally", - "cindy", - "eva", - "donna", - "brian", - "waan", - }), - ), - }, - - { - Model: "paraformer-realtime-v2", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerAlibaba, - RPM: 20, - InputPrice: 0.24, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(10000), - model.WithModelConfigSupportFormats([]string{"pcm", "wav", "opus", "speex", "aac", "amr"}), - ), - }, - - { - Model: "gte-rerank", - Type: relaymode.Rerank, - Owner: model.ModelOwnerAlibaba, - RPM: 5, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4000), - model.WithModelConfigMaxInputTokens(4000), - ), - }, - - { - Model: "text-embedding-v1", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0007, - RPM: 30, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(2048), - ), - }, - { - Model: "text-embedding-v2", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0007, - RPM: 30, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(2048), - ), - }, - { - Model: "text-embedding-v3", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerAlibaba, - InputPrice: 0.0007, - RPM: 30, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/ali/embeddings.go b/service/aiproxy/relay/adaptor/ali/embeddings.go deleted file mode 100644 index d1fc26859..000000000 --- a/service/aiproxy/relay/adaptor/ali/embeddings.go +++ /dev/null @@ -1,103 +0,0 @@ -package ali - -import ( - "bytes" - "errors" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -// Deprecated: Use openai.ConvertRequest instead -// /api/v1/services/embeddings/text-embedding/text-embedding - -func ConvertEmbeddingsRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - var reqMap map[string]any - err := common.UnmarshalBodyReusable(req, &reqMap) - if err != nil { - return "", nil, nil, err - } - reqMap["model"] = meta.ActualModel - input, ok := reqMap["input"] - if !ok { - return "", nil, nil, errors.New("input is required") - } - switch v := input.(type) { - case string: - reqMap["input"] = map[string]any{ - "texts": []string{v}, - } - case []any: - reqMap["input"] = map[string]any{ - "texts": v, - } - } - parameters := make(map[string]any) - for k, v := range reqMap { - if k == "model" || k == "input" { - continue - } - parameters[k] = v - delete(reqMap, k) - } - reqMap["parameters"] = parameters - jsonData, err := sonic.Marshal(reqMap) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} - -func embeddingResponse2OpenAI(meta *meta.Meta, response *EmbeddingResponse) *openai.EmbeddingResponse { - openAIEmbeddingResponse := openai.EmbeddingResponse{ - Object: "list", - Data: make([]*openai.EmbeddingResponseItem, 0, 1), - Model: meta.OriginModel, - Usage: response.Usage, - } - - for i, embedding := range response.Output.Embeddings { - openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, &openai.EmbeddingResponseItem{ - Object: "embedding", - Index: i, - Embedding: embedding.Embedding, - }) - } - return &openAIEmbeddingResponse -} - -func EmbeddingsHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "read_response_body_failed", resp.StatusCode) - } - var respBody EmbeddingResponse - err = sonic.Unmarshal(responseBody, &respBody) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", resp.StatusCode) - } - if respBody.Usage.PromptTokens == 0 { - respBody.Usage.PromptTokens = respBody.Usage.TotalTokens - } - openaiResponse := embeddingResponse2OpenAI(meta, &respBody) - data, err := sonic.Marshal(openaiResponse) - if err != nil { - return &respBody.Usage, openai.ErrorWrapper(err, "marshal_response_body_failed", resp.StatusCode) - } - _, err = c.Writer.Write(data) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return &openaiResponse.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/ali/image.go b/service/aiproxy/relay/adaptor/ali/image.go deleted file mode 100644 index b36bb441e..000000000 --- a/service/aiproxy/relay/adaptor/ali/image.go +++ /dev/null @@ -1,199 +0,0 @@ -package ali - -import ( - "bytes" - "context" - "errors" - "io" - "net/http" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/image" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" - log "github.com/sirupsen/logrus" -) - -const MetaResponseFormat = "response_format" - -func ConvertImageRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalImageRequest(req) - if err != nil { - return "", nil, nil, err - } - request.Model = meta.ActualModel - - var imageRequest ImageRequest - imageRequest.Input.Prompt = request.Prompt - imageRequest.Model = request.Model - imageRequest.Parameters.Size = strings.ReplaceAll(request.Size, "x", "*") - imageRequest.Parameters.N = request.N - imageRequest.ResponseFormat = request.ResponseFormat - - meta.Set(MetaResponseFormat, request.ResponseFormat) - - data, err := sonic.Marshal(&imageRequest) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, http.Header{ - "X-Dashscope-Async": {"enable"}, - }, bytes.NewReader(data), nil -} - -func ImageHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseFormat := meta.MustGet(MetaResponseFormat).(string) - - var aliTaskResponse TaskResponse - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - err = sonic.Unmarshal(responseBody, &aliTaskResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - if aliTaskResponse.Message != "" { - log.Error("aliAsyncTask err: " + aliTaskResponse.Message) - return nil, openai.ErrorWrapper(errors.New(aliTaskResponse.Message), "ali_async_task_failed", http.StatusInternalServerError) - } - - aliResponse, err := asyncTaskWait(c, aliTaskResponse.Output.TaskID, meta.Channel.Key) - if err != nil { - return nil, openai.ErrorWrapper(err, "ali_async_task_wait_failed", http.StatusInternalServerError) - } - - if aliResponse.Output.TaskStatus != "SUCCEEDED" { - return nil, &model.ErrorWithStatusCode{ - Error: model.Error{ - Message: aliResponse.Output.Message, - Type: "ali_error", - Code: aliResponse.Output.Code, - }, - StatusCode: resp.StatusCode, - } - } - - fullTextResponse := responseAli2OpenAIImage(c.Request.Context(), aliResponse, responseFormat) - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, err = c.Writer.Write(jsonResponse) - if err != nil { - log.Warnf("aliImageHandler write response body failed: %v", err) - } - return &model.Usage{}, nil -} - -func asyncTask(ctx context.Context, taskID string, key string) (*TaskResponse, error) { - url := "https://dashscope.aliyuncs.com/api/v1/tasks/" + taskID - - var aliResponse TaskResponse - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return &aliResponse, err - } - - req.Header.Set("Authorization", "Bearer "+key) - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - return &aliResponse, err - } - defer resp.Body.Close() - - var response TaskResponse - err = sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&response) - if err != nil { - return &aliResponse, err - } - - return &response, nil -} - -func asyncTaskWait(ctx context.Context, taskID string, key string) (*TaskResponse, error) { - waitSeconds := 2 - step := 0 - maxStep := 20 - - for { - step++ - rsp, err := asyncTask(ctx, taskID, key) - if err != nil { - return nil, err - } - - if rsp.Output.TaskStatus == "" { - return rsp, nil - } - - switch rsp.Output.TaskStatus { - case "FAILED": - fallthrough - case "CANCELED": - fallthrough - case "SUCCEEDED": - fallthrough - case "UNKNOWN": - return rsp, nil - } - if step >= maxStep { - break - } - time.Sleep(time.Duration(waitSeconds) * time.Second) - } - - return nil, errors.New("aliAsyncTaskWait timeout") -} - -func responseAli2OpenAIImage(ctx context.Context, response *TaskResponse, responseFormat string) *openai.ImageResponse { - imageResponse := openai.ImageResponse{ - Created: time.Now().Unix(), - } - - for _, data := range response.Output.Results { - var b64Json string - if responseFormat == "b64_json" { - // 读取 data.Url 的图片数据并转存到 b64Json - _, imageData, err := image.GetImageFromURL(ctx, data.URL) - if err != nil { - // 处理获取图片数据失败的情况 - log.Error("getImageData Error getting image data: " + err.Error()) - continue - } - - // 将图片数据转为 Base64 编码的字符串 - b64Json = imageData - } else { - // 如果 responseFormat 不是 "b64_json"，则直接使用 data.B64Image - b64Json = data.B64Image - } - - imageResponse.Data = append(imageResponse.Data, &openai.ImageData{ - URL: data.URL, - B64Json: b64Json, - RevisedPrompt: "", - }) - } - return &imageResponse -} diff --git a/service/aiproxy/relay/adaptor/ali/model.go b/service/aiproxy/relay/adaptor/ali/model.go deleted file mode 100644 index c7cc38a68..000000000 --- a/service/aiproxy/relay/adaptor/ali/model.go +++ /dev/null @@ -1,72 +0,0 @@ -package ali - -import relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - -type ImageRequest struct { - Input struct { - Prompt string `json:"prompt"` - NegativePrompt string `json:"negative_prompt,omitempty"` - } `json:"input"` - Model string `json:"model"` - ResponseFormat string `json:"response_format,omitempty"` - Parameters struct { - Size string `json:"size,omitempty"` - Steps string `json:"steps,omitempty"` - Scale string `json:"scale,omitempty"` - N int `json:"n,omitempty"` - } `json:"parameters,omitempty"` -} - -type TaskResponse struct { - RequestID string `json:"request_id,omitempty"` - Code string `json:"code,omitempty"` - Message string `json:"message,omitempty"` - Output struct { - TaskID string `json:"task_id,omitempty"` - TaskStatus string `json:"task_status,omitempty"` - Code string `json:"code,omitempty"` - Message string `json:"message,omitempty"` - Results []struct { - B64Image string `json:"b64_image,omitempty"` - URL string `json:"url,omitempty"` - Code string `json:"code,omitempty"` - Message string `json:"message,omitempty"` - } `json:"results,omitempty"` - TaskMetrics struct { - Total int `json:"TOTAL,omitempty"` - Succeeded int `json:"SUCCEEDED,omitempty"` - Failed int `json:"FAILED,omitempty"` - } `json:"task_metrics,omitempty"` - } `json:"output,omitempty"` - Usage relaymodel.Usage `json:"usage"` - StatusCode int `json:"status_code,omitempty"` -} - -type EmbeddingRequest struct { - Parameters *struct { - TextType string `json:"text_type,omitempty"` - } `json:"parameters,omitempty"` - Model string `json:"model"` - Input struct { - Texts []string `json:"texts"` - } `json:"input"` -} - -type Embedding struct { - Embedding []float64 `json:"embedding"` - TextIndex int `json:"text_index"` -} - -type EmbeddingResponse struct { - Error - Output struct { - Embeddings []Embedding `json:"embeddings"` - } `json:"output"` - Usage relaymodel.Usage `json:"usage"` -} - -type Error struct { - Code string `json:"code"` - Message string `json:"message"` - RequestID string `json:"request_id"` -} diff --git a/service/aiproxy/relay/adaptor/ali/rerank.go b/service/aiproxy/relay/adaptor/ali/rerank.go deleted file mode 100644 index 28b2c0404..000000000 --- a/service/aiproxy/relay/adaptor/ali/rerank.go +++ /dev/null @@ -1,113 +0,0 @@ -package ali - -import ( - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type RerankResponse struct { - Usage *RerankUsage `json:"usage"` - RequestID string `json:"request_id"` - Output RerankOutput `json:"output"` -} -type RerankOutput struct { - Results []*relaymodel.RerankResult `json:"results"` -} -type RerankUsage struct { - TotalTokens int `json:"total_tokens"` -} - -func ConvertRerankRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - reqMap := make(map[string]any) - err := common.UnmarshalBodyReusable(req, &reqMap) - if err != nil { - return "", nil, nil, err - } - reqMap["model"] = meta.ActualModel - reqMap["input"] = map[string]any{ - "query": reqMap["query"], - "documents": reqMap["documents"], - } - delete(reqMap, "query") - delete(reqMap, "documents") - parameters := make(map[string]any) - for k, v := range reqMap { - if k == "model" || k == "input" { - continue - } - parameters[k] = v - delete(reqMap, k) - } - reqMap["parameters"] = parameters - jsonData, err := sonic.Marshal(reqMap) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} - -func RerankHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - var rerankResponse RerankResponse - err = sonic.Unmarshal(responseBody, &rerankResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - c.Writer.WriteHeader(resp.StatusCode) - - rerankResp := relaymodel.RerankResponse{ - Meta: relaymodel.RerankMeta{ - Tokens: &relaymodel.RerankMetaTokens{ - InputTokens: rerankResponse.Usage.TotalTokens, - OutputTokens: 0, - }, - }, - Result: rerankResponse.Output.Results, - ID: rerankResponse.RequestID, - } - - var usage *relaymodel.Usage - if rerankResponse.Usage == nil { - usage = &relaymodel.Usage{ - PromptTokens: meta.InputTokens, - CompletionTokens: 0, - TotalTokens: meta.InputTokens, - } - } else { - usage = &relaymodel.Usage{ - PromptTokens: rerankResponse.Usage.TotalTokens, - TotalTokens: rerankResponse.Usage.TotalTokens, - } - } - - jsonResponse, err := sonic.Marshal(&rerankResp) - if err != nil { - return usage, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - _, err = c.Writer.Write(jsonResponse) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return usage, nil -} diff --git a/service/aiproxy/relay/adaptor/ali/stt-realtime.go b/service/aiproxy/relay/adaptor/ali/stt-realtime.go deleted file mode 100644 index 0f6fbc17c..000000000 --- a/service/aiproxy/relay/adaptor/ali/stt-realtime.go +++ /dev/null @@ -1,197 +0,0 @@ -package ali - -import ( - "bytes" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/gorilla/websocket" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type STTMessage struct { - Header STTHeader `json:"header"` - Payload STTPayload `json:"payload"` -} - -type STTHeader struct { - Attributes map[string]any `json:"attributes"` - Action string `json:"action,omitempty"` - TaskID string `json:"task_id"` - Streaming string `json:"streaming,omitempty"` - Event string `json:"event,omitempty"` - ErrorCode string `json:"error_code,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` -} - -type STTPayload struct { - Model string `json:"model,omitempty"` - TaskGroup string `json:"task_group,omitempty"` - Task string `json:"task,omitempty"` - Function string `json:"function,omitempty"` - Input STTInput `json:"input,omitempty"` - Output STTOutput `json:"output,omitempty"` - Parameters STTParameters `json:"parameters,omitempty"` - Usage STTUsage `json:"usage,omitempty"` -} - -type STTInput struct { - AudioData []byte `json:"audio_data"` -} - -type STTParameters struct { - Format string `json:"format,omitempty"` - SampleRate int `json:"sample_rate,omitempty"` -} - -type STTOutput struct { - STTSentence STTSentence `json:"sentence"` -} - -type STTSentence struct { - Text string `json:"text"` - EndTime *int `json:"end_time"` -} - -type STTUsage struct { - Characters int `json:"characters"` -} - -func ConvertSTTRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) { - err := request.ParseMultipartForm(1024 * 1024 * 4) - if err != nil { - return "", nil, nil, err - } - audioFile, _, err := request.FormFile("file") - if err != nil { - return "", nil, nil, err - } - audioData, err := io.ReadAll(audioFile) - if err != nil { - return "", nil, nil, err - } - sttRequest := STTMessage{ - Header: STTHeader{ - Action: "run-task", - Streaming: "duplex", - TaskID: uuid.New().String(), - }, - Payload: STTPayload{ - Model: meta.ActualModel, - Task: "asr", - TaskGroup: "audio", - Function: "recognition", - Input: STTInput{}, - }, - } - - data, err := sonic.Marshal(sttRequest) - if err != nil { - return "", nil, nil, err - } - meta.Set("audio_data", audioData) - meta.Set("task_id", sttRequest.Header.TaskID) - return http.MethodPost, http.Header{ - "X-DashScope-DataInspection": {"enable"}, - }, bytes.NewReader(data), nil -} - -func STTDoRequest(meta *meta.Meta, req *http.Request) (*http.Response, error) { - wsURL := req.URL - wsURL.Scheme = "wss" - - conn, _, err := websocket.DefaultDialer.Dial(wsURL.String(), req.Header) - if err != nil { - return nil, err - } - meta.Set("ws_conn", conn) - - jsonWriter, err := conn.NextWriter(websocket.TextMessage) - if err != nil { - return nil, err - } - defer jsonWriter.Close() - _, err = io.Copy(jsonWriter, req.Body) - if err != nil { - return nil, err - } - - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(nil), - }, nil -} - -func STTDoResponse(meta *meta.Meta, c *gin.Context, _ *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - audioData := meta.MustGet("audio_data").([]byte) - taskID := meta.MustGet("task_id").(string) - - conn := meta.MustGet("ws_conn").(*websocket.Conn) - defer conn.Close() - - output := strings.Builder{} - - usage = &relaymodel.Usage{} - - for { - messageType, data, err := conn.ReadMessage() - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_read_msg_failed", "ali_wss_read_msg_failed", http.StatusInternalServerError) - } - - if messageType != websocket.TextMessage { - return usage, openai.ErrorWrapperWithMessage("expect text message, but got binary message", "ali_wss_read_msg_failed", http.StatusInternalServerError) - } - - var msg STTMessage - err = sonic.Unmarshal(data, &msg) - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_read_msg_failed", "ali_wss_read_msg_failed", http.StatusInternalServerError) - } - switch msg.Header.Event { - case "task-started": - err = conn.WriteMessage(websocket.BinaryMessage, audioData) - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_write_msg_failed", "ali_wss_write_msg_failed", http.StatusInternalServerError) - } - finishMsg := STTMessage{ - Header: STTHeader{ - Action: "finish-task", - TaskID: taskID, - Streaming: "duplex", - }, - Payload: STTPayload{ - Input: STTInput{}, - }, - } - finishData, err := sonic.Marshal(finishMsg) - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_write_msg_failed", "ali_wss_write_msg_failed", http.StatusInternalServerError) - } - err = conn.WriteMessage(websocket.TextMessage, finishData) - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_write_msg_failed", "ali_wss_write_msg_failed", http.StatusInternalServerError) - } - case "result-generated": - if msg.Payload.Output.STTSentence.Text != "" { - output.WriteString(msg.Payload.Output.STTSentence.Text) - } - continue - case "task-finished": - usage.PromptTokens = msg.Payload.Usage.Characters - usage.TotalTokens = msg.Payload.Usage.Characters - c.JSON(http.StatusOK, gin.H{ - "text": output.String(), - }) - return usage, nil - case "task-failed": - return usage, openai.ErrorWrapperWithMessage(msg.Header.ErrorMessage, msg.Header.ErrorCode, http.StatusInternalServerError) - } - } -} diff --git a/service/aiproxy/relay/adaptor/ali/tts.go b/service/aiproxy/relay/adaptor/ali/tts.go deleted file mode 100644 index 471e05ee7..000000000 --- a/service/aiproxy/relay/adaptor/ali/tts.go +++ /dev/null @@ -1,233 +0,0 @@ -package ali - -import ( - "bytes" - "fmt" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/gorilla/websocket" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type TTSMessage struct { - Header TTSHeader `json:"header"` - Payload TTSPayload `json:"payload"` -} - -type TTSHeader struct { - Attributes map[string]any `json:"attributes"` - Action string `json:"action,omitempty"` - TaskID string `json:"task_id"` - Streaming string `json:"streaming,omitempty"` - Event string `json:"event,omitempty"` - ErrorCode string `json:"error_code,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` -} - -type TTSPayload struct { - Model string `json:"model,omitempty"` - TaskGroup string `json:"task_group,omitempty"` - Task string `json:"task,omitempty"` - Function string `json:"function,omitempty"` - Input TTSInput `json:"input,omitempty"` - Output TTSOutput `json:"output,omitempty"` - Parameters TTSParameters `json:"parameters,omitempty"` - Usage TTSUsage `json:"usage,omitempty"` -} - -type TTSInput struct { - Text string `json:"text"` -} - -type TTSParameters struct { - TextType string `json:"text_type"` - Format string `json:"format"` - SampleRate int `json:"sample_rate,omitempty"` - Volume int `json:"volume"` - Rate float64 `json:"rate"` - Pitch float64 `json:"pitch"` - WordTimestampEnabled bool `json:"word_timestamp_enabled"` - PhonemeTimestampEnabled bool `json:"phoneme_timestamp_enabled"` -} - -type TTSOutput struct { - Sentence TTSSentence `json:"sentence"` -} - -type TTSSentence struct { - Words []TTSWord `json:"words"` - BeginTime int `json:"begin_time"` - EndTime int `json:"end_time"` -} - -type TTSWord struct { - Text string `json:"text"` - Phonemes []TTSPhoneme `json:"phonemes"` - BeginTime int `json:"begin_time"` - EndTime int `json:"end_time"` -} - -type TTSPhoneme struct { - Text string `json:"text"` - BeginTime int `json:"begin_time"` - EndTime int `json:"end_time"` - Tone int `json:"tone"` -} - -type TTSUsage struct { - Characters int `json:"characters"` -} - -var ttsSupportedFormat = map[string]struct{}{ - "pcm": {}, - "wav": {}, - "mp3": {}, -} - -func ConvertTTSRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalTTSRequest(req) - if err != nil { - return "", nil, nil, err - } - reqMap, err := utils.UnmarshalMap(req) - if err != nil { - return "", nil, nil, err - } - var sampleRate int - sampleRateI, ok := reqMap["sample_rate"].(float64) - if ok { - sampleRate = int(sampleRateI) - } - request.Model = meta.ActualModel - - if strings.HasPrefix(request.Model, "sambert-v") { - voice := request.Voice - if voice == "" { - voice = "zhinan" - } - request.Model = fmt.Sprintf("sambert-%s-v%s", voice, strings.TrimPrefix(request.Model, "sambert-v")) - } - - ttsRequest := TTSMessage{ - Header: TTSHeader{ - Action: "run-task", - Streaming: "out", - TaskID: uuid.New().String(), - }, - Payload: TTSPayload{ - Model: request.Model, - Task: "tts", - TaskGroup: "audio", - Function: "SpeechSynthesizer", - Input: TTSInput{ - Text: request.Input, - }, - Parameters: TTSParameters{ - TextType: "PlainText", - Format: "wav", - Volume: 50, - SampleRate: sampleRate, - Rate: request.Speed, - Pitch: 1.0, - WordTimestampEnabled: true, - PhonemeTimestampEnabled: true, - }, - }, - } - - if _, ok := ttsSupportedFormat[request.ResponseFormat]; ok { - ttsRequest.Payload.Parameters.Format = request.ResponseFormat - } - - if ttsRequest.Payload.Parameters.Rate < 0.5 { - ttsRequest.Payload.Parameters.Rate = 0.5 - } else if ttsRequest.Payload.Parameters.Rate > 2 { - ttsRequest.Payload.Parameters.Rate = 2 - } - - data, err := sonic.Marshal(ttsRequest) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, http.Header{ - "X-DashScope-DataInspection": {"enable"}, - }, bytes.NewReader(data), nil -} - -func TTSDoRequest(meta *meta.Meta, req *http.Request) (*http.Response, error) { - wsURL := req.URL - wsURL.Scheme = "wss" - - conn, _, err := websocket.DefaultDialer.Dial(wsURL.String(), req.Header) - if err != nil { - return nil, err - } - meta.Set("ws_conn", conn) - - writer, err := conn.NextWriter(websocket.TextMessage) - if err != nil { - return nil, err - } - defer writer.Close() - - _, err = io.Copy(writer, req.Body) - if err != nil { - return nil, err - } - - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(nil), - }, nil -} - -func TTSDoResponse(meta *meta.Meta, c *gin.Context, _ *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - log := middleware.GetLogger(c) - - conn := meta.MustGet("ws_conn").(*websocket.Conn) - defer conn.Close() - - usage = &relaymodel.Usage{} - - for { - messageType, data, err := conn.ReadMessage() - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_read_msg_failed", "ali_wss_read_msg_failed", http.StatusInternalServerError) - } - - var msg TTSMessage - switch messageType { - case websocket.TextMessage: - err = sonic.Unmarshal(data, &msg) - if err != nil { - return usage, openai.ErrorWrapperWithMessage("ali_wss_read_msg_failed", "ali_wss_read_msg_failed", http.StatusInternalServerError) - } - switch msg.Header.Event { - case "task-started": - continue - case "result-generated": - continue - case "task-finished": - usage.PromptTokens = msg.Payload.Usage.Characters - usage.TotalTokens = msg.Payload.Usage.Characters - return usage, nil - case "task-failed": - return usage, openai.ErrorWrapperWithMessage(msg.Header.ErrorMessage, msg.Header.ErrorCode, http.StatusInternalServerError) - } - case websocket.BinaryMessage: - _, writeErr := c.Writer.Write(data) - if writeErr != nil { - log.Error("write tts response chunk failed: " + writeErr.Error()) - } - } - } -} diff --git a/service/aiproxy/relay/adaptor/anthropic/adaptor.go b/service/aiproxy/relay/adaptor/anthropic/adaptor.go deleted file mode 100644 index 96dc394e9..000000000 --- a/service/aiproxy/relay/adaptor/anthropic/adaptor.go +++ /dev/null @@ -1,83 +0,0 @@ -package anthropic - -import ( - "bytes" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const baseURL = "https://api.anthropic.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - return meta.Channel.BaseURL + "/messages", nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, c *gin.Context, req *http.Request) error { - req.Header.Set("X-Api-Key", meta.Channel.Key) - anthropicVersion := c.Request.Header.Get("Anthropic-Version") - if anthropicVersion == "" { - anthropicVersion = "2023-06-01" - } - req.Header.Set("Anthropic-Version", anthropicVersion) - req.Header.Set("Anthropic-Beta", "messages-2023-12-15") - - // https://x.com/alexalbert__/status/1812921642143900036 - // claude-3-5-sonnet can support 8k context - if strings.HasPrefix(meta.ActualModel, "claude-3-5-sonnet") { - req.Header.Set("Anthropic-Beta", "max-tokens-3-5-sonnet-2024-07-15") - } - - if strings.HasPrefix(meta.ActualModel, "claude-3-7-sonnet") { - req.Header.Set("Anthropic-Beta", "output-128k-2025-02-19") - } - - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - data, err := ConvertRequest(meta, req) - if err != nil { - return "", nil, nil, err - } - - data2, err := sonic.Marshal(data) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data2), nil -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - if utils.IsStreamResponse(resp) { - usage, err = StreamHandler(meta, c, resp) - } else { - usage, err = Handler(meta, c, resp) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "anthropic" -} diff --git a/service/aiproxy/relay/adaptor/anthropic/constants.go b/service/aiproxy/relay/adaptor/anthropic/constants.go deleted file mode 100644 index c58eee144..000000000 --- a/service/aiproxy/relay/adaptor/anthropic/constants.go +++ /dev/null @@ -1,79 +0,0 @@ -package anthropic - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "claude-3-haiku-20240307", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - InputPrice: 0.0025, - OutputPrice: 0.0125, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(200000), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "claude-3-opus-20240229", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - InputPrice: 0.015, - OutputPrice: 0.075, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(200000), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "claude-3-5-haiku-20241022", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - InputPrice: 0.0008, - OutputPrice: 0.004, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(200000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "claude-3-5-sonnet-20240620", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - InputPrice: 0.003, - OutputPrice: 0.015, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(200000), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "claude-3-5-sonnet-20241022", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - InputPrice: 0.003, - OutputPrice: 0.015, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(200000), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "claude-3-5-sonnet-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - InputPrice: 0.003, - OutputPrice: 0.015, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(200000), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/anthropic/main.go b/service/aiproxy/relay/adaptor/anthropic/main.go deleted file mode 100644 index 742c31492..000000000 --- a/service/aiproxy/relay/adaptor/anthropic/main.go +++ /dev/null @@ -1,447 +0,0 @@ -package anthropic - -import ( - "bufio" - "net/http" - "slices" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/image" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -const ( - toolUseType = "tool_use" - conetentTypeText = "text" - conetentTypeThinking = "thinking" - conetentTypeImage = "image" -) - -func stopReasonClaude2OpenAI(reason *string) string { - if reason == nil { - return "" - } - switch *reason { - case "end_turn", "stop_sequence": - return constant.StopFinishReason - case "max_tokens": - return "length" - case toolUseType: - return "tool_calls" - default: - return *reason - } -} - -type onlyThinkingRequest struct { - Thinking *Thinking `json:"thinking,omitempty"` -} - -func ConvertRequest(meta *meta.Meta, req *http.Request) (*Request, error) { - var textRequest model.GeneralOpenAIRequest - err := common.UnmarshalBodyReusable(req, &textRequest) - if err != nil { - return nil, err - } - - var onlyThinking onlyThinkingRequest - err = common.UnmarshalBodyReusable(req, &onlyThinking) - if err != nil { - return nil, err - } - - textRequest.Model = meta.ActualModel - meta.Set("stream", textRequest.Stream) - claudeTools := make([]Tool, 0, len(textRequest.Tools)) - - for _, tool := range textRequest.Tools { - if params, ok := tool.Function.Parameters.(map[string]any); ok { - t, _ := params["type"].(string) - claudeTools = append(claudeTools, Tool{ - Name: tool.Function.Name, - Description: tool.Function.Description, - InputSchema: InputSchema{ - Type: t, - Properties: params["properties"], - Required: params["required"], - }, - }) - } - } - - claudeRequest := Request{ - Model: meta.ActualModel, - MaxTokens: textRequest.MaxTokens, - Temperature: textRequest.Temperature, - TopP: textRequest.TopP, - TopK: textRequest.TopK, - Stream: textRequest.Stream, - Tools: claudeTools, - } - - if claudeRequest.MaxTokens == 0 { - claudeRequest.MaxTokens = 4096 - } - - if onlyThinking.Thinking != nil { - claudeRequest.Thinking = onlyThinking.Thinking - } else if strings.Contains(meta.OriginModel, "think") { - claudeRequest.Thinking = &Thinking{ - Type: "enabled", - } - } - - if claudeRequest.Thinking != nil { - if claudeRequest.Thinking.BudgetTokens == 0 || - claudeRequest.Thinking.BudgetTokens >= claudeRequest.MaxTokens { - claudeRequest.Thinking.BudgetTokens = claudeRequest.MaxTokens / 2 - } - if claudeRequest.Thinking.BudgetTokens < 1024 { - claudeRequest.Thinking.BudgetTokens = 1024 - } - claudeRequest.Temperature = nil - } - - if len(claudeTools) > 0 { - claudeToolChoice := struct { - Type string `json:"type"` - Name string `json:"name,omitempty"` - }{Type: "auto"} // default value https://docs.anthropic.com/en/docs/build-with-claude/tool-use#controlling-claudes-output - if choice, ok := textRequest.ToolChoice.(map[string]any); ok { - if function, ok := choice["function"].(map[string]any); ok { - claudeToolChoice.Type = "tool" - name, _ := function["name"].(string) - claudeToolChoice.Name = name - } - } else if toolChoiceType, ok := textRequest.ToolChoice.(string); ok { - if toolChoiceType == "any" { - claudeToolChoice.Type = toolChoiceType - } - } - claudeRequest.ToolChoice = claudeToolChoice - } - - for _, message := range textRequest.Messages { - if message.Role == "system" && claudeRequest.System == "" { - claudeRequest.System = message.StringContent() - continue - } - claudeMessage := Message{ - Role: message.Role, - } - var content Content - if message.IsStringContent() { - content.Type = conetentTypeText - content.Text = message.StringContent() - if message.Role == "tool" { - claudeMessage.Role = "user" - content.Type = "tool_result" - content.Content = content.Text - content.Text = "" - content.ToolUseID = message.ToolCallID - } - claudeMessage.Content = append(claudeMessage.Content, content) - for i := range message.ToolCalls { - inputParam := make(map[string]any) - _ = sonic.Unmarshal(conv.StringToBytes(message.ToolCalls[i].Function.Arguments), &inputParam) - claudeMessage.Content = append(claudeMessage.Content, Content{ - Type: toolUseType, - ID: message.ToolCalls[i].ID, - Name: message.ToolCalls[i].Function.Name, - Input: inputParam, - }) - } - claudeRequest.Messages = append(claudeRequest.Messages, claudeMessage) - continue - } - var contents []Content - openaiContent := message.ParseContent() - for _, part := range openaiContent { - var content Content - switch part.Type { - case model.ContentTypeText: - content.Type = conetentTypeText - content.Text = part.Text - case model.ContentTypeImageURL: - content.Type = conetentTypeImage - content.Source = &ImageSource{ - Type: "base64", - } - mimeType, data, err := image.GetImageFromURL(req.Context(), part.ImageURL.URL) - if err != nil { - return nil, err - } - content.Source.MediaType = mimeType - content.Source.Data = data - } - contents = append(contents, content) - } - claudeMessage.Content = contents - claudeRequest.Messages = append(claudeRequest.Messages, claudeMessage) - } - - return &claudeRequest, nil -} - -// https://docs.anthropic.com/claude/reference/messages-streaming -func StreamResponseClaude2OpenAI(claudeResponse *StreamResponse) *openai.ChatCompletionsStreamResponse { - openaiResponse := openai.ChatCompletionsStreamResponse{ - Object: "chat.completion.chunk", - } - var content string - var thinking string - var stopReason string - tools := make([]*model.Tool, 0) - - switch claudeResponse.Type { - case "message_start": - return nil - case "content_block_start": - if claudeResponse.ContentBlock != nil { - content = claudeResponse.ContentBlock.Text - if claudeResponse.ContentBlock.Type == toolUseType { - tools = append(tools, &model.Tool{ - ID: claudeResponse.ContentBlock.ID, - Type: "function", - Function: model.Function{ - Name: claudeResponse.ContentBlock.Name, - Arguments: "", - }, - }) - } - } - case "content_block_delta": - if claudeResponse.Delta != nil { - switch claudeResponse.Delta.Type { - case "input_json_delta": - tools = append(tools, &model.Tool{ - Function: model.Function{ - Arguments: claudeResponse.Delta.PartialJSON, - }, - }) - case "thinking_delta": - thinking = claudeResponse.Delta.Thinking - case "signature_delta": - default: - content = claudeResponse.Delta.Text - } - } - case "message_delta": - if claudeResponse.Usage != nil { - openaiResponse.Usage = &model.Usage{ - PromptTokens: claudeResponse.Usage.InputTokens, - CompletionTokens: claudeResponse.Usage.OutputTokens, - TotalTokens: claudeResponse.Usage.InputTokens + claudeResponse.Usage.OutputTokens, - } - } - if claudeResponse.Delta != nil && claudeResponse.Delta.StopReason != nil { - stopReason = *claudeResponse.Delta.StopReason - } - } - - var choice openai.ChatCompletionsStreamResponseChoice - choice.Delta.Content = content - choice.Delta.ReasoningContent = thinking - - if len(tools) > 0 { - choice.Delta.Content = nil // compatible with other OpenAI derivative applications, like LobeOpenAICompatibleFactory ... - choice.Delta.ToolCalls = tools - } - choice.Delta.Role = "assistant" - finishReason := stopReasonClaude2OpenAI(&stopReason) - if finishReason != "null" { - choice.FinishReason = &finishReason - } - openaiResponse.Choices = []*openai.ChatCompletionsStreamResponseChoice{&choice} - - return &openaiResponse -} - -func ResponseClaude2OpenAI(meta *meta.Meta, claudeResponse *Response) *openai.TextResponse { - var content string - var thinking string - for _, v := range claudeResponse.Content { - switch v.Type { - case conetentTypeText: - content = v.Text - case conetentTypeThinking: - thinking = v.Thinking - } - } - tools := make([]*model.Tool, 0) - for _, v := range claudeResponse.Content { - if v.Type == toolUseType { - args, _ := sonic.Marshal(v.Input) - tools = append(tools, &model.Tool{ - ID: v.ID, - Type: "function", // compatible with other OpenAI derivative applications - Function: model.Function{ - Name: v.Name, - Arguments: conv.BytesToString(args), - }, - }) - } - } - choice := openai.TextResponseChoice{ - Index: 0, - Message: model.Message{ - Role: "assistant", - Content: content, - ReasoningContent: thinking, - Name: nil, - ToolCalls: tools, - }, - FinishReason: stopReasonClaude2OpenAI(claudeResponse.StopReason), - } - - fullTextResponse := openai.TextResponse{ - ID: "chatcmpl-" + claudeResponse.ID, - Model: meta.OriginModel, - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []*openai.TextResponseChoice{&choice}, - Usage: model.Usage{ - PromptTokens: claudeResponse.Usage.InputTokens, - CompletionTokens: claudeResponse.Usage.OutputTokens, - TotalTokens: claudeResponse.Usage.InputTokens + claudeResponse.Usage.OutputTokens, - }, - } - return &fullTextResponse -} - -func StreamHandler(m *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - createdTime := time.Now().Unix() - scanner := bufio.NewScanner(resp.Body) - scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - if i := slices.Index(data, '\n'); i >= 0 { - return i + 1, data[0:i], nil - } - if atEOF { - return len(data), data, nil - } - return 0, nil, nil - }) - - common.SetEventStreamHeaders(c) - - var usage model.Usage - var id string - var lastToolCallChoice *openai.ChatCompletionsStreamResponseChoice - var usageWrited bool - - for scanner.Scan() { - data := scanner.Bytes() - if len(data) < 6 || conv.BytesToString(data[:6]) != "data: " { - continue - } - data = data[6:] - - if conv.BytesToString(data) == "[DONE]" { - break - } - - var claudeResponse StreamResponse - err := sonic.Unmarshal(data, &claudeResponse) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - - response := StreamResponseClaude2OpenAI(&claudeResponse) - if response == nil { - continue - } - if response.Usage != nil { - usage = *response.Usage - usageWrited = true - - if lastToolCallChoice != nil && len(lastToolCallChoice.Delta.ToolCalls) > 0 { - lastArgs := &lastToolCallChoice.Delta.ToolCalls[len(lastToolCallChoice.Delta.ToolCalls)-1].Function - if len(lastArgs.Arguments) == 0 { // compatible with OpenAI sending an empty object `{}` when no arguments. - lastArgs.Arguments = "{}" - response.Choices[len(response.Choices)-1].Delta.Content = nil - response.Choices[len(response.Choices)-1].Delta.ToolCalls = lastToolCallChoice.Delta.ToolCalls - } - } - } - - response.ID = id - response.Model = m.OriginModel - response.Created = createdTime - - for _, choice := range response.Choices { - if len(choice.Delta.ToolCalls) > 0 { - lastToolCallChoice = choice - } - } - _ = render.ObjectData(c, response) - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - if usage.CompletionTokens == 0 && usage.PromptTokens == 0 { - usage.PromptTokens = m.InputTokens - } - - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - - if !usageWrited { - _ = render.ObjectData(c, &openai.ChatCompletionsStreamResponse{ - Model: m.OriginModel, - Object: "chat.completion.chunk", - Created: createdTime, - Choices: []*openai.ChatCompletionsStreamResponseChoice{}, - Usage: &usage, - }) - } - - render.Done(c) - - return &usage, nil -} - -func Handler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - var claudeResponse Response - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&claudeResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - fullTextResponse := ResponseClaude2OpenAI(meta, &claudeResponse) - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &fullTextResponse.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/anthropic/model.go b/service/aiproxy/relay/adaptor/anthropic/model.go deleted file mode 100644 index d0c8eabb1..000000000 --- a/service/aiproxy/relay/adaptor/anthropic/model.go +++ /dev/null @@ -1,103 +0,0 @@ -package anthropic - -// https://docs.anthropic.com/claude/reference/messages_post - -type Metadata struct { - UserID string `json:"user_id"` -} - -type ImageSource struct { - Type string `json:"type"` - MediaType string `json:"media_type"` - Data string `json:"data"` -} - -type Content struct { - Type string `json:"type"` - Text string `json:"text,omitempty"` - Thinking string `json:"thinking,omitempty"` - Source *ImageSource `json:"source,omitempty"` - // tool_calls - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - Input any `json:"input,omitempty"` - Content string `json:"content,omitempty"` - ToolUseID string `json:"tool_use_id,omitempty"` -} - -type Message struct { - Role string `json:"role"` - Content []Content `json:"content"` -} - -type Tool struct { - InputSchema InputSchema `json:"input_schema"` - Name string `json:"name"` - Description string `json:"description,omitempty"` -} - -type InputSchema struct { - Properties any `json:"properties,omitempty"` - Required any `json:"required,omitempty"` - Type string `json:"type"` -} - -type Thinking struct { - Type string `json:"type"` - BudgetTokens int `json:"budget_tokens,omitempty"` -} - -type Request struct { - ToolChoice any `json:"tool_choice,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - Model string `json:"model"` - System string `json:"system,omitempty"` - Messages []Message `json:"messages"` - StopSequences []string `json:"stop_sequences,omitempty"` - Tools []Tool `json:"tools,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopK int `json:"top_k,omitempty"` - Stream bool `json:"stream,omitempty"` - Thinking *Thinking `json:"thinking,omitempty"` -} - -type Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -type Error struct { - Type string `json:"type"` - Message string `json:"message"` -} - -type Response struct { - StopReason *string `json:"stop_reason"` - StopSequence *string `json:"stop_sequence"` - Error Error `json:"error"` - ID string `json:"id"` - Type string `json:"type"` - Role string `json:"role"` - Model string `json:"model"` - Content []Content `json:"content"` - Usage Usage `json:"usage"` -} - -type Delta struct { - StopReason *string `json:"stop_reason"` - StopSequence *string `json:"stop_sequence"` - Type string `json:"type"` - Thinking string `json:"thinking,omitempty"` - Text string `json:"text"` - PartialJSON string `json:"partial_json,omitempty"` -} - -type StreamResponse struct { - Message *Response `json:"message"` - ContentBlock *Content `json:"content_block"` - Delta *Delta `json:"delta"` - Usage *Usage `json:"usage"` - Type string `json:"type"` - Index int `json:"index"` -} diff --git a/service/aiproxy/relay/adaptor/aws/adaptor.go b/service/aiproxy/relay/adaptor/aws/adaptor.go deleted file mode 100644 index 09fa2c103..000000000 --- a/service/aiproxy/relay/adaptor/aws/adaptor.go +++ /dev/null @@ -1,66 +0,0 @@ -package aws - -import ( - "errors" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -var _ adaptor.Adaptor = new(Adaptor) - -type Adaptor struct{} - -func (a *Adaptor) GetBaseURL() string { - return "" -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - adaptor := GetAdaptor(meta.ActualModel) - if adaptor == nil { - return "", nil, nil, errors.New("adaptor not found") - } - meta.Set("awsAdapter", adaptor) - return adaptor.ConvertRequest(meta, req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, _ *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - adaptor, ok := meta.Get("awsAdapter") - if !ok { - return nil, &relaymodel.ErrorWithStatusCode{ - StatusCode: http.StatusInternalServerError, - Error: relaymodel.Error{Message: "awsAdapter not found"}, - } - } - return adaptor.(utils.AwsAdapter).DoResponse(meta, c) -} - -func (a *Adaptor) GetModelList() (models []*model.ModelConfig) { - models = make([]*model.ModelConfig, 0, len(adaptors)) - for _, model := range adaptors { - models = append(models, model.config) - } - return -} - -func (a *Adaptor) GetChannelName() string { - return "aws" -} - -func (a *Adaptor) GetRequestURL(_ *meta.Meta) (string, error) { - return "", nil -} - -func (a *Adaptor) SetupRequestHeader(_ *meta.Meta, _ *gin.Context, _ *http.Request) error { - return nil -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, _ *http.Request) (*http.Response, error) { - return nil, nil -} diff --git a/service/aiproxy/relay/adaptor/aws/claude/adapter.go b/service/aiproxy/relay/adaptor/aws/claude/adapter.go deleted file mode 100644 index 1cf0813a2..000000000 --- a/service/aiproxy/relay/adaptor/aws/claude/adapter.go +++ /dev/null @@ -1,38 +0,0 @@ -package aws - -import ( - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/anthropic" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -const ( - ConvertedRequest = "convertedRequest" -) - -var _ utils.AwsAdapter = new(Adaptor) - -type Adaptor struct{} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - r, err := anthropic.ConvertRequest(meta, req) - if err != nil { - return "", nil, nil, err - } - meta.Set(ConvertedRequest, r) - return "", nil, nil, nil -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context) (usage *model.Usage, err *model.ErrorWithStatusCode) { - if meta.GetBool("stream") { - err, usage = StreamHandler(meta, c) - } else { - err, usage = Handler(meta, c) - } - return -} diff --git a/service/aiproxy/relay/adaptor/aws/claude/main.go b/service/aiproxy/relay/adaptor/aws/claude/main.go deleted file mode 100644 index 1e5db012b..000000000 --- a/service/aiproxy/relay/adaptor/aws/claude/main.go +++ /dev/null @@ -1,265 +0,0 @@ -// Package aws provides the AWS adaptor for the relay service. -package aws - -import ( - "io" - "net/http" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/bedrockruntime" - "github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types" - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/jinzhu/copier" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/anthropic" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/pkg/errors" -) - -type awsModelItem struct { - ID string - model.ModelConfig -} - -// AwsModelIDMap maps internal model identifiers to AWS model identifiers. -// For more details, see: https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html - -var AwsModelIDMap = map[string]awsModelItem{ - "claude-instant-1.2": { - ModelConfig: model.ModelConfig{ - Model: "claude-instant-1.2", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - ID: "anthropic.claude-instant-v1", - }, - "claude-2.0": { - ModelConfig: model.ModelConfig{ - Model: "claude-2.0", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - ID: "anthropic.claude-v2", - }, - "claude-2.1": { - ModelConfig: model.ModelConfig{ - Model: "claude-2.1", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - ID: "anthropic.claude-v2:1", - }, - "claude-3-haiku-20240307": { - ModelConfig: model.ModelConfig{ - Model: "claude-3-haiku-20240307", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - ID: "anthropic.claude-3-haiku-20240307-v1:0", - }, - "claude-3-5-sonnet-latest": { - ModelConfig: model.ModelConfig{ - Model: "claude-3-5-sonnet-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - ID: "anthropic.claude-3-5-sonnet-20241022-v2:0", - }, - "claude-3-5-haiku-20241022": { - ModelConfig: model.ModelConfig{ - Model: "claude-3-5-haiku-20241022", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - ID: "anthropic.claude-3-5-haiku-20241022-v1:0", - }, -} - -func awsModelID(requestModel string) (string, error) { - if awsModelID, ok := AwsModelIDMap[requestModel]; ok { - return awsModelID.ID, nil - } - - return "", errors.Errorf("model %s not found", requestModel) -} - -func Handler(meta *meta.Meta, c *gin.Context) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) { - awsModelID, err := awsModelID(meta.ActualModel) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil - } - - awsReq := &bedrockruntime.InvokeModelInput{ - ModelId: aws.String(awsModelID), - Accept: aws.String("application/json"), - ContentType: aws.String("application/json"), - } - - convReq, ok := meta.Get(ConvertedRequest) - if !ok { - return utils.WrapErr(errors.New("request not found")), nil - } - claudeReq := convReq.(*anthropic.Request) - awsClaudeReq := &Request{ - AnthropicVersion: "bedrock-2023-05-31", - } - if err = copier.Copy(awsClaudeReq, claudeReq); err != nil { - return utils.WrapErr(errors.Wrap(err, "copy request")), nil - } - - awsReq.Body, err = sonic.Marshal(awsClaudeReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "marshal request")), nil - } - - awsClient, err := utils.AwsClientFromMeta(meta) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "get aws client")), nil - } - - awsResp, err := awsClient.InvokeModel(c.Request.Context(), awsReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "InvokeModel")), nil - } - - claudeResponse := new(anthropic.Response) - err = sonic.Unmarshal(awsResp.Body, claudeResponse) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "unmarshal response")), nil - } - - openaiResp := anthropic.ResponseClaude2OpenAI(meta, claudeResponse) - c.JSON(http.StatusOK, openaiResp) - return nil, &openaiResp.Usage -} - -func StreamHandler(meta *meta.Meta, c *gin.Context) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) { - log := middleware.GetLogger(c) - createdTime := time.Now().Unix() - originModelName := meta.OriginModel - awsModelID, err := awsModelID(meta.ActualModel) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil - } - - awsReq := &bedrockruntime.InvokeModelWithResponseStreamInput{ - ModelId: aws.String(awsModelID), - Accept: aws.String("application/json"), - ContentType: aws.String("application/json"), - } - - convReq, ok := meta.Get(ConvertedRequest) - if !ok { - return utils.WrapErr(errors.New("request not found")), nil - } - claudeReq, ok := convReq.(*anthropic.Request) - if !ok { - return utils.WrapErr(errors.New("request not found")), nil - } - - awsClaudeReq := &Request{ - AnthropicVersion: "bedrock-2023-05-31", - } - if err = copier.Copy(awsClaudeReq, claudeReq); err != nil { - return utils.WrapErr(errors.Wrap(err, "copy request")), nil - } - awsReq.Body, err = sonic.Marshal(awsClaudeReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "marshal request")), nil - } - - awsClient, err := utils.AwsClientFromMeta(meta) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "get aws client")), nil - } - - awsResp, err := awsClient.InvokeModelWithResponseStream(c.Request.Context(), awsReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "InvokeModelWithResponseStream")), nil - } - stream := awsResp.GetStream() - defer stream.Close() - - c.Writer.Header().Set("Content-Type", "text/event-stream") - var usage relaymodel.Usage - var id string - var lastToolCallChoice *openai.ChatCompletionsStreamResponseChoice - var usageWrited bool - - c.Stream(func(_ io.Writer) bool { - event, ok := <-stream.Events() - if !ok { - render.Done(c) - return false - } - - switch v := event.(type) { - case *types.ResponseStreamMemberChunk: - claudeResp := anthropic.StreamResponse{} - err := sonic.Unmarshal(v.Value.Bytes, &claudeResp) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - return false - } - - response := anthropic.StreamResponseClaude2OpenAI(&claudeResp) - if response == nil { - return true - } - if response.Usage != nil { - usage = *response.Usage - usageWrited = true - - if lastToolCallChoice != nil && len(lastToolCallChoice.Delta.ToolCalls) > 0 { - lastArgs := &lastToolCallChoice.Delta.ToolCalls[len(lastToolCallChoice.Delta.ToolCalls)-1].Function - if len(lastArgs.Arguments) == 0 { // compatible with OpenAI sending an empty object `{}` when no arguments. - lastArgs.Arguments = "{}" - response.Choices[len(response.Choices)-1].Delta.Content = nil - response.Choices[len(response.Choices)-1].Delta.ToolCalls = lastToolCallChoice.Delta.ToolCalls - } - } - } - response.ID = id - response.Model = originModelName - response.Created = createdTime - - for _, choice := range response.Choices { - if len(choice.Delta.ToolCalls) > 0 { - lastToolCallChoice = choice - } - } - err = render.ObjectData(c, response) - if err != nil { - log.Error("error stream response: " + err.Error()) - return false - } - return true - case *types.UnknownUnionMember: - log.Error("unknown tag: " + v.Tag) - return false - default: - log.Errorf("union is nil or unknown type: %v", v) - return false - } - }) - - if !usageWrited { - _ = render.ObjectData(c, &openai.ChatCompletionsStreamResponse{ - Model: meta.OriginModel, - Object: "chat.completion.chunk", - Created: createdTime, - Choices: []*openai.ChatCompletionsStreamResponseChoice{}, - Usage: &usage, - }) - } - - return nil, &usage -} diff --git a/service/aiproxy/relay/adaptor/aws/claude/model.go b/service/aiproxy/relay/adaptor/aws/claude/model.go deleted file mode 100644 index d8a14dd62..000000000 --- a/service/aiproxy/relay/adaptor/aws/claude/model.go +++ /dev/null @@ -1,19 +0,0 @@ -package aws - -import "github.com/labring/sealos/service/aiproxy/relay/adaptor/anthropic" - -// Request is the request to AWS Claude -// -// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html -type Request struct { - ToolChoice any `json:"tool_choice,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - AnthropicVersion string `json:"anthropic_version"` - System string `json:"system,omitempty"` - Messages []anthropic.Message `json:"messages"` - StopSequences []string `json:"stop_sequences,omitempty"` - Tools []anthropic.Tool `json:"tools,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopK int `json:"top_k,omitempty"` -} diff --git a/service/aiproxy/relay/adaptor/aws/key.go b/service/aiproxy/relay/adaptor/aws/key.go deleted file mode 100644 index e2ef5f461..000000000 --- a/service/aiproxy/relay/adaptor/aws/key.go +++ /dev/null @@ -1,20 +0,0 @@ -package aws - -import ( - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, err := utils.GetAwsConfigFromKey(key) - if err != nil { - return err - } - return nil -} - -func (a *Adaptor) KeyHelp() string { - return "region|ak|sk" -} diff --git a/service/aiproxy/relay/adaptor/aws/llama3/adapter.go b/service/aiproxy/relay/adaptor/aws/llama3/adapter.go deleted file mode 100644 index ecfd2ee18..000000000 --- a/service/aiproxy/relay/adaptor/aws/llama3/adapter.go +++ /dev/null @@ -1,41 +0,0 @@ -package aws - -import ( - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - relayutils "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -const ( - ConvertedRequest = "convertedRequest" -) - -var _ utils.AwsAdapter = new(Adaptor) - -type Adaptor struct{} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := relayutils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - request.Model = meta.ActualModel - meta.Set("stream", request.Stream) - llamaReq := ConvertRequest(request) - meta.Set(ConvertedRequest, llamaReq) - return "", nil, nil, nil -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context) (usage *model.Usage, err *model.ErrorWithStatusCode) { - if meta.GetBool("stream") { - err, usage = StreamHandler(meta, c) - } else { - err, usage = Handler(meta, c) - } - return -} diff --git a/service/aiproxy/relay/adaptor/aws/llama3/main.go b/service/aiproxy/relay/adaptor/aws/llama3/main.go deleted file mode 100644 index 7b62f9bf6..000000000 --- a/service/aiproxy/relay/adaptor/aws/llama3/main.go +++ /dev/null @@ -1,264 +0,0 @@ -// Package aws provides the AWS adaptor for the relay service. -package aws - -import ( - "bytes" - "io" - "net/http" - "text/template" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/bedrockruntime" - "github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types" - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/random" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -type awsModelItem struct { - ID string - model.ModelConfig -} - -// AwsModelIDMap maps internal model identifiers to AWS model identifiers. -// It currently supports only llama-3-8b and llama-3-70b instruction models. -// For more details, see: https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids.html -var AwsModelIDMap = map[string]awsModelItem{ - "llama3-8b-8192": { - ModelConfig: model.ModelConfig{ - Model: "llama3-8b-8192", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - ID: "meta.llama3-8b-instruct-v1:0", - }, - "llama3-70b-8192": { - ModelConfig: model.ModelConfig{ - Model: "llama3-70b-8192", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - ID: "meta.llama3-70b-instruct-v1:0", - }, -} - -func awsModelID(requestModel string) (string, error) { - if awsModelID, ok := AwsModelIDMap[requestModel]; ok { - return awsModelID.ID, nil - } - - return "", errors.Errorf("model %s not found", requestModel) -} - -// promptTemplate with range -const promptTemplate = `<|begin_of_text|>{{range .Messages}}<|start_header_id|>{{.Role}}<|end_header_id|>{{.StringContent}}<|eot_id|>{{end}}<|start_header_id|>assistant<|end_header_id|> -` - -var promptTpl = template.Must(template.New("llama3-chat").Parse(promptTemplate)) - -func RenderPrompt(messages []*relaymodel.Message) string { - var buf bytes.Buffer - err := promptTpl.Execute(&buf, struct{ Messages []*relaymodel.Message }{messages}) - if err != nil { - log.Error("error rendering prompt messages: " + err.Error()) - } - return buf.String() -} - -func ConvertRequest(textRequest *relaymodel.GeneralOpenAIRequest) *Request { - llamaRequest := Request{ - MaxGenLen: textRequest.MaxTokens, - Temperature: textRequest.Temperature, - TopP: textRequest.TopP, - } - if llamaRequest.MaxGenLen == 0 { - llamaRequest.MaxGenLen = 2048 - } - prompt := RenderPrompt(textRequest.Messages) - llamaRequest.Prompt = prompt - return &llamaRequest -} - -func Handler(meta *meta.Meta, c *gin.Context) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) { - awsModelID, err := awsModelID(meta.ActualModel) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil - } - - awsReq := &bedrockruntime.InvokeModelInput{ - ModelId: aws.String(awsModelID), - Accept: aws.String("application/json"), - ContentType: aws.String("application/json"), - } - - llamaReq, ok := meta.Get(ConvertedRequest) - if !ok { - return utils.WrapErr(errors.New("request not found")), nil - } - - awsReq.Body, err = sonic.Marshal(llamaReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "marshal request")), nil - } - - awsClient, err := utils.AwsClientFromMeta(meta) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "get aws client")), nil - } - - awsResp, err := awsClient.InvokeModel(c.Request.Context(), awsReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "InvokeModel")), nil - } - - var llamaResponse Response - err = sonic.Unmarshal(awsResp.Body, &llamaResponse) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "unmarshal response")), nil - } - - openaiResp := ResponseLlama2OpenAI(&llamaResponse) - openaiResp.Model = meta.OriginModel - usage := relaymodel.Usage{ - PromptTokens: llamaResponse.PromptTokenCount, - CompletionTokens: llamaResponse.GenerationTokenCount, - TotalTokens: llamaResponse.PromptTokenCount + llamaResponse.GenerationTokenCount, - } - openaiResp.Usage = usage - - c.JSON(http.StatusOK, openaiResp) - return nil, &usage -} - -func ResponseLlama2OpenAI(llamaResponse *Response) *openai.TextResponse { - var responseText string - if len(llamaResponse.Generation) > 0 { - responseText = llamaResponse.Generation - } - choice := openai.TextResponseChoice{ - Index: 0, - Message: relaymodel.Message{ - Role: "assistant", - Content: responseText, - Name: nil, - }, - FinishReason: llamaResponse.StopReason, - } - fullTextResponse := openai.TextResponse{ - ID: "chatcmpl-" + random.GetUUID(), - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []*openai.TextResponseChoice{&choice}, - } - return &fullTextResponse -} - -func StreamHandler(meta *meta.Meta, c *gin.Context) (*relaymodel.ErrorWithStatusCode, *relaymodel.Usage) { - log := middleware.GetLogger(c) - - createdTime := time.Now().Unix() - awsModelID, err := awsModelID(meta.ActualModel) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "awsModelID")), nil - } - - awsReq := &bedrockruntime.InvokeModelWithResponseStreamInput{ - ModelId: aws.String(awsModelID), - Accept: aws.String("application/json"), - ContentType: aws.String("application/json"), - } - - llamaReq, ok := meta.Get(ConvertedRequest) - if !ok { - return utils.WrapErr(errors.New("request not found")), nil - } - - awsReq.Body, err = sonic.Marshal(llamaReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "marshal request")), nil - } - - awsClient, err := utils.AwsClientFromMeta(meta) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "get aws client")), nil - } - - awsResp, err := awsClient.InvokeModelWithResponseStream(c.Request.Context(), awsReq) - if err != nil { - return utils.WrapErr(errors.Wrap(err, "InvokeModelWithResponseStream")), nil - } - stream := awsResp.GetStream() - defer stream.Close() - - c.Writer.Header().Set("Content-Type", "text/event-stream") - var usage relaymodel.Usage - c.Stream(func(_ io.Writer) bool { - event, ok := <-stream.Events() - if !ok { - render.Done(c) - return false - } - - switch v := event.(type) { - case *types.ResponseStreamMemberChunk: - var llamaResp StreamResponse - err := sonic.Unmarshal(v.Value.Bytes, &llamaResp) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - return false - } - - if llamaResp.PromptTokenCount > 0 { - usage.PromptTokens = llamaResp.PromptTokenCount - } - if llamaResp.StopReason == constant.StopFinishReason { - usage.CompletionTokens = llamaResp.GenerationTokenCount - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - } - response := StreamResponseLlama2OpenAI(&llamaResp) - response.ID = "chatcmpl-" + random.GetUUID() - response.Model = meta.OriginModel - response.Created = createdTime - err = render.ObjectData(c, response) - if err != nil { - log.Error("error stream response: " + err.Error()) - return true - } - return true - case *types.UnknownUnionMember: - log.Error("unknown tag: " + v.Tag) - return false - default: - log.Errorf("union is nil or unknown type: %v", v) - return false - } - }) - - return nil, &usage -} - -func StreamResponseLlama2OpenAI(llamaResponse *StreamResponse) *openai.ChatCompletionsStreamResponse { - var choice openai.ChatCompletionsStreamResponseChoice - choice.Delta.Content = llamaResponse.Generation - choice.Delta.Role = "assistant" - finishReason := llamaResponse.StopReason - if finishReason != "null" { - choice.FinishReason = &finishReason - } - var openaiResponse openai.ChatCompletionsStreamResponse - openaiResponse.Object = "chat.completion.chunk" - openaiResponse.Choices = []*openai.ChatCompletionsStreamResponseChoice{&choice} - return &openaiResponse -} diff --git a/service/aiproxy/relay/adaptor/aws/llama3/main_test.go b/service/aiproxy/relay/adaptor/aws/llama3/main_test.go deleted file mode 100644 index 79d222ef6..000000000 --- a/service/aiproxy/relay/adaptor/aws/llama3/main_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package aws_test - -import ( - "testing" - - aws "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/llama3" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/stretchr/testify/assert" -) - -func TestRenderPrompt(t *testing.T) { - messages := []*relaymodel.Message{ - { - Role: "user", - Content: "What's your name?", - }, - } - prompt := aws.RenderPrompt(messages) - expected := `<|begin_of_text|><|start_header_id|>user<|end_header_id|>What's your name?<|eot_id|><|start_header_id|>assistant<|end_header_id|> -` - assert.Equal(t, expected, prompt) - - messages = []*relaymodel.Message{ - { - Role: "system", - Content: "Your name is Kat. You are a detective.", - }, - { - Role: "user", - Content: "What's your name?", - }, - { - Role: "assistant", - Content: "Kat", - }, - { - Role: "user", - Content: "What's your job?", - }, - } - prompt = aws.RenderPrompt(messages) - expected = `<|begin_of_text|><|start_header_id|>system<|end_header_id|>Your name is Kat. You are a detective.<|eot_id|><|start_header_id|>user<|end_header_id|>What's your name?<|eot_id|><|start_header_id|>assistant<|end_header_id|>Kat<|eot_id|><|start_header_id|>user<|end_header_id|>What's your job?<|eot_id|><|start_header_id|>assistant<|end_header_id|> -` - assert.Equal(t, expected, prompt) -} diff --git a/service/aiproxy/relay/adaptor/aws/llama3/model.go b/service/aiproxy/relay/adaptor/aws/llama3/model.go deleted file mode 100644 index 3d8ab8e09..000000000 --- a/service/aiproxy/relay/adaptor/aws/llama3/model.go +++ /dev/null @@ -1,29 +0,0 @@ -package aws - -// Request is the request to AWS Llama3 -// -// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html -type Request struct { - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - Prompt string `json:"prompt"` - MaxGenLen int `json:"max_gen_len,omitempty"` -} - -// Response is the response from AWS Llama3 -// -// https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-meta.html -type Response struct { - Generation string `json:"generation"` - StopReason string `json:"stop_reason"` - PromptTokenCount int `json:"prompt_token_count"` - GenerationTokenCount int `json:"generation_token_count"` -} - -// {'generation': 'Hi', 'prompt_token_count': 15, 'generation_token_count': 1, 'stop_reason': None} -type StreamResponse struct { - Generation string `json:"generation"` - StopReason string `json:"stop_reason"` - PromptTokenCount int `json:"prompt_token_count"` - GenerationTokenCount int `json:"generation_token_count"` -} diff --git a/service/aiproxy/relay/adaptor/aws/registry.go b/service/aiproxy/relay/adaptor/aws/registry.go deleted file mode 100644 index 734166618..000000000 --- a/service/aiproxy/relay/adaptor/aws/registry.go +++ /dev/null @@ -1,43 +0,0 @@ -package aws - -import ( - "github.com/labring/sealos/service/aiproxy/model" - claude "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/claude" - llama3 "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/llama3" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws/utils" -) - -type ModelType int - -const ( - AwsClaude ModelType = iota + 1 - AwsLlama3 -) - -type Model struct { - config *model.ModelConfig - _type ModelType -} - -var adaptors = map[string]Model{} - -func init() { - for _, model := range claude.AwsModelIDMap { - adaptors[model.Model] = Model{config: &model.ModelConfig, _type: AwsClaude} - } - for _, model := range llama3.AwsModelIDMap { - adaptors[model.Model] = Model{config: &model.ModelConfig, _type: AwsLlama3} - } -} - -func GetAdaptor(model string) utils.AwsAdapter { - adaptorType := adaptors[model] - switch adaptorType._type { - case AwsClaude: - return &claude.Adaptor{} - case AwsLlama3: - return &llama3.Adaptor{} - default: - return nil - } -} diff --git a/service/aiproxy/relay/adaptor/aws/utils/adaptor.go b/service/aiproxy/relay/adaptor/aws/utils/adaptor.go deleted file mode 100644 index cee82c555..000000000 --- a/service/aiproxy/relay/adaptor/aws/utils/adaptor.go +++ /dev/null @@ -1,68 +0,0 @@ -package utils - -import ( - "errors" - "io" - "net/http" - "strings" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/service/bedrockruntime" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type AwsAdapter interface { - ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) - DoResponse(meta *meta.Meta, c *gin.Context) (usage *model.Usage, err *model.ErrorWithStatusCode) -} - -type AwsConfig struct { - Region string - AK string - SK string -} - -func GetAwsConfigFromKey(key string) (*AwsConfig, error) { - split := strings.Split(key, "|") - if len(split) != 3 { - return nil, errors.New("invalid key format") - } - return &AwsConfig{ - Region: split[0], - AK: split[1], - SK: split[2], - }, nil -} - -func AwsClient(config *AwsConfig) *bedrockruntime.Client { - return bedrockruntime.New(bedrockruntime.Options{ - Region: config.Region, - Credentials: aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(config.AK, config.SK, "")), - }) -} - -func awsClientFromKey(key string) (*bedrockruntime.Client, error) { - config, err := GetAwsConfigFromKey(key) - if err != nil { - return nil, err - } - return AwsClient(config), nil -} - -const AwsClientKey = "aws_client" - -func AwsClientFromMeta(meta *meta.Meta) (*bedrockruntime.Client, error) { - awsClientI, ok := meta.Get(AwsClientKey) - if ok { - return awsClientI.(*bedrockruntime.Client), nil - } - awsClient, err := awsClientFromKey(meta.Channel.Key) - if err != nil { - return nil, err - } - meta.Set(AwsClientKey, awsClient) - return awsClient, nil -} diff --git a/service/aiproxy/relay/adaptor/aws/utils/utils.go b/service/aiproxy/relay/adaptor/aws/utils/utils.go deleted file mode 100644 index 0323f8c21..000000000 --- a/service/aiproxy/relay/adaptor/aws/utils/utils.go +++ /dev/null @@ -1,16 +0,0 @@ -package utils - -import ( - "net/http" - - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func WrapErr(err error) *relaymodel.ErrorWithStatusCode { - return &relaymodel.ErrorWithStatusCode{ - StatusCode: http.StatusInternalServerError, - Error: relaymodel.Error{ - Message: err.Error(), - }, - } -} diff --git a/service/aiproxy/relay/adaptor/azure/constants.go b/service/aiproxy/relay/adaptor/azure/constants.go deleted file mode 100644 index 86a2cf76d..000000000 --- a/service/aiproxy/relay/adaptor/azure/constants.go +++ /dev/null @@ -1,62 +0,0 @@ -package azure - -import ( - "fmt" - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type Adaptor struct { - openai.Adaptor -} - -func (a *Adaptor) GetBaseURL() string { - return "https://{resource_name}.openai.azure.com" -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - _, apiVersion, err := getTokenAndAPIVersion(meta.Channel.Key) - if err != nil { - return "", err - } - model := strings.ReplaceAll(meta.ActualModel, ".", "") - switch meta.Mode { - case relaymode.ImagesGenerations: - // https://learn.microsoft.com/en-us/azure/ai-services/openai/dall-e-quickstart?tabs=dalle3%2Ccommand-line&pivots=rest-api - // https://{resource_name}.openai.azure.com/openai/deployments/dall-e-3/images/generations?api-version=2024-03-01-preview - return fmt.Sprintf("%s/openai/deployments/%s/images/generations?api-version=%s", meta.Channel.BaseURL, model, apiVersion), nil - case relaymode.AudioTranscription: - // https://learn.microsoft.com/en-us/azure/ai-services/openai/whisper-quickstart?tabs=command-line#rest-api - return fmt.Sprintf("%s/openai/deployments/%s/audio/transcriptions?api-version=%s", meta.Channel.BaseURL, model, apiVersion), nil - case relaymode.AudioSpeech: - // https://learn.microsoft.com/en-us/azure/ai-services/openai/text-to-speech-quickstart?tabs=command-line#rest-api - return fmt.Sprintf("%s/openai/deployments/%s/audio/speech?api-version=%s", meta.Channel.BaseURL, model, apiVersion), nil - case relaymode.ChatCompletions: - // https://learn.microsoft.com/en-us/azure/cognitive-services/openai/chatgpt-quickstart?pivots=rest-api&tabs=command-line#rest-api - return fmt.Sprintf("%s/openai/deployments/%s/chat/completions?api-version=%s", meta.Channel.BaseURL, model, apiVersion), nil - case relaymode.Completions: - return fmt.Sprintf("%s/openai/deployments/%s/completions?api-version=%s", meta.Channel.BaseURL, model, apiVersion), nil - case relaymode.Embeddings: - return fmt.Sprintf("%s/openai/deployments/%s/embeddings?api-version=%s", meta.Channel.BaseURL, model, apiVersion), nil - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - token, _, err := getTokenAndAPIVersion(meta.Channel.Key) - if err != nil { - return err - } - req.Header.Set("Api-Key", token) - return nil -} - -func (a *Adaptor) GetChannelName() string { - return "azure" -} diff --git a/service/aiproxy/relay/adaptor/azure/key.go b/service/aiproxy/relay/adaptor/azure/key.go deleted file mode 100644 index dde766f7e..000000000 --- a/service/aiproxy/relay/adaptor/azure/key.go +++ /dev/null @@ -1,35 +0,0 @@ -package azure - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, _, err := getTokenAndAPIVersion(key) - if err != nil { - return err - } - return nil -} - -func (a *Adaptor) KeyHelp() string { - return "key or key|api-version" -} - -const defaultAPIVersion = "2024-02-01" - -func getTokenAndAPIVersion(key string) (string, string, error) { - split := strings.Split(key, "|") - if len(split) == 1 { - return key, defaultAPIVersion, nil - } - if len(split) != 2 { - return "", "", errors.New("invalid key format") - } - return split[0], split[1], nil -} diff --git a/service/aiproxy/relay/adaptor/baichuan/adaptor.go b/service/aiproxy/relay/adaptor/baichuan/adaptor.go deleted file mode 100644 index 481ec0792..000000000 --- a/service/aiproxy/relay/adaptor/baichuan/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package baichuan - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.baichuan-ai.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "baichuan" -} diff --git a/service/aiproxy/relay/adaptor/baichuan/constants.go b/service/aiproxy/relay/adaptor/baichuan/constants.go deleted file mode 100644 index a3ad41492..000000000 --- a/service/aiproxy/relay/adaptor/baichuan/constants.go +++ /dev/null @@ -1,69 +0,0 @@ -package baichuan - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "Baichuan4-Turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaichuan, - InputPrice: 0.015, - OutputPrice: 0.015, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - ), - }, - { - Model: "Baichuan4-Air", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaichuan, - InputPrice: 0.00098, - OutputPrice: 0.00098, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - ), - }, - { - Model: "Baichuan4", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaichuan, - InputPrice: 0.1, - OutputPrice: 0.1, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - ), - }, - { - Model: "Baichuan3-Turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaichuan, - InputPrice: 0.012, - OutputPrice: 0.012, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - ), - }, - { - Model: "Baichuan3-Turbo-128k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaichuan, - InputPrice: 0.024, - OutputPrice: 0.024, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, - - { - Model: "Baichuan-Text-Embedding", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerBaichuan, - InputPrice: 0.0005, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(512), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/baidu/adaptor.go b/service/aiproxy/relay/adaptor/baidu/adaptor.go deleted file mode 100644 index b3a607bc7..000000000 --- a/service/aiproxy/relay/adaptor/baidu/adaptor.go +++ /dev/null @@ -1,138 +0,0 @@ -package baidu - -import ( - "context" - "fmt" - "io" - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const ( - baseURL = "https://aip.baidubce.com" -) - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -// Get model-specific endpoint using map -var modelEndpointMap = map[string]string{ - "ERNIE-4.0-8K": "completions_pro", - "ERNIE-4.0": "completions_pro", - "ERNIE-Bot-4": "completions_pro", - "ERNIE-Bot": "completions", - "ERNIE-Bot-turbo": "eb-instant", - "ERNIE-Speed": "ernie_speed", - "ERNIE-3.5-8K": "completions", - "ERNIE-Bot-8K": "ernie_bot_8k", - "ERNIE-Speed-8K": "ernie_speed", - "ERNIE-Lite-8K-0922": "eb-instant", - "ERNIE-Lite-8K-0308": "ernie-lite-8k", - "BLOOMZ-7B": "bloomz_7b1", - "bge-large-zh": "bge_large_zh", - "bge-large-en": "bge_large_en", - "tao-8k": "tao_8k", - "bce-reranker-base_v1": "bce_reranker_base", - "Stable-Diffusion-XL": "sd_xl", - "Fuyu-8B": "fuyu_8b", -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - // Build base URL - if meta.Channel.BaseURL == "" { - meta.Channel.BaseURL = baseURL - } - - // Get API path suffix based on mode - var pathSuffix string - switch meta.Mode { - case relaymode.ChatCompletions: - pathSuffix = "chat" - case relaymode.Embeddings: - pathSuffix = "embeddings" - case relaymode.Rerank: - pathSuffix = "reranker" - case relaymode.ImagesGenerations: - pathSuffix = "text2image" - } - - modelEndpoint, ok := modelEndpointMap[meta.ActualModel] - if !ok { - modelEndpoint = strings.ToLower(meta.ActualModel) - } - - // Construct full URL - fullURL := fmt.Sprintf("%s/rpc/2.0/ai_custom/v1/wenxinworkshop/%s/%s", - meta.Channel.BaseURL, pathSuffix, modelEndpoint) - - return fullURL, nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - accessToken, err := GetAccessToken(context.Background(), meta.Channel.Key) - if err != nil { - return err - } - req.URL.RawQuery = "access_token=" + accessToken - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.Embeddings: - meta.Set(openai.MetaEmbeddingsPatchInputToSlices, true) - return openai.ConvertRequest(meta, req) - case relaymode.Rerank: - return openai.ConvertRequest(meta, req) - case relaymode.ImagesGenerations: - return openai.ConvertRequest(meta, req) - case relaymode.ChatCompletions: - return ConvertRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.Embeddings: - usage, err = EmbeddingsHandler(meta, c, resp) - case relaymode.Rerank: - usage, err = RerankHandler(meta, c, resp) - case relaymode.ImagesGenerations: - usage, err = ImageHandler(meta, c, resp) - case relaymode.ChatCompletions: - if utils.IsStreamResponse(resp) { - err, usage = StreamHandler(meta, c, resp) - } else { - usage, err = Handler(meta, c, resp) - } - default: - return nil, openai.ErrorWrapperWithMessage(fmt.Sprintf("unsupported mode: %s", meta.Mode), "unsupported_mode", http.StatusBadRequest) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "baidu" -} diff --git a/service/aiproxy/relay/adaptor/baidu/constants.go b/service/aiproxy/relay/adaptor/baidu/constants.go deleted file mode 100644 index 55a9c3d61..000000000 --- a/service/aiproxy/relay/adaptor/baidu/constants.go +++ /dev/null @@ -1,81 +0,0 @@ -package baidu - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "BLOOMZ-7B", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.004, - OutputPrice: 0.004, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4800), - ), - }, - - { - Model: "Embedding-V1", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0005, - OutputPrice: 0, - RPM: 1200, - }, - { - Model: "bge-large-zh", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerBAAI, - InputPrice: 0.0005, - OutputPrice: 0, - RPM: 1200, - }, - { - Model: "bge-large-en", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerBAAI, - InputPrice: 0.0005, - OutputPrice: 0, - RPM: 1200, - }, - { - Model: "tao-8k", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0005, - OutputPrice: 0, - RPM: 1200, - }, - - { - Model: "bce-reranker-base_v1", - Type: relaymode.Rerank, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0005, - OutputPrice: 0, - RPM: 1200, - }, - - { - Model: "Stable-Diffusion-XL", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - ImagePrices: map[string]float64{ - "768x768": 0.06, - "576x1024": 0.06, - "1024x576": 0.06, - "768x1024": 0.08, - "1024x768": 0.08, - "1024x1024": 0.08, - "1536x1536": 0.12, - "1152x2048": 0.12, - "2048x1152": 0.12, - "1536x2048": 0.16, - "2048x1536": 0.16, - "2048x2048": 0.16, - }, - }, -} diff --git a/service/aiproxy/relay/adaptor/baidu/embeddings.go b/service/aiproxy/relay/adaptor/baidu/embeddings.go deleted file mode 100644 index c9a9cf685..000000000 --- a/service/aiproxy/relay/adaptor/baidu/embeddings.go +++ /dev/null @@ -1,55 +0,0 @@ -package baidu - -import ( - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type EmbeddingsResponse struct { - *Error - Usage relaymodel.Usage `json:"usage"` -} - -func EmbeddingsHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - var baiduResponse EmbeddingsResponse - err = sonic.Unmarshal(body, &baiduResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if baiduResponse.Error != nil && baiduResponse.ErrorCode != 0 { - return &baiduResponse.Usage, ErrorHandler(baiduResponse.Error) - } - - respMap := make(map[string]any) - err = sonic.Unmarshal(body, &respMap) - if err != nil { - return &baiduResponse.Usage, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - respMap["model"] = meta.OriginModel - respMap["object"] = "list" - - data, err := sonic.Marshal(respMap) - if err != nil { - return &baiduResponse.Usage, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - _, err = c.Writer.Write(data) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return &baiduResponse.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/baidu/error.go b/service/aiproxy/relay/adaptor/baidu/error.go deleted file mode 100644 index c311dafa2..000000000 --- a/service/aiproxy/relay/adaptor/baidu/error.go +++ /dev/null @@ -1,48 +0,0 @@ -package baidu - -import ( - "net/http" - "strconv" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -// https://cloud.baidu.com/doc/WENXINWORKSHOP/s/tlmyncueh - -func ErrorHandler(baiduError *Error) *relaymodel.ErrorWithStatusCode { - switch baiduError.ErrorCode { - case 13, 14, 100, 110: - return openai.ErrorWrapperWithMessage( - baiduError.ErrorMsg, - "upstream_"+strconv.Itoa(baiduError.ErrorCode), - http.StatusUnauthorized, - ) - case 17, 19, 111: - return openai.ErrorWrapperWithMessage( - baiduError.ErrorMsg, - "upstream_"+strconv.Itoa(baiduError.ErrorCode), - http.StatusForbidden, - ) - case 336001, 336002, 336003, - 336005, 336006, 336007, - 336008, 336103, 336104, - 336106, 336118, 336122, - 336123, 336221, 337006, - 337008, 337009: - return openai.ErrorWrapperWithMessage( - baiduError.ErrorMsg, - "upstream_"+strconv.Itoa(baiduError.ErrorCode), - http.StatusBadRequest, - ) - case 4, 18, 336117, 336501, 336502, - 336503, 336504, 336505, - 336507: - return openai.ErrorWrapperWithMessage( - baiduError.ErrorMsg, - "upstream_"+strconv.Itoa(baiduError.ErrorCode), - http.StatusTooManyRequests, - ) - } - return openai.ErrorWrapperWithMessage(baiduError.ErrorMsg, "upstream_"+strconv.Itoa(baiduError.ErrorCode), http.StatusInternalServerError) -} diff --git a/service/aiproxy/relay/adaptor/baidu/image.go b/service/aiproxy/relay/adaptor/baidu/image.go deleted file mode 100644 index 8a717314d..000000000 --- a/service/aiproxy/relay/adaptor/baidu/image.go +++ /dev/null @@ -1,72 +0,0 @@ -package baidu - -import ( - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type ImageData struct { - B64Image string `json:"b64_image"` -} - -type ImageResponse struct { - *Error - ID string `json:"id"` - Data []*ImageData `json:"data"` - Created int64 `json:"created"` -} - -func ImageHandler(_ *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - var imageResponse ImageResponse - err = sonic.Unmarshal(body, &imageResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - usage := &model.Usage{ - PromptTokens: len(imageResponse.Data), - TotalTokens: len(imageResponse.Data), - } - - if imageResponse.Error != nil && imageResponse.Error.ErrorMsg != "" { - return usage, ErrorHandler(imageResponse.Error) - } - - openaiResponse := ToOpenAIImageResponse(&imageResponse) - data, err := sonic.Marshal(openaiResponse) - if err != nil { - return usage, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - _, err = c.Writer.Write(data) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return usage, nil -} - -func ToOpenAIImageResponse(imageResponse *ImageResponse) *openai.ImageResponse { - response := &openai.ImageResponse{ - Created: imageResponse.Created, - } - for _, data := range imageResponse.Data { - response.Data = append(response.Data, &openai.ImageData{ - B64Json: data.B64Image, - }) - } - return response -} diff --git a/service/aiproxy/relay/adaptor/baidu/key.go b/service/aiproxy/relay/adaptor/baidu/key.go deleted file mode 100644 index 54592f09b..000000000 --- a/service/aiproxy/relay/adaptor/baidu/key.go +++ /dev/null @@ -1,28 +0,0 @@ -package baidu - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, _, err := getClientIDAndSecret(key) - return err -} - -func (a *Adaptor) KeyHelp() string { - return "client_id|client_secret" -} - -// key格式: client_id|client_secret -func getClientIDAndSecret(key string) (string, string, error) { - parts := strings.Split(key, "|") - if len(parts) != 2 { - return "", "", errors.New("invalid key format") - } - return parts[0], parts[1], nil -} diff --git a/service/aiproxy/relay/adaptor/baidu/main.go b/service/aiproxy/relay/adaptor/baidu/main.go deleted file mode 100644 index 3d7839c2c..000000000 --- a/service/aiproxy/relay/adaptor/baidu/main.go +++ /dev/null @@ -1,192 +0,0 @@ -package baidu - -import ( - "bufio" - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -// https://cloud.baidu.com/doc/WENXINWORKSHOP/s/flfmc9do2 - -type Message struct { - Role string `json:"role"` - Content string `json:"content"` -} - -type ChatRequest struct { - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - PenaltyScore *float64 `json:"penalty_score,omitempty"` - System string `json:"system,omitempty"` - UserID string `json:"user_id,omitempty"` - Messages []*model.Message `json:"messages"` - MaxOutputTokens int `json:"max_output_tokens,omitempty"` - Stream bool `json:"stream,omitempty"` - DisableSearch bool `json:"disable_search,omitempty"` - EnableCitation bool `json:"enable_citation,omitempty"` -} - -func ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - request.Model = meta.ActualModel - baiduRequest := ChatRequest{ - Messages: request.Messages, - Temperature: request.Temperature, - TopP: request.TopP, - Stream: request.Stream, - DisableSearch: false, - EnableCitation: false, - MaxOutputTokens: request.MaxTokens, - UserID: request.User, - } - // Convert frequency penalty to penalty score range [1.0, 2.0] - if request.FrequencyPenalty != nil { - penaltyScore := *request.FrequencyPenalty - if penaltyScore < -2.0 { - penaltyScore = -2.0 - } - if penaltyScore > 2.0 { - penaltyScore = 2.0 - } - // Map [-2.0, 2.0] to [1.0, 2.0] - mappedScore := (penaltyScore+2.0)/4.0 + 1.0 - baiduRequest.PenaltyScore = &mappedScore - } - - for i, message := range request.Messages { - if message.Role == "system" { - baiduRequest.System = message.StringContent() - request.Messages = append(request.Messages[:i], request.Messages[i+1:]...) - break - } - } - - data, err := sonic.Marshal(baiduRequest) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func responseBaidu2OpenAI(response *ChatResponse) *openai.TextResponse { - choice := openai.TextResponseChoice{ - Index: 0, - Message: model.Message{ - Role: "assistant", - Content: response.Result, - }, - FinishReason: constant.StopFinishReason, - } - fullTextResponse := openai.TextResponse{ - ID: response.ID, - Object: "chat.completion", - Created: response.Created, - Choices: []*openai.TextResponseChoice{&choice}, - } - if response.Usage != nil { - fullTextResponse.Usage = *response.Usage - } - return &fullTextResponse -} - -func streamResponseBaidu2OpenAI(meta *meta.Meta, baiduResponse *ChatStreamResponse) *openai.ChatCompletionsStreamResponse { - var choice openai.ChatCompletionsStreamResponseChoice - choice.Delta.Content = baiduResponse.Result - if baiduResponse.IsEnd { - choice.FinishReason = &constant.StopFinishReason - } - response := openai.ChatCompletionsStreamResponse{ - ID: baiduResponse.ID, - Object: "chat.completion.chunk", - Created: baiduResponse.Created, - Model: meta.OriginModel, - Choices: []*openai.ChatCompletionsStreamResponseChoice{&choice}, - Usage: baiduResponse.Usage, - } - return &response -} - -func StreamHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *model.Usage) { - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - var usage model.Usage - scanner := bufio.NewScanner(resp.Body) - scanner.Split(bufio.ScanLines) - - common.SetEventStreamHeaders(c) - - for scanner.Scan() { - data := scanner.Bytes() - if len(data) < 6 || conv.BytesToString(data[:6]) != "data: " { - continue - } - data = data[6:] - - if conv.BytesToString(data) == "[DONE]" { - break - } - - var baiduResponse ChatStreamResponse - err := sonic.Unmarshal(data, &baiduResponse) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - if baiduResponse.Usage != nil { - usage.TotalTokens = baiduResponse.Usage.TotalTokens - usage.PromptTokens = baiduResponse.Usage.PromptTokens - usage.CompletionTokens = baiduResponse.Usage.TotalTokens - baiduResponse.Usage.PromptTokens - } - response := streamResponseBaidu2OpenAI(meta, &baiduResponse) - _ = render.ObjectData(c, response) - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - render.Done(c) - - return nil, &usage -} - -func Handler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - defer resp.Body.Close() - - var baiduResponse ChatResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&baiduResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if baiduResponse.Error != nil && baiduResponse.Error.ErrorCode != 0 { - return nil, ErrorHandler(baiduResponse.Error) - } - fullTextResponse := responseBaidu2OpenAI(&baiduResponse) - fullTextResponse.Model = meta.OriginModel - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &fullTextResponse.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/baidu/model.go b/service/aiproxy/relay/adaptor/baidu/model.go deleted file mode 100644 index cff514641..000000000 --- a/service/aiproxy/relay/adaptor/baidu/model.go +++ /dev/null @@ -1,32 +0,0 @@ -package baidu - -import ( - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type Error struct { - ErrorMsg string `json:"error_msg"` - ErrorCode int `json:"error_code"` -} - -type ErrorResponse struct { - *Error `json:"error"` - ID string `json:"id"` -} - -type ChatResponse struct { - Usage *model.Usage `json:"usage"` - *Error `json:"error"` - ID string `json:"id"` - Object string `json:"object"` - Result string `json:"result"` - Created int64 `json:"created"` - IsTruncated bool `json:"is_truncated"` - NeedClearHistory bool `json:"need_clear_history"` -} - -type ChatStreamResponse struct { - ChatResponse - SentenceID int `json:"sentence_id"` - IsEnd bool `json:"is_end"` -} diff --git a/service/aiproxy/relay/adaptor/baidu/rerank.go b/service/aiproxy/relay/adaptor/baidu/rerank.go deleted file mode 100644 index 0e52433a2..000000000 --- a/service/aiproxy/relay/adaptor/baidu/rerank.go +++ /dev/null @@ -1,61 +0,0 @@ -package baidu - -import ( - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type RerankResponse struct { - Error *Error `json:"error"` - Usage model.Usage `json:"usage"` -} - -func RerankHandler(_ *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - reRankResp := &RerankResponse{} - err = sonic.Unmarshal(respBody, reRankResp) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if reRankResp.Error != nil && reRankResp.Error.ErrorCode != 0 { - return nil, ErrorHandler(reRankResp.Error) - } - respMap := make(map[string]any) - err = sonic.Unmarshal(respBody, &respMap) - if err != nil { - return &reRankResp.Usage, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - delete(respMap, "model") - delete(respMap, "usage") - respMap["meta"] = &model.RerankMeta{ - Tokens: &model.RerankMetaTokens{ - InputTokens: reRankResp.Usage.TotalTokens, - OutputTokens: 0, - }, - } - respMap["result"] = respMap["results"] - delete(respMap, "results") - jsonData, err := sonic.Marshal(respMap) - if err != nil { - return &reRankResp.Usage, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - _, err = c.Writer.Write(jsonData) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return &reRankResp.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/baidu/token.go b/service/aiproxy/relay/adaptor/baidu/token.go deleted file mode 100644 index 248155484..000000000 --- a/service/aiproxy/relay/adaptor/baidu/token.go +++ /dev/null @@ -1,88 +0,0 @@ -package baidu - -import ( - "context" - "errors" - "fmt" - "net/http" - "sync" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/relay/utils" - log "github.com/sirupsen/logrus" -) - -type AccessToken struct { - ExpiresAt time.Time `json:"-"` - AccessToken string `json:"access_token"` - Error string `json:"error,omitempty"` - ErrorDescription string `json:"error_description,omitempty"` - ExpiresIn int64 `json:"expires_in,omitempty"` -} - -var baiduTokenStore sync.Map - -func GetAccessToken(ctx context.Context, apiKey string) (string, error) { - if val, ok := baiduTokenStore.Load(apiKey); ok { - var accessToken AccessToken - if accessToken, ok = val.(AccessToken); ok { - // soon this will expire - if time.Now().Add(time.Hour).After(accessToken.ExpiresAt) { - go func() { - _, err := getBaiduAccessTokenHelper(context.Background(), apiKey) - if err != nil { - log.Errorf("get baidu access token failed: %v", err) - } - }() - } - return accessToken.AccessToken, nil - } - } - accessToken, err := getBaiduAccessTokenHelper(ctx, apiKey) - if err != nil { - log.Errorf("get baidu access token failed: %v", err) - return "", errors.New("get baidu access token failed") - } - if accessToken == nil { - return "", errors.New("get baidu access token return a nil token") - } - return accessToken.AccessToken, nil -} - -func getBaiduAccessTokenHelper(ctx context.Context, apiKey string) (*AccessToken, error) { - clientID, clientSecret, err := getClientIDAndSecret(apiKey) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, - http.MethodPost, - fmt.Sprintf("https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=%s&client_secret=%s", - clientID, clientSecret), - nil) - if err != nil { - return nil, err - } - req.Header.Add("Content-Type", "application/json") - req.Header.Add("Accept", "application/json") - res, err := utils.DoRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - var accessToken AccessToken - err = sonic.ConfigDefault.NewDecoder(res.Body).Decode(&accessToken) - if err != nil { - return nil, err - } - if accessToken.Error != "" { - return nil, errors.New(accessToken.Error + ": " + accessToken.ErrorDescription) - } - if accessToken.AccessToken == "" { - return nil, errors.New("get baidu access token return empty access token") - } - accessToken.ExpiresAt = time.Now().Add(time.Duration(accessToken.ExpiresIn) * time.Second) - baiduTokenStore.Store(apiKey, accessToken) - return &accessToken, nil -} diff --git a/service/aiproxy/relay/adaptor/baiduv2/adaptor.go b/service/aiproxy/relay/adaptor/baiduv2/adaptor.go deleted file mode 100644 index c8273f321..000000000 --- a/service/aiproxy/relay/adaptor/baiduv2/adaptor.go +++ /dev/null @@ -1,98 +0,0 @@ -package baiduv2 - -import ( - "context" - "fmt" - "io" - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const ( - baseURL = "https://qianfan.baidubce.com/v2" -) - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -// https://cloud.baidu.com/doc/WENXINWORKSHOP/s/Fm2vrveyu -var v2ModelMap = map[string]string{ - "ERNIE-Character-8K": "ernie-char-8k", - "ERNIE-Character-Fiction-8K": "ernie-char-fiction-8k", -} - -func toV2ModelName(modelName string) string { - if v2Model, ok := v2ModelMap[modelName]; ok { - return v2Model - } - return strings.ToLower(modelName) -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - switch meta.Mode { - case relaymode.ChatCompletions: - return meta.Channel.BaseURL + "/chat/completions", nil - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - token, err := GetBearerToken(context.Background(), meta.Channel.Key) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+token.Token) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.ChatCompletions: - actModel := meta.ActualModel - v2Model := toV2ModelName(actModel) - if v2Model != actModel { - meta.ActualModel = v2Model - defer func() { meta.ActualModel = actModel }() - } - return openai.ConvertRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.ChatCompletions: - return openai.DoResponse(meta, c, resp) - default: - return nil, openai.ErrorWrapperWithMessage( - fmt.Sprintf("unsupported mode: %s", meta.Mode), - nil, - http.StatusBadRequest, - ) - } -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "baidu v2" -} diff --git a/service/aiproxy/relay/adaptor/baiduv2/constants.go b/service/aiproxy/relay/adaptor/baiduv2/constants.go deleted file mode 100644 index 61f6f21ef..000000000 --- a/service/aiproxy/relay/adaptor/baiduv2/constants.go +++ /dev/null @@ -1,294 +0,0 @@ -package baiduv2 - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://cloud.baidu.com/doc/WENXINWORKSHOP/s/Fm2vrveyu - -var ModelList = []*model.ModelConfig{ - { - Model: "ERNIE-4.0-8K-Latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.03, - OutputPrice: 0.09, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(5120), - model.WithModelConfigMaxInputTokens(5120), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-4.0-8K-Preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.03, - OutputPrice: 0.09, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(5120), - model.WithModelConfigMaxInputTokens(5120), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-4.0-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.03, - OutputPrice: 0.09, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(5120), - model.WithModelConfigMaxInputTokens(5120), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-4.0-Turbo-8K-Latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.02, - OutputPrice: 0.06, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-4.0-Turbo-8K-Preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.02, - OutputPrice: 0.06, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-4.0-Turbo-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.02, - OutputPrice: 0.06, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-4.0-Turbo-128K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.02, - OutputPrice: 0.06, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(126976), - model.WithModelConfigMaxInputTokens(126976), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-3.5-8K-Preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0008, - OutputPrice: 0.002, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(5120), - model.WithModelConfigMaxInputTokens(5120), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-3.5-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0008, - OutputPrice: 0.002, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(5120), - model.WithModelConfigMaxInputTokens(5120), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-3.5-128K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0008, - OutputPrice: 0.002, - RPM: 5000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(126976), - model.WithModelConfigMaxInputTokens(126976), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-Speed-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(7168), - model.WithModelConfigMaxInputTokens(7168), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - { - Model: "ERNIE-Speed-128K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(126976), - model.WithModelConfigMaxInputTokens(126976), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "ERNIE-Speed-Pro-128K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(126976), - model.WithModelConfigMaxInputTokens(126976), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "ERNIE-Lite-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - { - Model: "ERNIE-Lite-Pro-128K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0002, - OutputPrice: 0.0004, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(126976), - model.WithModelConfigMaxInputTokens(126976), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "ERNIE-Tiny-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - { - Model: "ERNIE-Character-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - { - Model: "ERNIE-Character-Fiction-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(5120), - model.WithModelConfigMaxInputTokens(5120), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - { - Model: "ERNIE-Novel-8K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerBaidu, - InputPrice: 0.04, - OutputPrice: 0.12, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(6144), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - - { - Model: "DeepSeek-V3", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - InputPrice: 0.0008, - OutputPrice: 0.0016, - RPM: 1000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(64000), - model.WithModelConfigMaxOutputTokens(8192), - ), - }, - { - Model: "DeepSeek-R1", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - InputPrice: 0.002, - OutputPrice: 0.008, - RPM: 1000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(64000), - model.WithModelConfigMaxOutputTokens(8192), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/baiduv2/key.go b/service/aiproxy/relay/adaptor/baiduv2/key.go deleted file mode 100644 index 0aa103a96..000000000 --- a/service/aiproxy/relay/adaptor/baiduv2/key.go +++ /dev/null @@ -1,28 +0,0 @@ -package baiduv2 - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, _, err := getAKAndSK(key) - return err -} - -func (a *Adaptor) KeyHelp() string { - return "ak|sk" -} - -// key格式: ak|sk -func getAKAndSK(key string) (string, string, error) { - parts := strings.Split(key, "|") - if len(parts) != 2 { - return "", "", errors.New("invalid key format") - } - return parts[0], parts[1], nil -} diff --git a/service/aiproxy/relay/adaptor/baiduv2/token.go b/service/aiproxy/relay/adaptor/baiduv2/token.go deleted file mode 100644 index 4bbb25d64..000000000 --- a/service/aiproxy/relay/adaptor/baiduv2/token.go +++ /dev/null @@ -1,108 +0,0 @@ -package baiduv2 - -import ( - "context" - "crypto/hmac" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "net/http" - "strings" - "sync" - "time" - - "github.com/bytedance/sonic" - log "github.com/sirupsen/logrus" -) - -type TokenResponse struct { - ExpireTime time.Time `json:"expireTime"` - Token string `json:"token"` -} - -var baiduTokenStore sync.Map - -func GetBearerToken(ctx context.Context, apiKey string) (*TokenResponse, error) { - parts := strings.Split(apiKey, "|") - if len(parts) != 2 { - return nil, errors.New("invalid baidu apikey") - } - if val, ok := baiduTokenStore.Load("bearer|" + apiKey); ok { - var tokenResponse TokenResponse - if tokenResponse, ok = val.(TokenResponse); ok { - if time.Now().Add(time.Hour).After(tokenResponse.ExpireTime) { - go func() { - _, err := getBaiduAccessTokenHelper(context.Background(), apiKey) - if err != nil { - log.Errorf("get baidu access token failed: %v", err) - } - }() - } - return &tokenResponse, nil - } - } - tokenResponse, err := getBaiduAccessTokenHelper(ctx, apiKey) - if err != nil { - return nil, err - } - return tokenResponse, nil -} - -func getBaiduAccessTokenHelper(ctx context.Context, apiKey string) (*TokenResponse, error) { - ak, sk, err := getAKAndSK(apiKey) - if err != nil { - return nil, err - } - authorization := generateAuthorizationString(ak, sk) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://iam.bj.baidubce.com/v1/BCE-BEARER/token", nil) - if err != nil { - return nil, err - } - query := req.URL.Query() - query.Add("expireInSeconds", "86400") - req.URL.RawQuery = query.Encode() - req.Header.Set("Authorization", authorization) - res, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusCreated { - return nil, fmt.Errorf("get token failed, status code: %d", res.StatusCode) - } - var tokenResponse TokenResponse - err = sonic.ConfigDefault.NewDecoder(res.Body).Decode(&tokenResponse) - if err != nil { - return nil, err - } - baiduTokenStore.Store("bearer|"+apiKey, tokenResponse) - return &tokenResponse, nil -} - -func generateAuthorizationString(ak, sk string) string { - httpMethod := http.MethodGet - uri := "/v1/BCE-BEARER/token" - queryString := "expireInSeconds=86400" - hostHeader := "iam.bj.baidubce.com" - canonicalRequest := fmt.Sprintf("%s\n%s\n%s\nhost:%s", httpMethod, uri, queryString, hostHeader) - - timestamp := time.Now().UTC().Format("2006-01-02T15:04:05Z") - expirationPeriodInSeconds := 1800 - authStringPrefix := fmt.Sprintf("bce-auth-v1/%s/%s/%d", ak, timestamp, expirationPeriodInSeconds) - - signingKey := hmacSHA256(sk, authStringPrefix) - - signature := hmacSHA256(signingKey, canonicalRequest) - - signedHeaders := "host" - authorization := fmt.Sprintf("%s/%s/%s", authStringPrefix, signedHeaders, signature) - - return authorization -} - -func hmacSHA256(key, data string) string { - h := hmac.New(sha256.New, []byte(key)) - h.Write([]byte(data)) - return hex.EncodeToString(h.Sum(nil)) -} diff --git a/service/aiproxy/relay/adaptor/cloudflare/adaptor.go b/service/aiproxy/relay/adaptor/cloudflare/adaptor.go deleted file mode 100644 index 47a80e8e6..000000000 --- a/service/aiproxy/relay/adaptor/cloudflare/adaptor.go +++ /dev/null @@ -1,59 +0,0 @@ -package cloudflare - -import ( - "fmt" - "strings" - - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.cloudflare.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -// WorkerAI cannot be used across accounts with AIGateWay -// https://developers.cloudflare.com/ai-gateway/providers/workersai/#openai-compatible-endpoints -// https://gateway.ai.cloudflare.com/v1/{account_id}/{gateway_id}/workers-ai -func isAIGateWay(baseURL string) bool { - return strings.HasPrefix(baseURL, "https://gateway.ai.cloudflare.com") && strings.HasSuffix(baseURL, "/workers-ai") -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - u := meta.Channel.BaseURL - isAIGateWay := isAIGateWay(u) - var urlPrefix string - if isAIGateWay { - urlPrefix = u - } else { - urlPrefix = fmt.Sprintf("%s/client/v4/accounts/%s/ai", u, meta.Channel.Key) - } - - switch meta.Mode { - case relaymode.ChatCompletions: - return urlPrefix + "/v1/chat/completions", nil - case relaymode.Embeddings: - return urlPrefix + "/v1/embeddings", nil - default: - if isAIGateWay { - return fmt.Sprintf("%s/%s", urlPrefix, meta.ActualModel), nil - } - return fmt.Sprintf("%s/run/%s", urlPrefix, meta.ActualModel), nil - } -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "cloudflare" -} diff --git a/service/aiproxy/relay/adaptor/cloudflare/constant.go b/service/aiproxy/relay/adaptor/cloudflare/constant.go deleted file mode 100644 index 5aba6dfb5..000000000 --- a/service/aiproxy/relay/adaptor/cloudflare/constant.go +++ /dev/null @@ -1,159 +0,0 @@ -package cloudflare - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "@cf/meta/llama-3.1-8b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@cf/meta/llama-2-7b-chat-fp16", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@cf/meta/llama-2-7b-chat-int8", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@cf/mistral/mistral-7b-instruct-v0.1", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "@hf/thebloke/deepseek-coder-6.7b-base-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - }, - { - Model: "@hf/thebloke/deepseek-coder-6.7b-instruct-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - }, - { - Model: "@cf/deepseek-ai/deepseek-math-7b-base", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - }, - { - Model: "@cf/deepseek-ai/deepseek-math-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - }, - { - Model: "@cf/google/gemma-2b-it-lora", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "@hf/google/gemma-7b-it", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "@cf/google/gemma-7b-it-lora", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "@hf/thebloke/llama-2-13b-chat-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@cf/meta-llama/llama-2-7b-chat-hf-lora", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@cf/meta/llama-3-8b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@hf/thebloke/llamaguard-7b-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@hf/thebloke/mistral-7b-instruct-v0.1-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "@hf/mistralai/mistral-7b-instruct-v0.2", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "@cf/mistral/mistral-7b-instruct-v0.2-lora", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "@hf/thebloke/neural-chat-7b-v3-1-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "@cf/openchat/openchat-3.5-0106", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenChat, - }, - { - Model: "@hf/thebloke/openhermes-2.5-mistral-7b-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "@cf/microsoft/phi-2", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMicrosoft, - }, - { - Model: "@cf/qwen/qwen1.5-0.5b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - }, - { - Model: "@cf/qwen/qwen1.5-1.8b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - }, - { - Model: "@cf/qwen/qwen1.5-14b-chat-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - }, - { - Model: "@cf/qwen/qwen1.5-7b-chat-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - }, - { - Model: "@cf/defog/sqlcoder-7b-2", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDefog, - }, - { - Model: "@hf/nexusflow/starling-lm-7b-beta", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerNexusFlow, - }, - { - Model: "@cf/tinyllama/tinyllama-1.1b-chat-v1.0", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "@hf/thebloke/zephyr-7b-beta-awq", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, -} diff --git a/service/aiproxy/relay/adaptor/cohere/adaptor.go b/service/aiproxy/relay/adaptor/cohere/adaptor.go deleted file mode 100644 index 5b8234f21..000000000 --- a/service/aiproxy/relay/adaptor/cohere/adaptor.go +++ /dev/null @@ -1,77 +0,0 @@ -package cohere - -import ( - "bytes" - "errors" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const baseURL = "https://api.cohere.ai" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - return meta.Channel.BaseURL + "/v1/chat", nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - request.Model = meta.ActualModel - requestBody := ConvertRequest(request) - if requestBody == nil { - return "", nil, nil, errors.New("request body is nil") - } - data, err := sonic.Marshal(requestBody) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.Rerank: - usage, err = openai.RerankHandler(meta, c, resp) - default: - if utils.IsStreamResponse(resp) { - usage, err = StreamHandler(c, resp) - } else { - usage, err = Handler(c, resp, meta.InputTokens, meta.ActualModel) - } - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "Cohere" -} diff --git a/service/aiproxy/relay/adaptor/cohere/constant.go b/service/aiproxy/relay/adaptor/cohere/constant.go deleted file mode 100644 index d39fa7f87..000000000 --- a/service/aiproxy/relay/adaptor/cohere/constant.go +++ /dev/null @@ -1,39 +0,0 @@ -package cohere - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "command", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerCohere, - }, - { - Model: "command-nightly", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerCohere, - }, - { - Model: "command-light", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerCohere, - }, - { - Model: "command-light-nightly", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerCohere, - }, - { - Model: "command-r", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerCohere, - }, - { - Model: "command-r-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerCohere, - }, -} diff --git a/service/aiproxy/relay/adaptor/cohere/main.go b/service/aiproxy/relay/adaptor/cohere/main.go deleted file mode 100644 index 79250f7f9..000000000 --- a/service/aiproxy/relay/adaptor/cohere/main.go +++ /dev/null @@ -1,218 +0,0 @@ -package cohere - -import ( - "bufio" - "fmt" - "net/http" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -var WebSearchConnector = Connector{ID: "web-search"} - -func stopReasonCohere2OpenAI(reason *string) string { - if reason == nil { - return "" - } - switch *reason { - case "COMPLETE": - return constant.StopFinishReason - default: - return *reason - } -} - -func ConvertRequest(textRequest *model.GeneralOpenAIRequest) *Request { - cohereRequest := Request{ - Model: textRequest.Model, - Message: "", - MaxTokens: textRequest.MaxTokens, - Temperature: textRequest.Temperature, - P: textRequest.TopP, - K: textRequest.TopK, - Stream: textRequest.Stream, - FrequencyPenalty: textRequest.FrequencyPenalty, - PresencePenalty: textRequest.PresencePenalty, - Seed: int(textRequest.Seed), - } - if cohereRequest.Model == "" { - cohereRequest.Model = "command-r" - } - if strings.HasSuffix(cohereRequest.Model, "-internet") { - cohereRequest.Model = strings.TrimSuffix(cohereRequest.Model, "-internet") - cohereRequest.Connectors = append(cohereRequest.Connectors, WebSearchConnector) - } - for _, message := range textRequest.Messages { - if message.Role == "user" { - cohereRequest.Message = message.Content.(string) - } else { - var role string - switch message.Role { - case "assistant": - role = "CHATBOT" - case "system": - role = "SYSTEM" - default: - role = "USER" - } - cohereRequest.ChatHistory = append(cohereRequest.ChatHistory, ChatMessage{ - Role: role, - Message: message.Content.(string), - }) - } - } - return &cohereRequest -} - -func StreamResponseCohere2OpenAI(cohereResponse *StreamResponse) (*openai.ChatCompletionsStreamResponse, *Response) { - var response *Response - var responseText string - var finishReason string - - switch cohereResponse.EventType { - case "stream-start": - return nil, nil - case "text-generation": - responseText += cohereResponse.Text - case "stream-end": - usage := cohereResponse.Response.Meta.Tokens - response = &Response{ - Meta: Meta{ - Tokens: Usage{ - InputTokens: usage.InputTokens, - OutputTokens: usage.OutputTokens, - }, - }, - } - finishReason = *cohereResponse.Response.FinishReason - default: - return nil, nil - } - - var choice openai.ChatCompletionsStreamResponseChoice - choice.Delta.Content = responseText - choice.Delta.Role = "assistant" - if finishReason != "" { - choice.FinishReason = &finishReason - } - var openaiResponse openai.ChatCompletionsStreamResponse - openaiResponse.Object = "chat.completion.chunk" - openaiResponse.Choices = []*openai.ChatCompletionsStreamResponseChoice{&choice} - return &openaiResponse, response -} - -func ResponseCohere2OpenAI(cohereResponse *Response) *openai.TextResponse { - choice := openai.TextResponseChoice{ - Index: 0, - Message: model.Message{ - Role: "assistant", - Content: cohereResponse.Text, - Name: nil, - }, - FinishReason: stopReasonCohere2OpenAI(cohereResponse.FinishReason), - } - fullTextResponse := openai.TextResponse{ - ID: "chatcmpl-" + cohereResponse.ResponseID, - Model: "model", - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []*openai.TextResponseChoice{&choice}, - } - return &fullTextResponse -} - -func StreamHandler(c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - createdTime := time.Now().Unix() - scanner := bufio.NewScanner(resp.Body) - scanner.Split(bufio.ScanLines) - - common.SetEventStreamHeaders(c) - var usage model.Usage - - for scanner.Scan() { - data := scanner.Text() - data = strings.TrimSuffix(data, "\r") - - var cohereResponse StreamResponse - err := sonic.Unmarshal(conv.StringToBytes(data), &cohereResponse) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - - response, meta := StreamResponseCohere2OpenAI(&cohereResponse) - if meta != nil { - usage.PromptTokens += meta.Meta.Tokens.InputTokens - usage.CompletionTokens += meta.Meta.Tokens.OutputTokens - continue - } - if response == nil { - continue - } - - response.ID = fmt.Sprintf("chatcmpl-%d", createdTime) - response.Model = c.GetString("original_model") - response.Created = createdTime - - _ = render.ObjectData(c, response) - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - render.Done(c) - - return &usage, nil -} - -func Handler(c *gin.Context, resp *http.Response, _ int, modelName string) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - var cohereResponse Response - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&cohereResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if cohereResponse.ResponseID == "" { - return nil, openai.ErrorWrapperWithMessage(cohereResponse.Message, resp.StatusCode, resp.StatusCode) - } - fullTextResponse := ResponseCohere2OpenAI(&cohereResponse) - fullTextResponse.Model = modelName - usage := model.Usage{ - PromptTokens: cohereResponse.Meta.Tokens.InputTokens, - CompletionTokens: cohereResponse.Meta.Tokens.OutputTokens, - TotalTokens: cohereResponse.Meta.Tokens.InputTokens + cohereResponse.Meta.Tokens.OutputTokens, - } - fullTextResponse.Usage = usage - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &usage, nil -} diff --git a/service/aiproxy/relay/adaptor/cohere/model.go b/service/aiproxy/relay/adaptor/cohere/model.go deleted file mode 100644 index 64e1ccc8f..000000000 --- a/service/aiproxy/relay/adaptor/cohere/model.go +++ /dev/null @@ -1,147 +0,0 @@ -package cohere - -type Request struct { - P *float64 `json:"p,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - PresencePenalty *float64 `json:"presence_penalty,omitempty"` - FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` - Model string `json:"model,omitempty"` - Message string `json:"message" required:"true"` - Preamble string `json:"preamble,omitempty"` - PromptTruncation string `json:"prompt_truncation,omitempty"` - ConversationID string `json:"conversation_id,omitempty"` - StopSequences []string `json:"stop_sequences,omitempty"` - Tools []Tool `json:"tools,omitempty"` - ToolResults []ToolResult `json:"tool_results,omitempty"` - Documents []Document `json:"documents,omitempty"` - Connectors []Connector `json:"connectors,omitempty"` - ChatHistory []ChatMessage `json:"chat_history,omitempty"` - K int `json:"k,omitempty"` - MaxInputTokens int `json:"max_input_tokens,omitempty"` - Seed int `json:"seed,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - Stream bool `json:"stream,omitempty"` -} - -type ChatMessage struct { - Role string `json:"role" required:"true"` - Message string `json:"message" required:"true"` -} - -type Tool struct { - ParameterDefinitions map[string]ParameterSpec `json:"parameter_definitions"` - Name string `json:"name" required:"true"` - Description string `json:"description" required:"true"` -} - -type ParameterSpec struct { - Description string `json:"description"` - Type string `json:"type" required:"true"` - Required bool `json:"required"` -} - -type ToolResult struct { - Call ToolCall `json:"call"` - Outputs []map[string]interface{} `json:"outputs"` -} - -type ToolCall struct { - Parameters map[string]interface{} `json:"parameters" required:"true"` - Name string `json:"name" required:"true"` -} - -type StreamResponse struct { - Response *Response `json:"response,omitempty"` - EventType string `json:"event_type"` - GenerationID string `json:"generation_id,omitempty"` - Text string `json:"text,omitempty"` - FinishReason string `json:"finish_reason,omitempty"` - SearchQueries []*SearchQuery `json:"search_queries,omitempty"` - SearchResults []*SearchResult `json:"search_results,omitempty"` - Documents []*Document `json:"documents,omitempty"` - Citations []*Citation `json:"citations,omitempty"` - IsFinished bool `json:"is_finished"` -} - -type SearchQuery struct { - Text string `json:"text"` - GenerationID string `json:"generation_id"` -} - -type SearchResult struct { - SearchQuery *SearchQuery `json:"search_query"` - Connector *Connector `json:"connector"` - DocumentIDs []string `json:"document_ids"` -} - -type Connector struct { - ID string `json:"id"` -} - -type Document struct { - ID string `json:"id"` - Snippet string `json:"snippet"` - Timestamp string `json:"timestamp"` - Title string `json:"title"` - URL string `json:"url"` -} - -type Citation struct { - Text string `json:"text"` - DocumentIDs []string `json:"document_ids"` - Start int `json:"start"` - End int `json:"end"` -} - -type Response struct { - FinishReason *string `json:"finish_reason"` - ResponseID string `json:"response_id"` - Text string `json:"text"` - GenerationID string `json:"generation_id"` - Message string `json:"message"` - ChatHistory []*Message `json:"chat_history"` - Citations []*Citation `json:"citations"` - Documents []*Document `json:"documents"` - SearchResults []*SearchResult `json:"search_results"` - SearchQueries []*SearchQuery `json:"search_queries"` - Meta Meta `json:"meta"` -} - -type Message struct { - Role string `json:"role"` - Message string `json:"message"` -} - -type Version struct { - Version string `json:"version"` -} - -type Units struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -type ChatEntry struct { - Role string `json:"role"` - Message string `json:"message"` -} - -type Meta struct { - APIVersion APIVersion `json:"api_version"` - BilledUnits BilledUnits `json:"billed_units"` - Tokens Usage `json:"tokens"` -} - -type APIVersion struct { - Version string `json:"version"` -} - -type BilledUnits struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -type Usage struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} diff --git a/service/aiproxy/relay/adaptor/coze/adaptor.go b/service/aiproxy/relay/adaptor/coze/adaptor.go deleted file mode 100644 index b4854f37d..000000000 --- a/service/aiproxy/relay/adaptor/coze/adaptor.go +++ /dev/null @@ -1,96 +0,0 @@ -package coze - -import ( - "bytes" - "errors" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const baseURL = "https://api.coze.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - return meta.Channel.BaseURL + "/open_api/v2/chat", nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - token, _, err := getTokenAndUserID(meta.Channel.Key) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+token) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - if meta.Mode != relaymode.ChatCompletions { - return "", nil, nil, errors.New("coze only support chat completions") - } - request, err := utils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - _, userID, err := getTokenAndUserID(meta.Channel.Key) - if err != nil { - return "", nil, nil, err - } - request.User = userID - request.Model = meta.ActualModel - cozeRequest := Request{ - Stream: request.Stream, - User: request.User, - BotID: strings.TrimPrefix(meta.ActualModel, "bot-"), - } - for i, message := range request.Messages { - if i == len(request.Messages)-1 { - cozeRequest.Query = message.StringContent() - continue - } - cozeMessage := Message{ - Role: message.Role, - Content: message.StringContent(), - } - cozeRequest.ChatHistory = append(cozeRequest.ChatHistory, cozeMessage) - } - data, err := sonic.Marshal(cozeRequest) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - if utils.IsStreamResponse(resp) { - usage, err = StreamHandler(meta, c, resp) - } else { - usage, err = Handler(meta, c, resp) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "coze" -} diff --git a/service/aiproxy/relay/adaptor/coze/constant/contenttype/define.go b/service/aiproxy/relay/adaptor/coze/constant/contenttype/define.go deleted file mode 100644 index 69c876bc4..000000000 --- a/service/aiproxy/relay/adaptor/coze/constant/contenttype/define.go +++ /dev/null @@ -1,5 +0,0 @@ -package contenttype - -const ( - Text = "text" -) diff --git a/service/aiproxy/relay/adaptor/coze/constant/event/define.go b/service/aiproxy/relay/adaptor/coze/constant/event/define.go deleted file mode 100644 index c03e8c173..000000000 --- a/service/aiproxy/relay/adaptor/coze/constant/event/define.go +++ /dev/null @@ -1,7 +0,0 @@ -package event - -const ( - Message = "message" - Done = "done" - Error = "error" -) diff --git a/service/aiproxy/relay/adaptor/coze/constant/messagetype/define.go b/service/aiproxy/relay/adaptor/coze/constant/messagetype/define.go deleted file mode 100644 index 6c1c25db4..000000000 --- a/service/aiproxy/relay/adaptor/coze/constant/messagetype/define.go +++ /dev/null @@ -1,6 +0,0 @@ -package messagetype - -const ( - Answer = "answer" - FollowUp = "follow_up" -) diff --git a/service/aiproxy/relay/adaptor/coze/constants.go b/service/aiproxy/relay/adaptor/coze/constants.go deleted file mode 100644 index 5a0ea7ebc..000000000 --- a/service/aiproxy/relay/adaptor/coze/constants.go +++ /dev/null @@ -1,5 +0,0 @@ -package coze - -import "github.com/labring/sealos/service/aiproxy/model" - -var ModelList = []*model.ModelConfig{} diff --git a/service/aiproxy/relay/adaptor/coze/key.go b/service/aiproxy/relay/adaptor/coze/key.go deleted file mode 100644 index 5440e4f0a..000000000 --- a/service/aiproxy/relay/adaptor/coze/key.go +++ /dev/null @@ -1,30 +0,0 @@ -package coze - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, _, err := getTokenAndUserID(key) - if err != nil { - return err - } - return nil -} - -func (a *Adaptor) KeyHelp() string { - return "token|user_id" -} - -func getTokenAndUserID(key string) (string, string, error) { - split := strings.Split(key, "|") - if len(split) != 2 { - return "", "", errors.New("invalid key format") - } - return split[0], split[1], nil -} diff --git a/service/aiproxy/relay/adaptor/coze/main.go b/service/aiproxy/relay/adaptor/coze/main.go deleted file mode 100644 index c356d179e..000000000 --- a/service/aiproxy/relay/adaptor/coze/main.go +++ /dev/null @@ -1,180 +0,0 @@ -package coze - -import ( - "bufio" - "net/http" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/coze/constant/messagetype" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -// https://www.coze.com/open - -func stopReasonCoze2OpenAI(reason *string) string { - if reason == nil { - return "" - } - switch *reason { - case "end_turn": - return "stop" - case "stop_sequence": - return "stop" - case "max_tokens": - return "length" - default: - return *reason - } -} - -func StreamResponseCoze2OpenAI(cozeResponse *StreamResponse) (*openai.ChatCompletionsStreamResponse, *Response) { - var response *Response - var stopReason string - var choice openai.ChatCompletionsStreamResponseChoice - - if cozeResponse.Message != nil { - if cozeResponse.Message.Type != messagetype.Answer { - return nil, nil - } - choice.Delta.Content = cozeResponse.Message.Content - } - choice.Delta.Role = "assistant" - finishReason := stopReasonCoze2OpenAI(&stopReason) - if finishReason != "null" { - choice.FinishReason = &finishReason - } - var openaiResponse openai.ChatCompletionsStreamResponse - openaiResponse.Object = "chat.completion.chunk" - openaiResponse.Choices = []*openai.ChatCompletionsStreamResponseChoice{&choice} - openaiResponse.ID = cozeResponse.ConversationID - return &openaiResponse, response -} - -func ResponseCoze2OpenAI(cozeResponse *Response) *openai.TextResponse { - var responseText string - for _, message := range cozeResponse.Messages { - if message.Type == messagetype.Answer { - responseText = message.Content - break - } - } - choice := openai.TextResponseChoice{ - Index: 0, - Message: model.Message{ - Role: "assistant", - Content: responseText, - Name: nil, - }, - FinishReason: constant.StopFinishReason, - } - fullTextResponse := openai.TextResponse{ - ID: "chatcmpl-" + cozeResponse.ConversationID, - Model: "coze-bot", - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []*openai.TextResponseChoice{&choice}, - } - return &fullTextResponse -} - -func StreamHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - var responseText string - createdTime := time.Now().Unix() - scanner := bufio.NewScanner(resp.Body) - scanner.Split(bufio.ScanLines) - - common.SetEventStreamHeaders(c) - - for scanner.Scan() { - data := scanner.Bytes() - if len(data) < 6 || conv.BytesToString(data[:6]) != "data: " { - continue - } - data = data[6:] - - if conv.BytesToString(data) == "[DONE]" { - break - } - - var cozeResponse StreamResponse - err := sonic.Unmarshal(data, &cozeResponse) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - - response, _ := StreamResponseCoze2OpenAI(&cozeResponse) - if response == nil { - continue - } - - for _, choice := range response.Choices { - responseText += conv.AsString(choice.Delta.Content) - } - response.Model = meta.OriginModel - response.Created = createdTime - - _ = render.ObjectData(c, response) - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - render.Done(c) - - return openai.ResponseText2Usage(responseText, meta.ActualModel, meta.InputTokens), nil -} - -func Handler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - var cozeResponse Response - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&cozeResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if cozeResponse.Code != 0 { - return nil, openai.ErrorWrapperWithMessage(cozeResponse.Msg, cozeResponse.Code, resp.StatusCode) - } - fullTextResponse := ResponseCoze2OpenAI(&cozeResponse) - fullTextResponse.Model = meta.OriginModel - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, err = c.Writer.Write(jsonResponse) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - var responseText string - if len(fullTextResponse.Choices) > 0 { - responseText = fullTextResponse.Choices[0].Message.StringContent() - } - return openai.ResponseText2Usage(responseText, meta.ActualModel, meta.InputTokens), nil -} diff --git a/service/aiproxy/relay/adaptor/coze/model.go b/service/aiproxy/relay/adaptor/coze/model.go deleted file mode 100644 index a43adf890..000000000 --- a/service/aiproxy/relay/adaptor/coze/model.go +++ /dev/null @@ -1,38 +0,0 @@ -package coze - -type Message struct { - Role string `json:"role"` - Type string `json:"type"` - Content string `json:"content"` - ContentType string `json:"content_type"` -} - -type ErrorInformation struct { - Msg string `json:"msg"` - Code int `json:"code"` -} - -type Request struct { - ConversationID string `json:"conversation_id,omitempty"` - BotID string `json:"bot_id"` - User string `json:"user"` - Query string `json:"query"` - ChatHistory []Message `json:"chat_history,omitempty"` - Stream bool `json:"stream"` -} - -type Response struct { - ConversationID string `json:"conversation_id,omitempty"` - Msg string `json:"msg,omitempty"` - Messages []Message `json:"messages,omitempty"` - Code int `json:"code,omitempty"` -} - -type StreamResponse struct { - Message *Message `json:"message,omitempty"` - ErrorInformation *ErrorInformation `json:"error_information,omitempty"` - Event string `json:"event,omitempty"` - ConversationID string `json:"conversation_id,omitempty"` - Index int `json:"index,omitempty"` - IsFinish bool `json:"is_finish,omitempty"` -} diff --git a/service/aiproxy/relay/adaptor/deepseek/adaptor.go b/service/aiproxy/relay/adaptor/deepseek/adaptor.go deleted file mode 100644 index 5ed5cd32b..000000000 --- a/service/aiproxy/relay/adaptor/deepseek/adaptor.go +++ /dev/null @@ -1,27 +0,0 @@ -package deepseek - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -var _ adaptor.Adaptor = (*Adaptor)(nil) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.deepseek.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "deepseek" -} diff --git a/service/aiproxy/relay/adaptor/deepseek/balance.go b/service/aiproxy/relay/adaptor/deepseek/balance.go deleted file mode 100644 index 9c691933d..000000000 --- a/service/aiproxy/relay/adaptor/deepseek/balance.go +++ /dev/null @@ -1,61 +0,0 @@ -package deepseek - -import ( - "context" - "errors" - "net/http" - "strconv" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.Balancer = (*Adaptor)(nil) - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - u := channel.BaseURL - if u == "" { - u = baseURL - } - url := u + "/user/balance" - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", "Bearer "+channel.Key) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - var usage UsageResponse - if err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&usage); err != nil { - return 0, err - } - index := -1 - for i, balanceInfo := range usage.BalanceInfos { - if balanceInfo.Currency == "CNY" { - index = i - break - } - } - if index == -1 { - return 0, errors.New("currency CNY not found") - } - balance, err := strconv.ParseFloat(usage.BalanceInfos[index].TotalBalance, 64) - if err != nil { - return 0, err - } - return balance, nil -} - -type UsageResponse struct { - BalanceInfos []struct { - Currency string `json:"currency"` - TotalBalance string `json:"total_balance"` - GrantedBalance string `json:"granted_balance"` - ToppedUpBalance string `json:"topped_up_balance"` - } `json:"balance_infos"` - IsAvailable bool `json:"is_available"` -} diff --git a/service/aiproxy/relay/adaptor/deepseek/constants.go b/service/aiproxy/relay/adaptor/deepseek/constants.go deleted file mode 100644 index 33b7f30a7..000000000 --- a/service/aiproxy/relay/adaptor/deepseek/constants.go +++ /dev/null @@ -1,35 +0,0 @@ -package deepseek - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "deepseek-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - InputPrice: 0.001, - OutputPrice: 0.002, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(64000), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "deepseek-reasoner", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDeepSeek, - InputPrice: 0.004, - OutputPrice: 0.016, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(64000), - model.WithModelConfigMaxOutputTokens(8192), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/doc2x/adaptor.go b/service/aiproxy/relay/adaptor/doc2x/adaptor.go deleted file mode 100644 index b0e9712ef..000000000 --- a/service/aiproxy/relay/adaptor/doc2x/adaptor.go +++ /dev/null @@ -1,70 +0,0 @@ -package doc2x - -import ( - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -var _ adaptor.Adaptor = (*Adaptor)(nil) - -type Adaptor struct{} - -const baseURL = "https://v2.doc2x.noedgeai.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - switch meta.Mode { - case relaymode.ParsePdf: - return meta.Channel.BaseURL + "/api/v2/parse/pdf", nil - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.ParsePdf: - return ConvertParsePdfRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.ParsePdf: - return HandleParsePdfResponse(meta, c, resp) - default: - return nil, openai.ErrorWrapperWithMessage(fmt.Sprintf("unsupported mode: %s", meta.Mode), "unsupported_mode", http.StatusBadRequest) - } -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, c *gin.Context, req *http.Request) error { - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - return nil -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "doc2x" -} diff --git a/service/aiproxy/relay/adaptor/doc2x/constants.go b/service/aiproxy/relay/adaptor/doc2x/constants.go deleted file mode 100644 index 4a075d080..000000000 --- a/service/aiproxy/relay/adaptor/doc2x/constants.go +++ /dev/null @@ -1,16 +0,0 @@ -package doc2x - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "pdf", - Type: relaymode.ParsePdf, - Owner: model.ModelOwnerDoc2x, - InputPrice: 20, - RPM: 10, - }, -} diff --git a/service/aiproxy/relay/adaptor/doc2x/html2md_test.go b/service/aiproxy/relay/adaptor/doc2x/html2md_test.go deleted file mode 100644 index 8f87d50a2..000000000 --- a/service/aiproxy/relay/adaptor/doc2x/html2md_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package doc2x_test - -import ( - "context" - "testing" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor/doc2x" -) - -func TestHTMLTable2Md(t *testing.T) { - tables := []struct { - name string - html string - expected string - }{ - { - name: "basic table", - html: `
sadsa
sadasdsasad
dsadsadsa
`, - expected: `| sadsa | | | -| --- | --- | --- | -| | sadasdsa | sad | -| | | dsadsadsa | -| | | |`, - }, - { - name: "simple table", - html: `
Header 1Header 2
Data 1Data 2
`, - expected: `| Header 1 | Header 2 | -| --- | --- | -| Data 1 | Data 2 |`, - }, - { - name: "empty table", - html: `
`, - expected: `| | | -| --- | --- | -| | |`, - }, - } - - for _, tc := range tables { - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - result := doc2x.HTMLTable2Md(tc.html) - - if result != tc.expected { - t.Errorf("Expected:\n%s\nGot:\n%s", tc.expected, result) - } - }) - } -} - -var htmlImage = `` - -func TestInlineMdImage(t *testing.T) { - result := doc2x.InlineMdImage(context.Background(), htmlImage) - t.Log(result) -} diff --git a/service/aiproxy/relay/adaptor/doc2x/pdf.go b/service/aiproxy/relay/adaptor/doc2x/pdf.go deleted file mode 100644 index 060a08f37..000000000 --- a/service/aiproxy/relay/adaptor/doc2x/pdf.go +++ /dev/null @@ -1,419 +0,0 @@ -package doc2x - -import ( - "context" - "encoding/base64" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strconv" - "strings" - "sync" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - log "github.com/sirupsen/logrus" -) - -func ConvertParsePdfRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - err := req.ParseMultipartForm(1024 * 1024 * 4) - if err != nil { - return "", nil, nil, err - } - - file, _, err := req.FormFile("file") - if err != nil { - return "", nil, nil, err - } - - responseFormat := req.FormValue("response_format") - meta.Set("response_format", responseFormat) - - return http.MethodPost, nil, file, nil -} - -type ParsePdfResponse struct { - Code string `json:"code"` - Data ParsePdfResponseData `json:"data"` - Msg string `json:"msg"` -} - -type ParsePdfResponseData struct { - UID string `json:"uid"` -} - -func HandleParsePdfResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - var response ParsePdfResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&response) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("decode response failed: "+err.Error(), "decode_response_failed", http.StatusBadRequest) - } - - if response.Code != "success" { - return nil, openai.ErrorWrapperWithMessage("parse pdf failed: "+response.Msg, "parse_pdf_failed", http.StatusBadRequest) - } - - for { - status, err := GetStatus(context.Background(), meta, response.Data.UID) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("get status failed: "+err.Error(), "get_status_failed", http.StatusInternalServerError) - } - - switch status.Status { - case StatusResponseDataStatusSuccess: - return handleParsePdfResponse(meta, c, status.Result) - case StatusResponseDataStatusProcessing: - time.Sleep(1 * time.Second) - case StatusResponseDataStatusFailed: - return nil, openai.ErrorWrapperWithMessage("parse pdf failed: "+status.Detail, "parse_pdf_failed", http.StatusBadRequest) - } - } -} - -// Start of Selection -var ( - tableRegex = regexp.MustCompile(`[\s\S]*?
`) - rowRegex = regexp.MustCompile(`(.*?)`) - cellRegex = regexp.MustCompile(`]*/>|]*>(.*?)`) - whitespaceRegex = regexp.MustCompile(`\n\s*`) - tdCleanRegex = regexp.MustCompile(`|`) - colspanRegex = regexp.MustCompile(`colspan="(\d+)"`) - rowspanRegex = regexp.MustCompile(`rowspan="(\d+)"`) - - htmlImageRegex = regexp.MustCompile(`]*)?(?:\s*\/>|>)`) - imageRegex = regexp.MustCompile(`!\[(.*?)\]\((http[^)]+)\)`) - - mediaCommentRegex = regexp.MustCompile(``) - footnoteCommentRegex = regexp.MustCompile(``) -) - -func HTMLTable2Md(content string) string { - return tableRegex.ReplaceAllStringFunc(content, func(htmlTable string) string { - cleanHTML := whitespaceRegex.ReplaceAllString(htmlTable, "") - rows := rowRegex.FindAllString(cleanHTML, -1) - if len(rows) == 0 { - return htmlTable - } - var tableData [][]string - maxColumns := 0 - for rowIndex, row := range rows { - for len(tableData) <= rowIndex { - tableData = append(tableData, []string{}) - } - colIndex := 0 - cells := cellRegex.FindAllString(row, -1) - if len(cells) > maxColumns { - maxColumns = len(cells) - } - for _, cell := range cells { - colspan := 1 - if matches := colspanRegex.FindStringSubmatch(cell); len(matches) > 1 { - colspan, _ = strconv.Atoi(matches[1]) - } - rowspan := 1 - if matches := rowspanRegex.FindStringSubmatch(cell); len(matches) > 1 { - rowspan, _ = strconv.Atoi(matches[1]) - } - content := strings.TrimSpace(tdCleanRegex.ReplaceAllString(cell, "")) - for i := 0; i < rowspan; i++ { - for j := 0; j < colspan; j++ { - for len(tableData) <= rowIndex+i { - tableData = append(tableData, []string{}) - } - for len(tableData[rowIndex+i]) <= colIndex+j { - tableData[rowIndex+i] = append(tableData[rowIndex+i], "") - } - if i == 0 && j == 0 { - tableData[rowIndex+i][colIndex+j] = content - } else { - tableData[rowIndex+i][colIndex+j] = "^^" - } - } - } - colIndex += colspan - } - } - for i := range tableData { - for len(tableData[i]) < maxColumns { - tableData[i] = append(tableData[i], " ") - } - } - var chunks []string - headerCells := make([]string, maxColumns) - for i := 0; i < maxColumns; i++ { - if i < len(tableData[0]) { - headerCells[i] = tableData[0][i] - } else { - headerCells[i] = " " - } - } - chunks = append(chunks, fmt.Sprintf("| %s |", strings.Join(headerCells, " | "))) - separatorCells := make([]string, maxColumns) - for i := 0; i < maxColumns; i++ { - separatorCells[i] = "---" - } - chunks = append(chunks, fmt.Sprintf("| %s |", strings.Join(separatorCells, " | "))) - for _, row := range tableData[1:] { - chunks = append(chunks, fmt.Sprintf("| %s |", strings.Join(row, " | "))) - } - return strings.Join(chunks, "\n") - }) -} - -func HTMLImage2Md(content string) string { - return htmlImageRegex.ReplaceAllString(content, "![img]($1)") -} - -func InlineMdImage(ctx context.Context, text string) string { - text = HTMLImage2Md(text) - - matches := imageRegex.FindAllStringSubmatchIndex(text, -1) - if len(matches) == 0 { - return text - } - - var resultText strings.Builder - var wg sync.WaitGroup - var mutex sync.Mutex - - type imageInfo struct { - startPos int - endPos int - altText string - url string - replacement string - } - - imageInfos := make([]imageInfo, len(matches)) - - for i, match := range matches { - altTextStart, altTextEnd := match[2], match[3] - urlStart, urlEnd := match[4], match[5] - - imageInfos[i] = imageInfo{ - startPos: match[0], - endPos: match[1], - altText: text[altTextStart:altTextEnd], - url: text[urlStart:urlEnd], - } - } - - for i := range imageInfos { - wg.Add(1) - go func(index int) { - defer wg.Done() - info := &imageInfos[index] - - replacement, err := imageURL2MdBase64(ctx, info.url, info.altText) - if err != nil { - log.Printf("failed to process image %s: %v", info.url, err) - // when the image is not found, keep the original link - mutex.Lock() - info.replacement = text[info.startPos:info.endPos] - mutex.Unlock() - return - } - - mutex.Lock() - info.replacement = replacement - mutex.Unlock() - }(i) - } - - wg.Wait() - - lastPos := 0 - for _, info := range imageInfos { - resultText.WriteString(text[lastPos:info.startPos]) - resultText.WriteString(info.replacement) - lastPos = info.endPos - } - resultText.WriteString(text[lastPos:]) - - return resultText.String() -} - -func imageURL2MdBase64(ctx context.Context, url string, altText string) (string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return "", fmt.Errorf("failed to create request: %w", err) - } - - var resp *http.Response - var downloadErr error - retries := 0 - maxRetries := 3 - - for retries <= maxRetries { - resp, downloadErr = http.DefaultClient.Do(req) - if downloadErr != nil { - return "", fmt.Errorf("failed to download image: %w", downloadErr) - } - if resp.StatusCode == http.StatusNotFound { - resp.Body.Close() - if retries == maxRetries { - return "", fmt.Errorf("failed to download image, status code: %d after %d retries", resp.StatusCode, retries) - } - retries++ - time.Sleep(1 * time.Second) - continue - } - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return "", fmt.Errorf("failed to download image, status code: %d", resp.StatusCode) - } - break - } - defer resp.Body.Close() - - data, err := io.ReadAll(resp.Body) - if err != nil { - return "", fmt.Errorf("failed to read image data: %w", err) - } - mime := resp.Header.Get("Content-Type") - if mime == "" { - mime = inferMimeType(url) - } - base64Data := base64.StdEncoding.EncodeToString(data) - return fmt.Sprintf("![%s](data:%s;base64,%s)", altText, mime, base64Data), nil -} - -func inferMimeType(u string) string { - p, err := url.Parse(u) - if err != nil { - return "image/jpeg" - } - - lowerURL := strings.ToLower(p.Path) - switch { - case strings.HasSuffix(lowerURL, ".png"): - return "image/png" - case strings.HasSuffix(lowerURL, ".gif"): - return "image/gif" - case strings.HasSuffix(lowerURL, ".webp"): - return "image/webp" - case strings.HasSuffix(lowerURL, ".svg"): - return "image/svg+xml" - default: - return "image/jpeg" - } -} - -func handleConvertPdfToMd(ctx context.Context, str string) (string, error) { - result := InlineMdImage(ctx, str) - result = HTMLTable2Md(result) - - result = mediaCommentRegex.ReplaceAllString(result, "") - result = footnoteCommentRegex.ReplaceAllString(result, "") - - return result, nil -} - -func handleParsePdfResponse(meta *meta.Meta, c *gin.Context, response *StatusResponseDataResult) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - mds := make([]string, 0, len(response.Pages)) - totalLength := 0 - for _, page := range response.Pages { - mds = append(mds, page.MD) - totalLength += len(page.MD) - } - pages := len(response.Pages) - - switch meta.GetString("response_format") { - case "list": - for i, md := range mds { - result, err := handleConvertPdfToMd(c.Request.Context(), md) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("convert pdf to md failed: "+err.Error(), "convert_pdf_to_md_failed", http.StatusInternalServerError) - } - mds[i] = result - } - c.JSON(http.StatusOK, relaymodel.ParsePdfListResponse{ - Markdowns: mds, - }) - default: - builder := strings.Builder{} - builder.Grow(totalLength) - for _, md := range mds { - builder.WriteString(md) - } - result, err := handleConvertPdfToMd(c.Request.Context(), builder.String()) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("convert pdf to md failed: "+err.Error(), "convert_pdf_to_md_failed", http.StatusInternalServerError) - } - c.JSON(http.StatusOK, relaymodel.ParsePdfResponse{ - Pages: pages, - Markdown: result, - }) - } - - return &relaymodel.Usage{ - PromptTokens: pages, - TotalTokens: pages, - }, nil -} - -type StatusResponse struct { - Code string `json:"code"` - Msg string `json:"msg"` - Data *StatusResponseData `json:"data"` -} - -const ( - StatusResponseDataStatusSuccess = "success" - StatusResponseDataStatusProcessing = "processing" - StatusResponseDataStatusFailed = "failed" -) - -type StatusResponseData struct { - Progress int `json:"progress"` - Status string `json:"status"` - Detail string `json:"detail"` - Result *StatusResponseDataResult `json:"result"` -} - -type StatusResponseDataResult struct { - Version string `json:"version"` - Pages []StatusResponseDataResultPage `json:"pages"` -} - -type StatusResponseDataResultPage struct { - URL string `json:"url"` - PageIdx int `json:"page_idx"` - PageWidth int `json:"page_width"` - PageHeight int `json:"page_height"` - MD string `json:"md"` -} - -func GetStatus(ctx context.Context, meta *meta.Meta, uid string) (*StatusResponseData, error) { - url := fmt.Sprintf("%s/api/v2/parse/status?uid=%s", meta.Channel.BaseURL, uid) - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - var response StatusResponse - err = sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&response) - if err != nil { - return nil, err - } - - if response.Code != "success" { - return nil, errors.New("get status failed: " + response.Msg) - } - - return response.Data, nil -} diff --git a/service/aiproxy/relay/adaptor/doubao/constants.go b/service/aiproxy/relay/adaptor/doubao/constants.go deleted file mode 100644 index 75f56506c..000000000 --- a/service/aiproxy/relay/adaptor/doubao/constants.go +++ /dev/null @@ -1,198 +0,0 @@ -package doubao - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://console.volcengine.com/ark/region:ark+cn-beijing/model - -var ModelList = []*model.ModelConfig{ - { - Model: "Doubao-1.5-vision-pro-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.003, - OutputPrice: 0.009, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(32768), - model.WithModelConfigVision(true), - ), - }, - { - Model: "Doubao-1.5-pro-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0008, - OutputPrice: 0.0020, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(32768), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-1.5-pro-256k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.005, - OutputPrice: 0.009, - RPM: 2000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(256000), - model.WithModelConfigMaxOutputTokens(12000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-1.5-lite-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "Doubao-vision-lite-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.008, - OutputPrice: 0.008, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(32768), - model.WithModelConfigVision(true), - ), - }, - { - Model: "Doubao-vision-pro-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.02, - OutputPrice: 0.02, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(32768), - model.WithModelConfigVision(true), - ), - }, - { - Model: "Doubao-pro-256k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0050, - OutputPrice: 0.0090, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(256000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-pro-128k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0050, - OutputPrice: 0.0090, - RPM: 1000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(128000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-pro-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0008, - OutputPrice: 0.0020, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-pro-4k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0008, - OutputPrice: 0.0020, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-lite-128k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0008, - OutputPrice: 0.0010, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(128000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-lite-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 15000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "Doubao-lite-4k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0003, - OutputPrice: 0.0006, - RPM: 10000, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "Doubao-embedding", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0005, - RPM: 1200, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(4096), - ), - }, - { - Model: "Doubao-embedding-large", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.0007, - RPM: 1000, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(4096), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/doubao/main.go b/service/aiproxy/relay/adaptor/doubao/main.go deleted file mode 100644 index b1e8a3a50..000000000 --- a/service/aiproxy/relay/adaptor/doubao/main.go +++ /dev/null @@ -1,91 +0,0 @@ -package doubao - -import ( - "bytes" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -func GetRequestURL(meta *meta.Meta) (string, error) { - u := meta.Channel.BaseURL - switch meta.Mode { - case relaymode.ChatCompletions: - if strings.HasPrefix(meta.ActualModel, "bot-") { - return u + "/api/v3/bots/chat/completions", nil - } - return u + "/api/v3/chat/completions", nil - case relaymode.Embeddings: - return u + "/api/v3/embeddings", nil - default: - return "", fmt.Errorf("unsupported relay mode %d for doubao", meta.Mode) - } -} - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://ark.cn-beijing.volces.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - return GetRequestURL(meta) -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - method, header, body, err := a.Adaptor.ConvertRequest(meta, req) - if err != nil { - return "", nil, nil, err - } - if meta.Mode != relaymode.ChatCompletions || meta.OriginModel != "deepseek-reasoner" { - return method, header, body, nil - } - - m := make(map[string]any) - err = sonic.ConfigDefault.NewDecoder(body).Decode(&m) - if err != nil { - return "", nil, nil, err - } - messages, _ := m["messages"].([]any) - if len(messages) == 0 { - return "", nil, nil, errors.New("messages is empty") - } - sysMessage := relaymodel.Message{ - Role: "system", - Content: "回答前，都先用 输出你的思考过程。", - } - messages = append([]any{sysMessage}, messages...) - m["messages"] = messages - newBody, err := sonic.Marshal(m) - if err != nil { - return "", nil, nil, err - } - - return method, header, bytes.NewReader(newBody), nil -} - -func (a *Adaptor) GetChannelName() string { - return "doubao" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/doubaoaudio/constants.go b/service/aiproxy/relay/adaptor/doubaoaudio/constants.go deleted file mode 100644 index f56d43db2..000000000 --- a/service/aiproxy/relay/adaptor/doubaoaudio/constants.go +++ /dev/null @@ -1,106 +0,0 @@ -package doubaoaudio - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://www.volcengine.com/docs/6561/1257543 - -var ModelList = []*model.ModelConfig{ - { - Model: "Doubao-tts", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerDoubao, - InputPrice: 0.5, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigSupportFormats([]string{ - "pcm", - "mp3", - "wav", - "ogg_opus", - }), - model.WithModelConfigSupportVoices([]string{ - "zh_female_cancan_mars_bigtts", - "zh_female_qingxinnvsheng_mars_bigtts", - "zh_female_shuangkuaisisi_moon_bigtts", - "zh_male_wennuanahu_moon_bigtts", - "zh_male_shaonianzixin_moon_bigtts", - "zh_female_zhixingnvsheng_mars_bigtts", - "zh_male_qingshuangnanda_mars_bigtts", - "zh_female_linjianvhai_moon_bigtts", - "zh_male_yuanboxiaoshu_moon_bigtts", - "zh_male_yangguangqingnian_moon_bigtts", - "zh_female_tianmeixiaoyuan_moon_bigtts", - "zh_female_qingchezizi_moon_bigtts", - "zh_male_jieshuoxiaoming_moon_bigtts", - "zh_female_kailangjiejie_moon_bigtts", - "zh_male_linjiananhai_moon_bigtts", - "zh_female_tianmeiyueyue_moon_bigtts", - "zh_female_xinlingjitang_moon_bigtts", - "en_male_smith_mars_bigtts", - "en_female_anna_mars_bigtts", - "en_male_adam_mars_bigtts", - "en_female_sarah_mars_bigtts", - "en_male_dryw_mars_bigtts", - "multi_male_jingqiangkanye_moon_bigtts", - "multi_female_shuangkuaisisi_moon_bigtts", - "multi_male_wanqudashu_moon_bigtts", - "multi_female_gaolengyujie_moon_bigtts", - "zh_male_jingqiangkanye_moon_bigtts", - "zh_female_wanwanxiaohe_moon_bigtts", - "zh_female_wanqudashu_moon_bigtts", - "zh_female_daimengchuanmei_moon_bigtts", - "zh_male_guozhoudege_moon_bigtts", - "zh_male_beijingxiaoye_moon_bigtts", - "zh_male_haoyuxiaoge_moon_bigtts", - "zh_male_guangxiyuanzhou_moon_bigtts", - "zh_female_meituojieer_moon_bigtts", - "zh_male_yuzhouzixuan_moon_bigtts", - "zh_male_naiqimengwa_mars_bigtts", - "zh_female_popo_mars_bigtts", - "zh_female_gaolengyujie_moon_bigtts", - "zh_male_aojiaobazong_moon_bigtts", - "zh_female_meilinvyou_moon_bigtts", - "zh_male_shenyeboke_moon_bigtts", - "zh_female_sajiaonvyou_moon_bigtts", - "zh_female_yuanqinvyou_moon_bigtts", - "ICL_zh_female_bingruoshaonv_tob", - "ICL_zh_female_huoponvhai_tob", - "zh_male_dongfanghaoran_moon_bigtts", - "ICL_zh_female_heainainai_tob", - "ICL_zh_female_linjuayi_tob", - "zh_female_wenrouxiaoya_moon_bigtts", - "zh_male_tiancaitongsheng_mars_bigtts", - "zh_male_sunwukong_mars_bigtts", - "zh_male_xionger_mars_bigtts", - "zh_female_peiqi_mars_bigtts", - "zh_female_wuzetian_mars_bigtts", - "zh_female_gujie_mars_bigtts", - "zh_female_yingtaowanzi_mars_bigtts", - "zh_male_chunhui_mars_bigtts", - "zh_female_shaoergushi_mars_bigtts", - "zh_male_silang_mars_bigtts", - "zh_male_jieshuonansheng_mars_bigtts", - "zh_female_jitangmeimei_mars_bigtts", - "zh_female_tiexinnvsheng_mars_bigtts", - "zh_female_qiaopinvsheng_mars_bigtts", - "zh_female_mengyatou_mars_bigtts", - "zh_male_changtianyi_mars_bigtts", - "zh_male_ruyaqingnian_mars_bigtts", - "zh_male_baqiqingshu_mars_bigtts", - "zh_male_qingcang_mars_bigtts", - "zh_female_gufengshaoyu_mars_bigtts", - "zh_female_wenroushunv_mars_bigtts", - }), - ), - }, - - // { - // Model: "Doubao-stt", - // Type: relaymode.AudioTranscription, - // Owner: model.ModelOwnerDoubao, - // InputPrice: 2.3, - // }, -} diff --git a/service/aiproxy/relay/adaptor/doubaoaudio/key.go b/service/aiproxy/relay/adaptor/doubaoaudio/key.go deleted file mode 100644 index 46a6b99ef..000000000 --- a/service/aiproxy/relay/adaptor/doubaoaudio/key.go +++ /dev/null @@ -1,28 +0,0 @@ -package doubaoaudio - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, _, err := getAppIDAndToken(key) - return err -} - -func (a *Adaptor) KeyHelp() string { - return "app_id|app_token" -} - -// key格式: app_id|app_token -func getAppIDAndToken(key string) (string, string, error) { - parts := strings.Split(key, "|") - if len(parts) != 2 { - return "", "", errors.New("invalid key format") - } - return parts[0], parts[1], nil -} diff --git a/service/aiproxy/relay/adaptor/doubaoaudio/main.go b/service/aiproxy/relay/adaptor/doubaoaudio/main.go deleted file mode 100644 index 96e31e338..000000000 --- a/service/aiproxy/relay/adaptor/doubaoaudio/main.go +++ /dev/null @@ -1,89 +0,0 @@ -package doubaoaudio - -import ( - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -func GetRequestURL(meta *meta.Meta) (string, error) { - u := meta.Channel.BaseURL - switch meta.Mode { - case relaymode.AudioSpeech: - return u + "/api/v1/tts/ws_binary", nil - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -type Adaptor struct{} - -const baseURL = "https://openspeech.bytedance.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - return GetRequestURL(meta) -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.AudioSpeech: - return ConvertTTSRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - switch meta.Mode { - case relaymode.AudioSpeech: - _, token, err := getAppIDAndToken(meta.Channel.Key) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer;"+token) - return nil - default: - return fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoRequest(meta *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - switch meta.Mode { - case relaymode.AudioSpeech: - return TTSDoRequest(meta, req) - default: - return nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.AudioSpeech: - return TTSDoResponse(meta, c, resp) - default: - return nil, openai.ErrorWrapperWithMessage( - fmt.Sprintf("unsupported mode: %s", meta.Mode), - nil, - http.StatusBadRequest, - ) - } -} - -func (a *Adaptor) GetChannelName() string { - return "doubao audio" -} diff --git a/service/aiproxy/relay/adaptor/doubaoaudio/tts.go b/service/aiproxy/relay/adaptor/doubaoaudio/tts.go deleted file mode 100644 index 526c3bb0b..000000000 --- a/service/aiproxy/relay/adaptor/doubaoaudio/tts.go +++ /dev/null @@ -1,295 +0,0 @@ -package doubaoaudio - -import ( - "bytes" - "compress/gzip" - "encoding/binary" - "errors" - "io" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/google/uuid" - "github.com/gorilla/websocket" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type DoubaoTTSRequest struct { - Request RequestConfig `json:"request"` - App AppConfig `json:"app"` - User UserConfig `json:"user"` - Audio AudioConfig `json:"audio"` -} - -type AppConfig struct { - AppID string `json:"appid"` - Token string `json:"token"` - Cluster string `json:"cluster"` -} - -type UserConfig struct { - UID string `json:"uid,omitempty"` -} - -type AudioConfig struct { - VoiceType string `json:"voice_type"` - Encoding string `json:"encoding"` - SpeedRatio float64 `json:"speed_ratio,omitempty"` - VolumeRatio float64 `json:"volume_ratio,omitempty"` - PitchRatio float64 `json:"pitch_ratio,omitempty"` -} - -type RequestConfig struct { - ReqID string `json:"reqid"` - Text string `json:"text"` - TextType string `json:"text_type"` - Operation string `json:"operation"` -} - -// version: b0001 (4 bits) -// header size: b0001 (4 bits) -// message type: b0001 (Full client request) (4bits) -// message type specific flags: b0000 (none) (4bits) -// message serialization method: b0001 (JSON) (4 bits) -// message compression: b0001 (gzip) (4bits) -// reserved data: 0x00 (1 byte) -var defaultHeader = []byte{0x11, 0x10, 0x11, 0x00} - -//nolint:gosec -func ConvertTTSRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalTTSRequest(req) - if err != nil { - return "", nil, nil, err - } - - reqMap, err := utils.UnmarshalMap(req) - if err != nil { - return "", nil, nil, err - } - - appID, token, err := getAppIDAndToken(meta.Channel.Key) - if err != nil { - return "", nil, nil, err - } - - cluster := "volcano_tts" - textType := "ssml" - if strings.HasPrefix(request.Voice, "S_") { - cluster = "volcano_mega" - textType = "plain" - } - - doubaoRequest := DoubaoTTSRequest{ - App: AppConfig{ - AppID: appID, - Token: token, - Cluster: cluster, - }, - User: UserConfig{ - UID: meta.RequestID, - }, - Audio: AudioConfig{ - SpeedRatio: request.Speed, - }, - Request: RequestConfig{ - ReqID: uuid.New().String(), - Text: request.Input, - TextType: textType, - Operation: "submit", - }, - } - - if request.Voice == "" { - request.Voice = "zh_female_cancan_mars_bigtts" - } - doubaoRequest.Audio.VoiceType = request.Voice - - if request.ResponseFormat == "" { - request.ResponseFormat = "pcm" - } - doubaoRequest.Audio.Encoding = request.ResponseFormat - - volumeRatio, ok := reqMap["volume_ratio"].(float64) - if ok { - doubaoRequest.Audio.VolumeRatio = volumeRatio - } - pitchRatio, ok := reqMap["pitch_ratio"].(float64) - if ok { - doubaoRequest.Audio.PitchRatio = pitchRatio - } - - data, err := sonic.Marshal(doubaoRequest) - if err != nil { - return "", nil, nil, err - } - - compressedData, err := gzipCompress(data) - if err != nil { - return "", nil, nil, err - } - - payloadArr := make([]byte, 4) - binary.BigEndian.PutUint32(payloadArr, uint32(len(compressedData))) - clientRequest := make([]byte, len(defaultHeader)) - copy(clientRequest, defaultHeader) - clientRequest = append(clientRequest, payloadArr...) - clientRequest = append(clientRequest, compressedData...) - - return http.MethodPost, nil, bytes.NewReader(clientRequest), nil -} - -func TTSDoRequest(meta *meta.Meta, req *http.Request) (*http.Response, error) { - wsURL := req.URL - wsURL.Scheme = "wss" - - conn, _, err := websocket.DefaultDialer.Dial(wsURL.String(), req.Header) - if err != nil { - return nil, err - } - meta.Set("ws_conn", conn) - - writer, err := conn.NextWriter(websocket.BinaryMessage) - if err != nil { - return nil, err - } - defer writer.Close() - - _, err = io.Copy(writer, req.Body) - if err != nil { - return nil, err - } - - return &http.Response{ - StatusCode: http.StatusOK, - Body: io.NopCloser(nil), - }, nil -} - -func TTSDoResponse(meta *meta.Meta, c *gin.Context, _ *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - log := middleware.GetLogger(c) - - conn := meta.MustGet("ws_conn").(*websocket.Conn) - defer conn.Close() - - usage := &relaymodel.Usage{ - PromptTokens: meta.InputTokens, - TotalTokens: meta.InputTokens, - } - - for { - _, message, err := conn.ReadMessage() - if err != nil { - return usage, openai.ErrorWrapperWithMessage(err.Error(), "doubao_wss_read_msg_failed", http.StatusInternalServerError) - } - - resp, err := parseResponse(message) - if err != nil { - return usage, openai.ErrorWrapperWithMessage(err.Error(), "doubao_tts_parse_response_failed", http.StatusInternalServerError) - } - - _, err = c.Writer.Write(resp.Audio) - if err != nil { - log.Error("write tts response chunk failed: " + err.Error()) - } - - if resp.IsLast { - break - } - } - - return usage, nil -} - -func gzipCompress(input []byte) ([]byte, error) { - var b bytes.Buffer - w := gzip.NewWriter(&b) - _, err := w.Write(input) - if err != nil { - _ = w.Close() - return nil, err - } - err = w.Close() - if err != nil { - return nil, err - } - return b.Bytes(), nil -} - -func gzipDecompress(input []byte) ([]byte, error) { - b := bytes.NewBuffer(input) - r, err := gzip.NewReader(b) - if err != nil { - return nil, err - } - defer r.Close() - out, err := io.ReadAll(r) - if err != nil { - return nil, err - } - return out, nil -} - -type synResp struct { - Audio []byte - IsLast bool -} - -//nolint:gosec -func parseResponse(res []byte) (resp synResp, err error) { - // protoVersion := res[0] >> 4 - headSize := res[0] & 0x0f - messageType := res[1] >> 4 - messageTypeSpecificFlags := res[1] & 0x0f - // serializationMethod := res[2] >> 4 - messageCompression := res[2] & 0x0f - // reserve := res[3] - // headerExtensions := res[4 : headSize*4] - payload := res[headSize*4:] - - // audio-only server response - switch messageType { - case 0xb: - // no sequence number as ACK - if messageTypeSpecificFlags != 0 { - sequenceNumber := int32(binary.BigEndian.Uint32(payload[0:4])) - // payloadSize := int32(binary.BigEndian.Uint32(payload[4:8])) - payload = payload[8:] - resp.Audio = payload - if sequenceNumber < 0 { - resp.IsLast = true - } - } - return - case 0xf: - // code := int32(binary.BigEndian.Uint32(payload[0:4])) - errMsg := payload[8:] - if messageCompression == 1 { - errMsg, err = gzipDecompress(errMsg) - if err != nil { - return - } - } - err = errors.New(conv.BytesToString(errMsg)) - return - case 0xc: - // msgSize = int32(binary.BigEndian.Uint32(payload[0:4])) - // payload = payload[4:] - // if messageCompression == 1 { - // payload, err = gzipDecompress(payload) - // if err != nil { - // return - // } - // } - return - default: - err = errors.New("wrong message type") - return - } -} diff --git a/service/aiproxy/relay/adaptor/gemini/adaptor.go b/service/aiproxy/relay/adaptor/gemini/adaptor.go deleted file mode 100644 index 88027a9ce..000000000 --- a/service/aiproxy/relay/adaptor/gemini/adaptor.go +++ /dev/null @@ -1,96 +0,0 @@ -package gemini - -import ( - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const baseURL = "https://generativelanguage.googleapis.com" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -var v1ModelMap = map[string]struct{}{} - -func getRequestURL(meta *meta.Meta, action string) string { - u := meta.Channel.BaseURL - if u == "" { - u = baseURL - } - version := "v1beta" - if _, ok := v1ModelMap[meta.ActualModel]; ok { - version = "v1" - } - return fmt.Sprintf("%s/%s/models/%s:%s", u, version, meta.ActualModel, action) -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - var action string - switch meta.Mode { - case relaymode.Embeddings: - action = "batchEmbedContents" - default: - action = "generateContent" - } - - if meta.GetBool("stream") { - action = "streamGenerateContent?alt=sse" - } - return getRequestURL(meta, action), nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - req.Header.Set("X-Goog-Api-Key", meta.Channel.Key) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.Embeddings: - return ConvertEmbeddingRequest(meta, req) - case relaymode.ChatCompletions: - return ConvertRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.Embeddings: - usage, err = EmbeddingHandler(c, resp) - case relaymode.ChatCompletions: - if utils.IsStreamResponse(resp) { - usage, err = StreamHandler(meta, c, resp) - } else { - usage, err = Handler(meta, c, resp) - } - default: - return nil, openai.ErrorWrapperWithMessage(fmt.Sprintf("unsupported mode: %s", meta.Mode), "unsupported_mode", http.StatusBadRequest) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "google gemini" -} diff --git a/service/aiproxy/relay/adaptor/gemini/constants.go b/service/aiproxy/relay/adaptor/gemini/constants.go deleted file mode 100644 index bdf2eab28..000000000 --- a/service/aiproxy/relay/adaptor/gemini/constants.go +++ /dev/null @@ -1,121 +0,0 @@ -package gemini - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://ai.google.dev/models/gemini -// https://ai.google.dev/gemini-api/docs/pricing - -var ModelList = []*model.ModelConfig{ - { - Model: "gemini-1.5-pro", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.0025, - OutputPrice: 0.01, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(2097152), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "gemini-1.5-flash", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.00015, - OutputPrice: 0.0006, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1048576), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "gemini-1.5-flash-8b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.000075, - OutputPrice: 0.0003, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1048576), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "gemini-2.0-flash", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.0001, - OutputPrice: 0.0004, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1048576), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "gemini-2.0-flash-lite-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.000075, - OutputPrice: 0.0003, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1048576), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "gemini-2.0-flash-thinking-exp", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.0001, - OutputPrice: 0.0004, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1048576), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigVision(true), - ), - }, - { - Model: "gemini-2.0-pro-exp", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.0025, - OutputPrice: 0.01, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(2097152), - model.WithModelConfigMaxOutputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - - { - Model: "text-embedding-004", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerGoogle, - InputPrice: 0.0001, - RPM: 1500, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(2048), - model.WithModelConfigMaxOutputTokens(768), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/gemini/embeddings.go b/service/aiproxy/relay/adaptor/gemini/embeddings.go deleted file mode 100644 index 65cc0f6c2..000000000 --- a/service/aiproxy/relay/adaptor/gemini/embeddings.go +++ /dev/null @@ -1,90 +0,0 @@ -package gemini - -import ( - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func ConvertEmbeddingRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - request.Model = meta.ActualModel - - inputs := request.ParseInput() - requests := make([]EmbeddingRequest, len(inputs)) - model := "models/" + request.Model - - for i, input := range inputs { - requests[i] = EmbeddingRequest{ - Model: model, - Content: ChatContent{ - Parts: []Part{ - { - Text: input, - }, - }, - }, - } - } - - data, err := sonic.Marshal(BatchEmbeddingRequest{ - Requests: requests, - }) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func EmbeddingHandler(c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - var geminiEmbeddingResponse EmbeddingResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&geminiEmbeddingResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if geminiEmbeddingResponse.Error != nil { - return nil, openai.ErrorWrapperWithMessage(geminiEmbeddingResponse.Error.Message, geminiEmbeddingResponse.Error.Code, resp.StatusCode) - } - fullTextResponse := embeddingResponseGemini2OpenAI(&geminiEmbeddingResponse) - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &fullTextResponse.Usage, nil -} - -func embeddingResponseGemini2OpenAI(response *EmbeddingResponse) *openai.EmbeddingResponse { - openAIEmbeddingResponse := openai.EmbeddingResponse{ - Object: "list", - Data: make([]*openai.EmbeddingResponseItem, 0, len(response.Embeddings)), - Model: "gemini-embedding", - Usage: model.Usage{TotalTokens: 0}, - } - for _, item := range response.Embeddings { - openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, &openai.EmbeddingResponseItem{ - Object: `embedding`, - Index: 0, - Embedding: item.Values, - }) - } - return &openAIEmbeddingResponse -} diff --git a/service/aiproxy/relay/adaptor/gemini/main.go b/service/aiproxy/relay/adaptor/gemini/main.go deleted file mode 100644 index 830d1d277..000000000 --- a/service/aiproxy/relay/adaptor/gemini/main.go +++ /dev/null @@ -1,538 +0,0 @@ -package gemini - -import ( - "bufio" - "bytes" - "context" - "io" - "net/http" - "strings" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/image" - "github.com/labring/sealos/service/aiproxy/common/random" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" - log "github.com/sirupsen/logrus" -) - -// https://ai.google.dev/docs/gemini_api_overview?hl=zh-cn - -const ( - VisionMaxImageNum = 16 -) - -var toolChoiceTypeMap = map[string]string{ - "none": "NONE", - "auto": "AUTO", - "required": "ANY", -} - -var mimeTypeMap = map[string]string{ - "json_object": "application/json", - "text": "text/plain", -} - -type CountTokensResponse struct { - Error *Error `json:"error,omitempty"` - TotalTokens int `json:"totalTokens"` -} - -func buildSafetySettings() []ChatSafetySettings { - safetySetting := config.GetGeminiSafetySetting() - return []ChatSafetySettings{ - {Category: "HARM_CATEGORY_HARASSMENT", Threshold: safetySetting}, - {Category: "HARM_CATEGORY_HATE_SPEECH", Threshold: safetySetting}, - {Category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", Threshold: safetySetting}, - {Category: "HARM_CATEGORY_DANGEROUS_CONTENT", Threshold: safetySetting}, - {Category: "HARM_CATEGORY_CIVIC_INTEGRITY", Threshold: safetySetting}, - } -} - -func buildGenerationConfig(textRequest *model.GeneralOpenAIRequest) *ChatGenerationConfig { - config := ChatGenerationConfig{ - Temperature: textRequest.Temperature, - TopP: textRequest.TopP, - MaxOutputTokens: textRequest.MaxTokens, - } - - if textRequest.ResponseFormat != nil { - if mimeType, ok := mimeTypeMap[textRequest.ResponseFormat.Type]; ok { - config.ResponseMimeType = mimeType - } - if textRequest.ResponseFormat.JSONSchema != nil { - config.ResponseSchema = textRequest.ResponseFormat.JSONSchema.Schema - config.ResponseMimeType = mimeTypeMap["json_object"] - } - } - - return &config -} - -func buildTools(textRequest *model.GeneralOpenAIRequest) []ChatTools { - if textRequest.Tools != nil { - functions := make([]model.Function, 0, len(textRequest.Tools)) - for _, tool := range textRequest.Tools { - if parameters, ok := tool.Function.Parameters.(map[string]any); ok { - if properties, ok := parameters["properties"].(map[string]any); ok { - if len(properties) == 0 { - tool.Function.Parameters = nil - } - } - } - functions = append(functions, tool.Function) - } - return []ChatTools{{FunctionDeclarations: functions}} - } - if textRequest.Functions != nil { - return []ChatTools{{FunctionDeclarations: textRequest.Functions}} - } - return nil -} - -func buildToolConfig(textRequest *model.GeneralOpenAIRequest) *ToolConfig { - if textRequest.ToolChoice == nil { - return nil - } - toolConfig := ToolConfig{ - FunctionCallingConfig: FunctionCallingConfig{ - Mode: "auto", - }, - } - switch mode := textRequest.ToolChoice.(type) { - case string: - if toolChoiceType, ok := toolChoiceTypeMap[mode]; ok { - toolConfig.FunctionCallingConfig.Mode = toolChoiceType - } - case map[string]interface{}: - toolConfig.FunctionCallingConfig.Mode = "ANY" - if fn, ok := mode["function"].(map[string]interface{}); ok { - if fnName, ok := fn["name"].(string); ok { - toolConfig.FunctionCallingConfig.AllowedFunctionNames = []string{fnName} - } - } - } - return &toolConfig -} - -func buildMessageParts(ctx context.Context, part model.MessageContent) ([]Part, error) { - if part.Type == model.ContentTypeText { - return []Part{{Text: part.Text}}, nil - } - - if part.Type == model.ContentTypeImageURL { - mimeType, data, err := image.GetImageFromURL(ctx, part.ImageURL.URL) - if err != nil { - return nil, err - } - return []Part{{ - InlineData: &InlineData{ - MimeType: mimeType, - Data: data, - }, - }}, nil - } - - return nil, nil -} - -func buildContents(ctx context.Context, textRequest *model.GeneralOpenAIRequest) (*ChatContent, []*ChatContent, error) { - contents := make([]*ChatContent, 0, len(textRequest.Messages)) - imageNum := 0 - - var systemContent *ChatContent - - for _, message := range textRequest.Messages { - content := ChatContent{ - Role: message.Role, - Parts: make([]Part, 0), - } - - if message.Role == "assistant" && len(message.ToolCalls) > 0 { - for _, toolCall := range message.ToolCalls { - var args map[string]any - if toolCall.Function.Arguments != "" { - if err := sonic.UnmarshalString(toolCall.Function.Arguments, &args); err != nil { - args = make(map[string]any) - } - } else { - args = make(map[string]any) - } - content.Parts = append(content.Parts, Part{ - FunctionCall: &FunctionCall{ - Name: toolCall.Function.Name, - Args: args, - }, - }) - } - } else if message.Role == "tool" && message.ToolCallID != "" { - var contentMap map[string]any - if message.Content != nil { - switch content := message.Content.(type) { - case map[string]any: - contentMap = content - case string: - if err := sonic.UnmarshalString(content, &contentMap); err != nil { - log.Error("unmarshal content failed: " + err.Error()) - } - } - } else { - contentMap = make(map[string]any) - } - content.Parts = append(content.Parts, Part{ - FunctionResponse: &FunctionResponse{ - Name: *message.Name, - Response: struct { - Name string `json:"name"` - Content map[string]any `json:"content"` - }{ - Name: *message.Name, - Content: contentMap, - }, - }, - }) - } else { - openaiContent := message.ParseContent() - for _, part := range openaiContent { - if part.Type == model.ContentTypeImageURL { - imageNum++ - if imageNum > VisionMaxImageNum { - continue - } - } - - parts, err := buildMessageParts(ctx, part) - if err != nil { - return nil, nil, err - } - content.Parts = append(content.Parts, parts...) - } - } - - switch content.Role { - case "assistant": - content.Role = "model" - case "tool": - content.Role = "user" - case "system": - systemContent = &content - continue - } - contents = append(contents, &content) - } - - return systemContent, contents, nil -} - -// Setting safety to the lowest possible values since Gemini is already powerless enough -func ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - textRequest, err := utils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - - textRequest.Model = meta.ActualModel - meta.Set("stream", textRequest.Stream) - - systemContent, contents, err := buildContents(req.Context(), textRequest) - if err != nil { - return "", nil, nil, err - } - - // Build actual request - geminiRequest := ChatRequest{ - Contents: contents, - SystemInstruction: systemContent, - SafetySettings: buildSafetySettings(), - GenerationConfig: buildGenerationConfig(textRequest), - Tools: buildTools(textRequest), - ToolConfig: buildToolConfig(textRequest), - } - - data, err := sonic.Marshal(geminiRequest) - if err != nil { - return "", nil, nil, err - } - - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -type ChatResponse struct { - Candidates []*ChatCandidate `json:"candidates"` - PromptFeedback ChatPromptFeedback `json:"promptFeedback"` - UsageMetadata *UsageMetadata `json:"usageMetadata"` - ModelVersion string `json:"modelVersion"` -} - -type UsageMetadata struct { - PromptTokenCount int `json:"promptTokenCount"` - CandidatesTokenCount int `json:"candidatesTokenCount"` - TotalTokenCount int `json:"totalTokenCount"` -} - -func (g *ChatResponse) GetResponseText() string { - if g == nil { - return "" - } - builder := strings.Builder{} - for _, candidate := range g.Candidates { - for i, part := range candidate.Content.Parts { - if i > 0 { - builder.WriteString("\n") - } - builder.WriteString(part.Text) - } - } - return builder.String() -} - -type ChatCandidate struct { - FinishReason string `json:"finishReason"` - Content ChatContent `json:"content"` - SafetyRatings []ChatSafetyRating `json:"safetyRatings"` - Index int64 `json:"index"` -} - -type ChatSafetyRating struct { - Category string `json:"category"` - Probability string `json:"probability"` -} - -type ChatPromptFeedback struct { - SafetyRatings []ChatSafetyRating `json:"safetyRatings"` -} - -func getToolCalls(candidate *ChatCandidate, toolCallIndex int) []*model.Tool { - if len(candidate.Content.Parts) <= toolCallIndex { - return nil - } - - var toolCalls []*model.Tool - item := candidate.Content.Parts[toolCallIndex] - if item.FunctionCall == nil { - return toolCalls - } - argsBytes, err := sonic.Marshal(item.FunctionCall.Args) - if err != nil { - log.Error("getToolCalls failed: " + err.Error()) - return toolCalls - } - toolCall := model.Tool{ - ID: "call_" + random.GetUUID(), - Type: "function", - Function: model.Function{ - Arguments: conv.BytesToString(argsBytes), - Name: item.FunctionCall.Name, - }, - } - toolCalls = append(toolCalls, &toolCall) - return toolCalls -} - -func responseGeminiChat2OpenAI(meta *meta.Meta, response *ChatResponse) *openai.TextResponse { - fullTextResponse := openai.TextResponse{ - ID: "chatcmpl-" + random.GetUUID(), - Model: meta.OriginModel, - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: make([]*openai.TextResponseChoice, 0, len(response.Candidates)), - } - for i, candidate := range response.Candidates { - choice := openai.TextResponseChoice{ - Index: i, - Message: model.Message{ - Role: "assistant", - }, - FinishReason: constant.StopFinishReason, - } - if len(candidate.Content.Parts) > 0 { - toolCallIndex := -1 - for i, part := range candidate.Content.Parts { - if part.FunctionCall != nil { - toolCallIndex = i - break - } - } - if toolCallIndex != -1 { - choice.Message.ToolCalls = getToolCalls(candidate, toolCallIndex) - content := strings.Builder{} - for i, part := range candidate.Content.Parts { - if i == toolCallIndex { - continue - } - content.WriteString(part.Text) - } - choice.Message.Content = content.String() - } else { - builder := strings.Builder{} - for i, part := range candidate.Content.Parts { - if i > 0 { - builder.WriteString("\n") - } - builder.WriteString(part.Text) - } - choice.Message.Content = builder.String() - } - } else { - choice.Message.Content = "" - choice.FinishReason = candidate.FinishReason - } - fullTextResponse.Choices = append(fullTextResponse.Choices, &choice) - } - return &fullTextResponse -} - -func streamResponseGeminiChat2OpenAI(meta *meta.Meta, geminiResponse *ChatResponse) *openai.ChatCompletionsStreamResponse { - response := &openai.ChatCompletionsStreamResponse{ - ID: "chatcmpl-" + random.GetUUID(), - Created: time.Now().Unix(), - Model: meta.OriginModel, - Object: "chat.completion.chunk", - Choices: make([]*openai.ChatCompletionsStreamResponseChoice, 0, len(geminiResponse.Candidates)), - } - if geminiResponse.UsageMetadata != nil { - response.Usage = &model.Usage{ - PromptTokens: geminiResponse.UsageMetadata.PromptTokenCount, - CompletionTokens: geminiResponse.UsageMetadata.CandidatesTokenCount, - TotalTokens: geminiResponse.UsageMetadata.TotalTokenCount, - } - } - for i, candidate := range geminiResponse.Candidates { - choice := openai.ChatCompletionsStreamResponseChoice{ - Index: i, - } - if len(candidate.Content.Parts) > 0 { - toolCallIndex := -1 - for i, part := range candidate.Content.Parts { - if part.FunctionCall != nil { - toolCallIndex = i - break - } - } - if toolCallIndex != -1 { - choice.Delta.ToolCalls = getToolCalls(candidate, toolCallIndex) - content := strings.Builder{} - for i, part := range candidate.Content.Parts { - if i == toolCallIndex { - continue - } - content.WriteString(part.Text) - } - choice.Delta.Content = content.String() - } else { - builder := strings.Builder{} - for i, part := range candidate.Content.Parts { - if i > 0 { - builder.WriteString("\n") - } - builder.WriteString(part.Text) - } - choice.Delta.Content = builder.String() - } - } else { - choice.Delta.Content = "" - choice.FinishReason = &candidate.FinishReason - } - response.Choices = append(response.Choices, &choice) - } - return response -} - -func StreamHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - log := middleware.GetLogger(c) - - responseText := strings.Builder{} - scanner := bufio.NewScanner(resp.Body) - scanner.Split(bufio.ScanLines) - - common.SetEventStreamHeaders(c) - - usage := model.Usage{ - PromptTokens: meta.InputTokens, - } - - for scanner.Scan() { - data := scanner.Bytes() - if len(data) < 6 || conv.BytesToString(data[:6]) != "data: " { - continue - } - data = data[6:] - - if conv.BytesToString(data) == "[DONE]" { - break - } - - var geminiResponse ChatResponse - err := sonic.Unmarshal(data, &geminiResponse) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - response := streamResponseGeminiChat2OpenAI(meta, &geminiResponse) - if response.Usage != nil { - usage = *response.Usage - } - - responseText.WriteString(response.Choices[0].Delta.StringContent()) - - _ = render.ObjectData(c, response) - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - render.Done(c) - - return &usage, nil -} - -func Handler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - defer resp.Body.Close() - - var geminiResponse ChatResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&geminiResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if len(geminiResponse.Candidates) == 0 { - return nil, openai.ErrorWrapperWithMessage("No candidates returned", "gemini_error", resp.StatusCode) - } - fullTextResponse := responseGeminiChat2OpenAI(meta, &geminiResponse) - fullTextResponse.Model = meta.OriginModel - - usage := model.Usage{ - PromptTokens: geminiResponse.UsageMetadata.PromptTokenCount, - CompletionTokens: geminiResponse.UsageMetadata.CandidatesTokenCount, - TotalTokens: geminiResponse.UsageMetadata.TotalTokenCount, - } - fullTextResponse.Usage = usage - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &usage, nil -} diff --git a/service/aiproxy/relay/adaptor/gemini/model.go b/service/aiproxy/relay/adaptor/gemini/model.go deleted file mode 100644 index 8ed1a9973..000000000 --- a/service/aiproxy/relay/adaptor/gemini/model.go +++ /dev/null @@ -1,96 +0,0 @@ -package gemini - -type ChatRequest struct { - Contents []*ChatContent `json:"contents"` - SystemInstruction *ChatContent `json:"system_instruction,omitempty"` - SafetySettings []ChatSafetySettings `json:"safety_settings,omitempty"` - GenerationConfig *ChatGenerationConfig `json:"generation_config,omitempty"` - Tools []ChatTools `json:"tools,omitempty"` - ToolConfig *ToolConfig `json:"tool_config,omitempty"` -} - -type EmbeddingRequest struct { - Model string `json:"model"` - TaskType string `json:"taskType,omitempty"` - Title string `json:"title,omitempty"` - Content ChatContent `json:"content"` - OutputDimensionality int `json:"outputDimensionality,omitempty"` -} - -type BatchEmbeddingRequest struct { - Requests []EmbeddingRequest `json:"requests"` -} - -type EmbeddingData struct { - Values []float64 `json:"values"` -} - -type EmbeddingResponse struct { - Error *Error `json:"error,omitempty"` - Embeddings []EmbeddingData `json:"embeddings"` -} - -type Error struct { - Message string `json:"message,omitempty"` - Status string `json:"status,omitempty"` - Code int `json:"code,omitempty"` -} - -type InlineData struct { - MimeType string `json:"mimeType"` - Data string `json:"data"` -} - -type FunctionCall struct { - Args map[string]any `json:"args"` - Name string `json:"name"` -} - -type FunctionResponse struct { - Name string `json:"name"` - Response struct { - Name string `json:"name"` - Content map[string]any `json:"content"` - } `json:"response"` -} - -type Part struct { - InlineData *InlineData `json:"inlineData,omitempty"` - FunctionCall *FunctionCall `json:"functionCall,omitempty"` - FunctionResponse *FunctionResponse `json:"functionResponse,omitempty"` - Text string `json:"text,omitempty"` -} - -type ChatContent struct { - Role string `json:"role,omitempty"` - Parts []Part `json:"parts"` -} - -type ChatSafetySettings struct { - Category string `json:"category"` - Threshold string `json:"threshold"` -} - -type ChatTools struct { - FunctionDeclarations any `json:"function_declarations,omitempty"` -} - -type ChatGenerationConfig struct { - ResponseSchema any `json:"responseSchema,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"topP,omitempty"` - ResponseMimeType string `json:"responseMimeType,omitempty"` - StopSequences []string `json:"stopSequences,omitempty"` - TopK float64 `json:"topK,omitempty"` - MaxOutputTokens int `json:"maxOutputTokens,omitempty"` - CandidateCount int `json:"candidateCount,omitempty"` -} - -type FunctionCallingConfig struct { - Mode string `json:"mode,omitempty"` - AllowedFunctionNames []string `json:"allowed_function_names,omitempty"` -} - -type ToolConfig struct { - FunctionCallingConfig FunctionCallingConfig `json:"function_calling_config"` -} diff --git a/service/aiproxy/relay/adaptor/geminiopenai/adaptor.go b/service/aiproxy/relay/adaptor/geminiopenai/adaptor.go deleted file mode 100644 index b6c20ae29..000000000 --- a/service/aiproxy/relay/adaptor/geminiopenai/adaptor.go +++ /dev/null @@ -1,25 +0,0 @@ -package geminiopenai - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/gemini" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://generativelanguage.googleapis.com/v1beta/openai" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return gemini.ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "google gemini (openai)" -} diff --git a/service/aiproxy/relay/adaptor/groq/adaptor.go b/service/aiproxy/relay/adaptor/groq/adaptor.go deleted file mode 100644 index 0db752eb2..000000000 --- a/service/aiproxy/relay/adaptor/groq/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package groq - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.groq.com/openai/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "groq" -} diff --git a/service/aiproxy/relay/adaptor/groq/constants.go b/service/aiproxy/relay/adaptor/groq/constants.go deleted file mode 100644 index b7e622782..000000000 --- a/service/aiproxy/relay/adaptor/groq/constants.go +++ /dev/null @@ -1,116 +0,0 @@ -package groq - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://console.groq.com/docs/models - -var ModelList = []*model.ModelConfig{ - { - Model: "gemma-7b-it", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "gemma2-9b-it", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "llama-3.1-70b-versatile", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.1-8b-instant", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-11b-text-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-11b-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-1b-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-3b-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-11b-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-90b-text-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-3.2-90b-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama-guard-3-8b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama3-70b-8192", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama3-8b-8192", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama3-groq-70b-8192-tool-use-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama3-groq-8b-8192-tool-use-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llava-v1.5-7b-4096-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "mixtral-8x7b-32768", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "distil-whisper-large-v3-en", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerHuggingFace, - }, - { - Model: "whisper-large-v3", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "whisper-large-v3-turbo", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerOpenAI, - }, -} diff --git a/service/aiproxy/relay/adaptor/interface.go b/service/aiproxy/relay/adaptor/interface.go deleted file mode 100644 index 32904f403..000000000 --- a/service/aiproxy/relay/adaptor/interface.go +++ /dev/null @@ -1,34 +0,0 @@ -package adaptor - -import ( - "errors" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type Adaptor interface { - GetChannelName() string - GetBaseURL() string - GetRequestURL(meta *meta.Meta) (string, error) - SetupRequestHeader(meta *meta.Meta, c *gin.Context, req *http.Request) error - ConvertRequest(meta *meta.Meta, req *http.Request) (method string, header http.Header, body io.Reader, err error) - DoRequest(meta *meta.Meta, c *gin.Context, req *http.Request) (*http.Response, error) - DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) - GetModelList() []*model.ModelConfig -} - -var ErrGetBalanceNotImplemented = errors.New("get balance not implemented") - -type Balancer interface { - GetBalance(channel *model.Channel) (float64, error) -} - -type KeyValidator interface { - ValidateKey(key string) error - KeyHelp() string -} diff --git a/service/aiproxy/relay/adaptor/lingyiwanwu/adaptor.go b/service/aiproxy/relay/adaptor/lingyiwanwu/adaptor.go deleted file mode 100644 index cf599fbeb..000000000 --- a/service/aiproxy/relay/adaptor/lingyiwanwu/adaptor.go +++ /dev/null @@ -1,29 +0,0 @@ -package lingyiwanwu - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.lingyiwanwu.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "lingyiwanwu" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/lingyiwanwu/constants.go b/service/aiproxy/relay/adaptor/lingyiwanwu/constants.go deleted file mode 100644 index 3b95f8412..000000000 --- a/service/aiproxy/relay/adaptor/lingyiwanwu/constants.go +++ /dev/null @@ -1,36 +0,0 @@ -package lingyiwanwu - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://platform.lingyiwanwu.com/docs - -var ModelList = []*model.ModelConfig{ - { - Model: "yi-lightning", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerLingyiWanwu, - InputPrice: 0.00099, - OutputPrice: 0.00099, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(16384), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "yi-vision-v2", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerLingyiWanwu, - InputPrice: 0.006, - OutputPrice: 0.006, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(16384), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/minimax/adaptor.go b/service/aiproxy/relay/adaptor/minimax/adaptor.go deleted file mode 100644 index e4d109777..000000000 --- a/service/aiproxy/relay/adaptor/minimax/adaptor.go +++ /dev/null @@ -1,84 +0,0 @@ -package minimax - -import ( - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.minimax.chat/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - apiKey, _, err := GetAPIKeyAndGroupID(meta.Channel.Key) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+apiKey) - return nil -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - _, groupID, err := GetAPIKeyAndGroupID(meta.Channel.Key) - if err != nil { - return "", err - } - switch meta.Mode { - case relaymode.ChatCompletions: - return meta.Channel.BaseURL + "/text/chatcompletion_v2", nil - case relaymode.Embeddings: - return fmt.Sprintf("%s/embeddings?GroupId=%s", meta.Channel.BaseURL, groupID), nil - case relaymode.AudioSpeech: - return fmt.Sprintf("%s/t2a_v2?GroupId=%s", meta.Channel.BaseURL, groupID), nil - default: - return a.Adaptor.GetRequestURL(meta) - } -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.ChatCompletions: - meta.Set(openai.DoNotPatchStreamOptionsIncludeUsageMetaKey, true) - return a.Adaptor.ConvertRequest(meta, req) - case relaymode.AudioSpeech: - return ConvertTTSRequest(meta, req) - default: - return a.Adaptor.ConvertRequest(meta, req) - } -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.AudioSpeech: - return TTSHandler(meta, c, resp) - default: - return a.Adaptor.DoResponse(meta, c, resp) - } -} - -func (a *Adaptor) GetChannelName() string { - return "minimax" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/minimax/constants.go b/service/aiproxy/relay/adaptor/minimax/constants.go deleted file mode 100644 index efc9edf1f..000000000 --- a/service/aiproxy/relay/adaptor/minimax/constants.go +++ /dev/null @@ -1,109 +0,0 @@ -package minimax - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://www.minimaxi.com/document/guides/chat-model/V2?id=65e0736ab2845de20908e2dd - -var ModelList = []*model.ModelConfig{ - { - Model: "abab7-chat-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.01, - OutputPrice: 0.01, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(245760), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "abab6.5s-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.001, - OutputPrice: 0.001, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(245760), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "abab6.5g-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.005, - OutputPrice: 0.005, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "abab6.5t-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.005, - OutputPrice: 0.005, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "abab5.5s-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.005, - OutputPrice: 0.005, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "abab5.5-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.015, - OutputPrice: 0.015, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(16384), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "speech-01-turbo", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerMiniMax, - InputPrice: 0.2, - RPM: 20, - Config: model.NewModelConfig( - model.WithModelConfigSupportFormats([]string{"pcm", "wav", "flac", "mp3"}), - model.WithModelConfigSupportVoices([]string{ - "male-qn-qingse", "male-qn-jingying", "male-qn-badao", "male-qn-daxuesheng", - "female-shaonv", "female-yujie", "female-chengshu", "female-tianmei", - "presenter_male", "presenter_female", - "audiobook_male_1", "audiobook_male_2", "audiobook_female_1", "audiobook_female_2", - "male-qn-qingse-jingpin", "male-qn-jingying-jingpin", "male-qn-badao-jingpin", "male-qn-daxuesheng-jingpin", - "female-shaonv-jingpin", "female-yujie-jingpin", "female-chengshu-jingpin", "female-tianmei-jingpin", - "clever_boy", "cute_boy", "lovely_girl", "cartoon_pig", - "bingjiao_didi", "junlang_nanyou", "chunzhen_xuedi", "lengdan_xiongzhang", - "badao_shaoye", "tianxin_xiaoling", "qiaopi_mengmei", "wumei_yujie", - "diadia_xuemei", "danya_xuejie", - "Santa_Claus", "Grinch", "Rudolph", "Arnold", - "Charming_Santa", "Charming_Lady", "Sweet_Girl", "Cute_Elf", - "Attractive_Girl", "Serene_Woman", - }), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/minimax/key.go b/service/aiproxy/relay/adaptor/minimax/key.go deleted file mode 100644 index b352343e4..000000000 --- a/service/aiproxy/relay/adaptor/minimax/key.go +++ /dev/null @@ -1,30 +0,0 @@ -package minimax - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, _, err := GetAPIKeyAndGroupID(key) - if err != nil { - return err - } - return nil -} - -func (a *Adaptor) KeyHelp() string { - return "api_key|group_id" -} - -func GetAPIKeyAndGroupID(key string) (string, string, error) { - keys := strings.Split(key, "|") - if len(keys) != 2 { - return "", "", errors.New("invalid key format") - } - return keys[0], keys[1], nil -} diff --git a/service/aiproxy/relay/adaptor/minimax/tts.go b/service/aiproxy/relay/adaptor/minimax/tts.go deleted file mode 100644 index ef0e655f2..000000000 --- a/service/aiproxy/relay/adaptor/minimax/tts.go +++ /dev/null @@ -1,204 +0,0 @@ -package minimax - -import ( - "bufio" - "bytes" - "encoding/hex" - "io" - "net/http" - "strconv" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func ConvertTTSRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - reqMap, err := utils.UnmarshalMap(req) - if err != nil { - return "", nil, nil, err - } - - reqMap["model"] = meta.ActualModel - - reqMap["text"] = reqMap["input"] - delete(reqMap, "input") - - voice, _ := reqMap["voice"].(string) - delete(reqMap, "voice") - if voice == "" { - voice = "male-qn-qingse" - } - - voiceSetting, ok := reqMap["voice_setting"].(map[string]any) - if !ok { - voiceSetting = map[string]any{} - reqMap["voice_setting"] = voiceSetting - } - if timberWeights, ok := reqMap["timber_weights"].([]any); !ok || len(timberWeights) == 0 { - voiceSetting["voice_id"] = voice - } - - speed, ok := reqMap["speed"].(float64) - if ok { - voiceSetting["speed"] = int(speed) - } - delete(reqMap, "speed") - - audioSetting, ok := reqMap["audio_setting"].(map[string]any) - if !ok { - audioSetting = map[string]any{} - reqMap["audio_setting"] = audioSetting - } - - responseFormat, ok := reqMap["response_format"].(string) - if ok && responseFormat != "" { - audioSetting["format"] = responseFormat - } - delete(reqMap, "response_format") - - sampleRate, ok := reqMap["sample_rate"].(float64) - if ok { - audioSetting["sample_rate"] = int(sampleRate) - } - delete(reqMap, "sample_rate") - - if responseFormat == "wav" { - reqMap["stream"] = false - meta.Set("stream", false) - } else { - stream, _ := reqMap["stream"].(bool) - meta.Set("stream", stream) - } - - body, err := sonic.Marshal(reqMap) - if err != nil { - return "", nil, nil, err - } - - return http.MethodPost, nil, bytes.NewReader(body), nil -} - -type TTSExtraInfo struct { - AudioFormat string `json:"audio_format"` - UsageCharacters int `json:"usage_characters"` -} - -type TTSBaseResp struct { - StatusMsg string `json:"status_msg"` - StatusCode int `json:"status_code"` -} - -type TTSData struct { - Audio string `json:"audio"` - Status int `json:"status"` -} - -type TTSResponse struct { - BaseResp *TTSBaseResp `json:"base_resp"` - ExtraInfo TTSExtraInfo `json:"extra_info"` - Data TTSData `json:"data"` -} - -func TTSHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, openai.ErrorHanlder(resp) - } - - if !strings.Contains(resp.Header.Get("Content-Type"), "application/json") && meta.GetBool("stream") { - return ttsStreamHandler(meta, c, resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, openai.ErrorWrapper(err, "TTS_ERROR", http.StatusInternalServerError) - } - - var result TTSResponse - if err := sonic.Unmarshal(body, &result); err != nil { - return nil, openai.ErrorWrapper(err, "TTS_ERROR", http.StatusInternalServerError) - } - if result.BaseResp != nil && result.BaseResp.StatusCode != 0 { - return nil, openai.ErrorWrapperWithMessage(result.BaseResp.StatusMsg, "TTS_ERROR_"+strconv.Itoa(result.BaseResp.StatusCode), http.StatusInternalServerError) - } - - resp.Header.Set("Content-Type", "audio/"+result.ExtraInfo.AudioFormat) - - audioBytes, err := hex.DecodeString(result.Data.Audio) - if err != nil { - return nil, openai.ErrorWrapper(err, "TTS_ERROR", http.StatusInternalServerError) - } - - _, err = c.Writer.Write(audioBytes) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - - usageCharacters := meta.InputTokens - if result.ExtraInfo.UsageCharacters > 0 { - usageCharacters = result.ExtraInfo.UsageCharacters - } - - return &relaymodel.Usage{ - PromptTokens: usageCharacters, - TotalTokens: usageCharacters, - }, nil -} - -func ttsStreamHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - defer resp.Body.Close() - - resp.Header.Set("Content-Type", "application/octet-stream") - - log := middleware.GetLogger(c) - - scanner := bufio.NewScanner(resp.Body) - scanner.Split(bufio.ScanLines) - - usageCharacters := meta.InputTokens - - for scanner.Scan() { - data := scanner.Text() - if len(data) < openai.DataPrefixLength { // ignore blank line or wrong format - continue - } - if data[:openai.DataPrefixLength] != openai.DataPrefix { - continue - } - data = data[openai.DataPrefixLength:] - - var result TTSResponse - if err := sonic.UnmarshalString(data, &result); err != nil { - log.Error("unmarshal tts response failed: " + err.Error()) - continue - } - if result.ExtraInfo.UsageCharacters > 0 { - usageCharacters = result.ExtraInfo.UsageCharacters - } - - audioBytes, err := hex.DecodeString(result.Data.Audio) - if err != nil { - log.Error("decode audio failed: " + err.Error()) - continue - } - - _, err = c.Writer.Write(audioBytes) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - } - - return &relaymodel.Usage{ - PromptTokens: usageCharacters, - TotalTokens: usageCharacters, - }, nil -} diff --git a/service/aiproxy/relay/adaptor/mistral/adaptor.go b/service/aiproxy/relay/adaptor/mistral/adaptor.go deleted file mode 100644 index 413cbc179..000000000 --- a/service/aiproxy/relay/adaptor/mistral/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package mistral - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.mistral.ai/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "mistral" -} diff --git a/service/aiproxy/relay/adaptor/mistral/constants.go b/service/aiproxy/relay/adaptor/mistral/constants.go deleted file mode 100644 index 82ad8a607..000000000 --- a/service/aiproxy/relay/adaptor/mistral/constants.go +++ /dev/null @@ -1,39 +0,0 @@ -package mistral - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "open-mistral-7b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "open-mixtral-8x7b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "mistral-small-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "mistral-medium-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "mistral-large-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "mistral-embed", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerMistral, - }, -} diff --git a/service/aiproxy/relay/adaptor/moonshot/adaptor.go b/service/aiproxy/relay/adaptor/moonshot/adaptor.go deleted file mode 100644 index 0f109a105..000000000 --- a/service/aiproxy/relay/adaptor/moonshot/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package moonshot - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.moonshot.cn/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "moonshot" -} diff --git a/service/aiproxy/relay/adaptor/moonshot/balance.go b/service/aiproxy/relay/adaptor/moonshot/balance.go deleted file mode 100644 index 7dbc0e078..000000000 --- a/service/aiproxy/relay/adaptor/moonshot/balance.go +++ /dev/null @@ -1,56 +0,0 @@ -package moonshot - -import ( - "context" - "fmt" - "net/http" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.Balancer = (*Adaptor)(nil) - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - u := channel.BaseURL - if u == "" { - u = baseURL - } - url := u + "/users/me/balance" - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", "Bearer "+channel.Key) - resp, err := http.DefaultClient.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - - var response BalanceResponse - if err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&response); err != nil { - return 0, err - } - - if response.Error != nil { - return 0, fmt.Errorf("type: %s, message: %s", response.Error.Type, response.Error.Message) - } - - return response.Data.AvailableBalance, nil -} - -type BalanceResponse struct { - Error *BalanceError `json:"error"` - Data BalanceData `json:"data"` -} - -type BalanceData struct { - AvailableBalance float64 `json:"available_balance"` -} - -type BalanceError struct { - Message string `json:"message"` - Type string `json:"type"` -} diff --git a/service/aiproxy/relay/adaptor/moonshot/constants.go b/service/aiproxy/relay/adaptor/moonshot/constants.go deleted file mode 100644 index 905f015c7..000000000 --- a/service/aiproxy/relay/adaptor/moonshot/constants.go +++ /dev/null @@ -1,85 +0,0 @@ -package moonshot - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "moonshot-v1-8k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMoonshot, - InputPrice: 0.012, - OutputPrice: 0.012, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "moonshot-v1-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMoonshot, - InputPrice: 0.024, - OutputPrice: 0.024, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(32768), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "moonshot-v1-128k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMoonshot, - InputPrice: 0.06, - OutputPrice: 0.06, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(131072), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "moonshot-v1-8k-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMoonshot, - InputPrice: 0.012, - OutputPrice: 0.012, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "moonshot-v1-32k-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMoonshot, - InputPrice: 0.024, - OutputPrice: 0.024, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(32768), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "moonshot-v1-128k-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMoonshot, - InputPrice: 0.06, - OutputPrice: 0.06, - RPM: 500, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(131072), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/novita/adaptor.go b/service/aiproxy/relay/adaptor/novita/adaptor.go deleted file mode 100644 index a1b45a0d3..000000000 --- a/service/aiproxy/relay/adaptor/novita/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package novita - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.novita.ai/v3/openai" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "novita" -} diff --git a/service/aiproxy/relay/adaptor/novita/constants.go b/service/aiproxy/relay/adaptor/novita/constants.go deleted file mode 100644 index d5566bf21..000000000 --- a/service/aiproxy/relay/adaptor/novita/constants.go +++ /dev/null @@ -1,46 +0,0 @@ -package novita - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://novita.ai/llm-api - -var ModelList = []*model.ModelConfig{ - { - Model: "meta-llama/llama-3-8b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "meta-llama/llama-3-70b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "nousresearch/hermes-2-pro-llama-3-8b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "nousresearch/nous-hermes-llama2-13b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "mistralai/mistral-7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "teknium/openhermes-2.5-mistral-7b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMistral, - }, - { - Model: "microsoft/wizardlm-2-8x22b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMicrosoft, - }, -} diff --git a/service/aiproxy/relay/adaptor/ollama/adaptor.go b/service/aiproxy/relay/adaptor/ollama/adaptor.go deleted file mode 100644 index ddc3a90d9..000000000 --- a/service/aiproxy/relay/adaptor/ollama/adaptor.go +++ /dev/null @@ -1,84 +0,0 @@ -package ollama - -import ( - "errors" - "fmt" - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -type Adaptor struct{} - -const baseURL = "http://localhost:11434" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - // https://github.com/ollama/ollama/blob/main/docs/api.md - u := meta.Channel.BaseURL - switch meta.Mode { - case relaymode.Embeddings: - return u + "/api/embed", nil - case relaymode.ChatCompletions: - return u + "/api/chat", nil - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) { - if request == nil { - return "", nil, nil, errors.New("request is nil") - } - switch meta.Mode { - case relaymode.Embeddings: - return ConvertEmbeddingRequest(meta, request) - case relaymode.ChatCompletions: - return ConvertRequest(meta, request) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.Embeddings: - usage, err = EmbeddingHandler(meta, c, resp) - case relaymode.ChatCompletions: - if utils.IsStreamResponse(resp) { - usage, err = StreamHandler(meta, c, resp) - } else { - usage, err = Handler(meta, c, resp) - } - default: - return nil, openai.ErrorWrapperWithMessage(fmt.Sprintf("unsupported mode: %s", meta.Mode), "unsupported_mode", http.StatusBadRequest) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "ollama" -} diff --git a/service/aiproxy/relay/adaptor/ollama/constants.go b/service/aiproxy/relay/adaptor/ollama/constants.go deleted file mode 100644 index 739ec1f0a..000000000 --- a/service/aiproxy/relay/adaptor/ollama/constants.go +++ /dev/null @@ -1,44 +0,0 @@ -package ollama - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "codellama:7b-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama2:7b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama2:latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "llama3:latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMeta, - }, - { - Model: "phi3:latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerMicrosoft, - }, - { - Model: "qwen:0.5b-chat", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - }, - { - Model: "qwen:7b", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAlibaba, - }, -} diff --git a/service/aiproxy/relay/adaptor/ollama/error.go b/service/aiproxy/relay/adaptor/ollama/error.go deleted file mode 100644 index 8e7795f2e..000000000 --- a/service/aiproxy/relay/adaptor/ollama/error.go +++ /dev/null @@ -1,24 +0,0 @@ -package ollama - -import ( - "net/http" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type errorResponse struct { - Error string `json:"error"` -} - -func ErrorHandler(resp *http.Response) *relaymodel.ErrorWithStatusCode { - defer resp.Body.Close() - - var er errorResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&er) - if err != nil { - return openai.ErrorWrapperWithMessage("decode response error: "+err.Error(), nil, http.StatusInternalServerError) - } - return openai.ErrorWrapperWithMessage(er.Error, nil, http.StatusInternalServerError) -} diff --git a/service/aiproxy/relay/adaptor/ollama/main.go b/service/aiproxy/relay/adaptor/ollama/main.go deleted file mode 100644 index c7af1011f..000000000 --- a/service/aiproxy/relay/adaptor/ollama/main.go +++ /dev/null @@ -1,281 +0,0 @@ -package ollama - -import ( - "bufio" - "bytes" - "io" - "net/http" - "time" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/image" - "github.com/labring/sealos/service/aiproxy/common/random" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/common/splitter" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/constant" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - var request relaymodel.GeneralOpenAIRequest - err := common.UnmarshalBodyReusable(req, &request) - if err != nil { - return "", nil, nil, err - } - - ollamaRequest := ChatRequest{ - Model: meta.ActualModel, - Options: &Options{ - Seed: int(request.Seed), - Temperature: request.Temperature, - TopP: request.TopP, - FrequencyPenalty: request.FrequencyPenalty, - PresencePenalty: request.PresencePenalty, - NumPredict: request.MaxTokens, - NumCtx: request.NumCtx, - }, - Stream: request.Stream, - } - for _, message := range request.Messages { - openaiContent := message.ParseContent() - var imageUrls []string - var contentText string - for _, part := range openaiContent { - switch part.Type { - case relaymodel.ContentTypeText: - contentText = part.Text - case relaymodel.ContentTypeImageURL: - _, data, err := image.GetImageFromURL(req.Context(), part.ImageURL.URL) - if err != nil { - return "", nil, nil, err - } - imageUrls = append(imageUrls, data) - } - } - ollamaRequest.Messages = append(ollamaRequest.Messages, Message{ - Role: message.Role, - Content: contentText, - Images: imageUrls, - }) - } - - data, err := sonic.Marshal(ollamaRequest) - if err != nil { - return "", nil, nil, err - } - - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func responseOllama2OpenAI(meta *meta.Meta, response *ChatResponse) *openai.TextResponse { - choice := openai.TextResponseChoice{ - Index: 0, - Message: relaymodel.Message{ - Role: response.Message.Role, - Content: response.Message.Content, - }, - } - if response.Done { - choice.FinishReason = constant.StopFinishReason - } - fullTextResponse := openai.TextResponse{ - ID: "chatcmpl-" + random.GetUUID(), - Model: meta.OriginModel, - Object: "chat.completion", - Created: time.Now().Unix(), - Choices: []*openai.TextResponseChoice{&choice}, - Usage: relaymodel.Usage{ - PromptTokens: response.PromptEvalCount, - CompletionTokens: response.EvalCount, - TotalTokens: response.PromptEvalCount + response.EvalCount, - }, - } - return &fullTextResponse -} - -func streamResponseOllama2OpenAI(meta *meta.Meta, ollamaResponse *ChatResponse) *openai.ChatCompletionsStreamResponse { - var choice openai.ChatCompletionsStreamResponseChoice - choice.Delta.Role = ollamaResponse.Message.Role - choice.Delta.Content = ollamaResponse.Message.Content - if ollamaResponse.Done { - choice.FinishReason = &constant.StopFinishReason - } - response := openai.ChatCompletionsStreamResponse{ - ID: "chatcmpl-" + random.GetUUID(), - Object: "chat.completion.chunk", - Created: time.Now().Unix(), - Model: meta.OriginModel, - Choices: []*openai.ChatCompletionsStreamResponseChoice{&choice}, - } - - if ollamaResponse.EvalCount != 0 { - response.Usage = &relaymodel.Usage{ - PromptTokens: ollamaResponse.PromptEvalCount, - CompletionTokens: ollamaResponse.EvalCount, - TotalTokens: ollamaResponse.PromptEvalCount + ollamaResponse.EvalCount, - } - } - - return &response -} - -func StreamHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHandler(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - var usage *relaymodel.Usage - scanner := bufio.NewScanner(resp.Body) - - common.SetEventStreamHeaders(c) - - var thinkSplitter *splitter.Splitter - if meta.ChannelConfig.SplitThink { - thinkSplitter = splitter.NewThinkSplitter() - } - - for scanner.Scan() { - data := scanner.Bytes() - - var ollamaResponse ChatResponse - err := sonic.Unmarshal(data, &ollamaResponse) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - - response := streamResponseOllama2OpenAI(meta, &ollamaResponse) - if response.Usage != nil { - usage = response.Usage - } - - if meta.ChannelConfig.SplitThink { - openai.StreamSplitThinkModeld(response, thinkSplitter, func(data *openai.ChatCompletionsStreamResponse) { - _ = render.ObjectData(c, data) - }) - continue - } - - _ = render.ObjectData(c, response) - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - render.Done(c) - - return usage, nil -} - -func ConvertEmbeddingRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - request, err := utils.UnmarshalGeneralOpenAIRequest(req) - if err != nil { - return "", nil, nil, err - } - request.Model = meta.ActualModel - data, err := sonic.Marshal(&EmbeddingRequest{ - Model: request.Model, - Input: request.ParseInput(), - Options: &Options{ - Seed: int(request.Seed), - Temperature: request.Temperature, - TopP: request.TopP, - FrequencyPenalty: request.FrequencyPenalty, - PresencePenalty: request.PresencePenalty, - }, - }) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func EmbeddingHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHandler(resp) - } - - defer resp.Body.Close() - - var ollamaResponse EmbeddingResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&ollamaResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - if ollamaResponse.Error != "" { - return nil, openai.ErrorWrapperWithMessage(ollamaResponse.Error, openai.ErrorTypeUpstream, resp.StatusCode) - } - - fullTextResponse := embeddingResponseOllama2OpenAI(meta, &ollamaResponse) - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &fullTextResponse.Usage, nil -} - -func embeddingResponseOllama2OpenAI(meta *meta.Meta, response *EmbeddingResponse) *openai.EmbeddingResponse { - openAIEmbeddingResponse := openai.EmbeddingResponse{ - Object: "list", - Data: make([]*openai.EmbeddingResponseItem, 0, len(response.Embeddings)), - Model: meta.OriginModel, - Usage: relaymodel.Usage{ - PromptTokens: response.PromptEvalCount, - TotalTokens: response.PromptEvalCount, - }, - } - for i, embedding := range response.Embeddings { - openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, &openai.EmbeddingResponseItem{ - Object: "embedding", - Index: i, - Embedding: embedding, - }) - } - return &openAIEmbeddingResponse -} - -func Handler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHandler(resp) - } - - defer resp.Body.Close() - - var ollamaResponse ChatResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&ollamaResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - if ollamaResponse.Error != "" { - return nil, openai.ErrorWrapperWithMessage(ollamaResponse.Error, openai.ErrorTypeUpstream, resp.StatusCode) - } - fullTextResponse := responseOllama2OpenAI(meta, &ollamaResponse) - - if meta.ChannelConfig.SplitThink { - openai.SplitThinkModeld(fullTextResponse) - } - - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return nil, openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return &fullTextResponse.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/ollama/model.go b/service/aiproxy/relay/adaptor/ollama/model.go deleted file mode 100644 index 454af1823..000000000 --- a/service/aiproxy/relay/adaptor/ollama/model.go +++ /dev/null @@ -1,53 +0,0 @@ -package ollama - -type Options struct { - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` - PresencePenalty *float64 `json:"presence_penalty,omitempty"` - Seed int `json:"seed,omitempty"` - TopK int `json:"top_k,omitempty"` - NumPredict int `json:"num_predict,omitempty"` - NumCtx int `json:"num_ctx,omitempty"` -} - -type Message struct { - Role string `json:"role,omitempty"` - Content string `json:"content,omitempty"` - Images []string `json:"images,omitempty"` -} - -type ChatRequest struct { - Options *Options `json:"options,omitempty"` - Model string `json:"model,omitempty"` - Messages []Message `json:"messages,omitempty"` - Stream bool `json:"stream"` - Format any `json:"format,omitempty"` -} - -type ChatResponse struct { - Model string `json:"model,omitempty"` - CreatedAt string `json:"created_at,omitempty"` - Response string `json:"response,omitempty"` - Error string `json:"error,omitempty"` - Message Message `json:"message,omitempty"` - TotalDuration int `json:"total_duration,omitempty"` - LoadDuration int `json:"load_duration,omitempty"` - PromptEvalCount int `json:"prompt_eval_count,omitempty"` - EvalCount int `json:"eval_count,omitempty"` - EvalDuration int `json:"eval_duration,omitempty"` - Done bool `json:"done,omitempty"` -} - -type EmbeddingRequest struct { - Options *Options `json:"options,omitempty"` - Model string `json:"model"` - Input []string `json:"input"` -} - -type EmbeddingResponse struct { - Error string `json:"error,omitempty"` - Model string `json:"model"` - Embeddings [][]float64 `json:"embeddings"` - PromptEvalCount int `json:"prompt_eval_count,omitempty"` -} diff --git a/service/aiproxy/relay/adaptor/openai/adaptor.go b/service/aiproxy/relay/adaptor/openai/adaptor.go deleted file mode 100644 index c2baae3ba..000000000 --- a/service/aiproxy/relay/adaptor/openai/adaptor.go +++ /dev/null @@ -1,192 +0,0 @@ -package openai - -import ( - "bytes" - "errors" - "fmt" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -var _ adaptor.Adaptor = (*Adaptor)(nil) - -type Adaptor struct{} - -const baseURL = "https://api.openai.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - u := meta.Channel.BaseURL - - var path string - switch meta.Mode { - case relaymode.ChatCompletions: - path = "/chat/completions" - case relaymode.Completions: - path = "/completions" - case relaymode.Embeddings: - path = "/embeddings" - case relaymode.Moderations: - path = "/moderations" - case relaymode.ImagesGenerations: - path = "/images/generations" - case relaymode.Edits: - path = "/edits" - case relaymode.AudioSpeech: - path = "/audio/speech" - case relaymode.AudioTranscription: - path = "/audio/transcriptions" - case relaymode.AudioTranslation: - path = "/audio/translations" - case relaymode.Rerank: - path = "/rerank" - default: - return "", fmt.Errorf("unsupported mode: %s", meta.Mode) - } - - return u + path, nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - req.Header.Set("Authorization", "Bearer "+meta.Channel.Key) - return nil -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - return ConvertRequest(meta, req) -} - -func ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - if req == nil { - return "", nil, nil, errors.New("request is nil") - } - switch meta.Mode { - case relaymode.Moderations: - meta.Set(MetaEmbeddingsPatchInputToSlices, true) - return ConvertEmbeddingsRequest(meta, req) - case relaymode.Embeddings, relaymode.Completions: - return ConvertEmbeddingsRequest(meta, req) - case relaymode.ChatCompletions: - return ConvertTextRequest(meta, req, meta.GetBool(DoNotPatchStreamOptionsIncludeUsageMetaKey)) - case relaymode.ImagesGenerations: - return ConvertImageRequest(meta, req) - case relaymode.AudioTranscription, relaymode.AudioTranslation: - return ConvertSTTRequest(meta, req) - case relaymode.AudioSpeech: - return ConvertTTSRequest(meta, req, "") - case relaymode.Rerank: - return ConvertRerankRequest(meta, req) - default: - return "", nil, nil, fmt.Errorf("unsupported mode: %s", meta.Mode) - } -} - -func DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.ImagesGenerations: - usage, err = ImageHandler(meta, c, resp) - case relaymode.AudioTranscription, relaymode.AudioTranslation: - usage, err = STTHandler(meta, c, resp) - case relaymode.AudioSpeech: - usage, err = TTSHandler(meta, c, resp) - case relaymode.Rerank: - usage, err = RerankHandler(meta, c, resp) - case relaymode.Moderations: - usage, err = ModerationsHandler(meta, c, resp) - case relaymode.Embeddings, relaymode.Completions: - fallthrough - case relaymode.ChatCompletions: - if utils.IsStreamResponse(resp) { - usage, err = StreamHandler(meta, c, resp) - } else { - usage, err = Handler(meta, c, resp) - } - default: - return nil, ErrorWrapperWithMessage(fmt.Sprintf("unsupported mode: %s", meta.Mode), "unsupported_mode", http.StatusBadRequest) - } - return -} - -const DoNotPatchStreamOptionsIncludeUsageMetaKey = "do_not_patch_stream_options_include_usage" - -func ConvertTextRequest(meta *meta.Meta, req *http.Request, doNotPatchStreamOptionsIncludeUsage bool) (string, http.Header, io.Reader, error) { - reqMap := make(map[string]any) - err := common.UnmarshalBodyReusable(req, &reqMap) - if err != nil { - return "", nil, nil, err - } - - if !doNotPatchStreamOptionsIncludeUsage { - if err := patchStreamOptions(reqMap); err != nil { - return "", nil, nil, err - } - } - - reqMap["model"] = meta.ActualModel - jsonData, err := sonic.Marshal(reqMap) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} - -func patchStreamOptions(reqMap map[string]any) error { - stream, ok := reqMap["stream"] - if !ok { - return nil - } - - streamBool, ok := stream.(bool) - if !ok { - return errors.New("stream is not a boolean") - } - - if !streamBool { - return nil - } - - streamOptions, ok := reqMap["stream_options"].(map[string]any) - if !ok { - if reqMap["stream_options"] != nil { - return errors.New("stream_options is not a map") - } - reqMap["stream_options"] = map[string]any{ - "include_usage": true, - } - return nil - } - - streamOptions["include_usage"] = true - return nil -} - -const MetaResponseFormat = "response_format" - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - return DoResponse(meta, c, resp) -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "openai" -} diff --git a/service/aiproxy/relay/adaptor/openai/balance.go b/service/aiproxy/relay/adaptor/openai/balance.go deleted file mode 100644 index f92740b33..000000000 --- a/service/aiproxy/relay/adaptor/openai/balance.go +++ /dev/null @@ -1,64 +0,0 @@ -package openai - -import ( - "context" - "net/http" - "time" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.Balancer = (*Adaptor)(nil) - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return GetBalance(channel) -} - -func GetBalance(channel *model.Channel) (float64, error) { - u := channel.BaseURL - if u == "" { - u = baseURL - } - url := u + "/v1/dashboard/billing/subscription" - - req1, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - return 0, err - } - req1.Header.Set("Authorization", "Bearer "+channel.Key) - res1, err := http.DefaultClient.Do(req1) - if err != nil { - return 0, err - } - defer res1.Body.Close() - subscription := SubscriptionResponse{} - err = sonic.ConfigDefault.NewDecoder(res1.Body).Decode(&subscription) - if err != nil { - return 0, err - } - now := time.Now() - startDate := now.Format("2006-01") + "-01" - endDate := now.Format("2006-01-02") - if !subscription.HasPaymentMethod { - startDate = now.AddDate(0, 0, -100).Format("2006-01-02") - } - url = u + "/v1/dashboard/billing/usage?start_date=" + startDate + "&end_date=" + endDate - req2, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - return 0, err - } - req2.Header.Set("Authorization", "Bearer "+channel.Key) - res2, err := http.DefaultClient.Do(req2) - if err != nil { - return 0, err - } - usage := UsageResponse{} - err = sonic.ConfigDefault.NewDecoder(res2.Body).Decode(&usage) - if err != nil { - return 0, err - } - balance := subscription.HardLimitUSD - usage.TotalUsage/100 - return balance, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/constants.go b/service/aiproxy/relay/adaptor/openai/constants.go deleted file mode 100644 index a72ce8e86..000000000 --- a/service/aiproxy/relay/adaptor/openai/constants.go +++ /dev/null @@ -1,223 +0,0 @@ -package openai - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "gpt-3.5-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.022, - OutputPrice: 0.044, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "gpt-3.5-turbo-16k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.022, - OutputPrice: 0.044, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(16384), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "gpt-3.5-turbo-instruct", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "gpt-4", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.22, - OutputPrice: 0.44, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "gpt-4-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.44, - OutputPrice: 0.88, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "gpt-4-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.071, - OutputPrice: 0.213, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "gpt-4o", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.01775, - OutputPrice: 0.071, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "chatgpt-4o-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "gpt-4o-mini", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.001065, - OutputPrice: 0.00426, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "gpt-4-vision-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "o1-mini", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.0213, - OutputPrice: 0.0852, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, - { - Model: "o1-preview", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerOpenAI, - InputPrice: 0.1065, - OutputPrice: 0.426, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, - - { - Model: "text-embedding-ada-002", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-embedding-3-small", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-embedding-3-large", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-curie-001", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-babbage-001", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-ada-001", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-davinci-002", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-davinci-003", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-moderation-latest", - Type: relaymode.Moderations, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-moderation-stable", - Type: relaymode.Moderations, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "text-davinci-edit-001", - Type: relaymode.Edits, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "davinci-002", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "babbage-002", - Type: relaymode.Completions, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "dall-e-2", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "dall-e-3", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "whisper-1", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "tts-1", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "tts-1-1106", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "tts-1-hd", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerOpenAI, - }, - { - Model: "tts-1-hd-1106", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerOpenAI, - }, -} diff --git a/service/aiproxy/relay/adaptor/openai/embeddings.go b/service/aiproxy/relay/adaptor/openai/embeddings.go deleted file mode 100644 index ab720bb22..000000000 --- a/service/aiproxy/relay/adaptor/openai/embeddings.go +++ /dev/null @@ -1,37 +0,0 @@ -package openai - -import ( - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/relay/meta" -) - -const MetaEmbeddingsPatchInputToSlices = "embeddings_input_to_slices" - -//nolint:gocritic -func ConvertEmbeddingsRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - reqMap := make(map[string]any) - err := common.UnmarshalBodyReusable(req, &reqMap) - if err != nil { - return "", nil, nil, err - } - - reqMap["model"] = meta.ActualModel - - if meta.GetBool(MetaEmbeddingsPatchInputToSlices) { - switch v := reqMap["input"].(type) { - case string: - reqMap["input"] = []string{v} - } - } - - jsonData, err := sonic.Marshal(reqMap) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} diff --git a/service/aiproxy/relay/adaptor/openai/error.go b/service/aiproxy/relay/adaptor/openai/error.go deleted file mode 100644 index f57709d75..000000000 --- a/service/aiproxy/relay/adaptor/openai/error.go +++ /dev/null @@ -1,115 +0,0 @@ -package openai - -import ( - "fmt" - "io" - "net/http" - "strconv" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type GeneralErrorResponse struct { - Error model.Error `json:"error"` - Message string `json:"message"` - Msg string `json:"msg"` - Err string `json:"err"` - ErrorMsg string `json:"error_msg"` - Header struct { - Message string `json:"message"` - } `json:"header"` - Response struct { - Error struct { - Message string `json:"message"` - } `json:"error"` - } `json:"response"` -} - -func (e GeneralErrorResponse) ToMessage() string { - if e.Error.Message != "" { - return e.Error.Message - } - if e.Message != "" { - return e.Message - } - if e.Msg != "" { - return e.Msg - } - if e.Err != "" { - return e.Err - } - if e.ErrorMsg != "" { - return e.ErrorMsg - } - if e.Header.Message != "" { - return e.Header.Message - } - if e.Response.Error.Message != "" { - return e.Response.Error.Message - } - return "" -} - -const ( - ErrorTypeAIProxy = middleware.ErrorTypeAIPROXY - ErrorTypeUpstream = "upstream_error" - ErrorCodeBadResponse = "bad_response" -) - -func ErrorHanlder(resp *http.Response) *model.ErrorWithStatusCode { - defer resp.Body.Close() - - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return &model.ErrorWithStatusCode{ - StatusCode: resp.StatusCode, - Error: model.Error{ - Message: err.Error(), - Type: ErrorTypeUpstream, - Code: ErrorCodeBadResponse, - }, - } - } - - ErrorWithStatusCode := &model.ErrorWithStatusCode{ - StatusCode: resp.StatusCode, - Error: model.Error{ - Type: ErrorTypeUpstream, - Code: ErrorCodeBadResponse, - Param: strconv.Itoa(resp.StatusCode), - }, - } - - var errResponse GeneralErrorResponse - err = sonic.Unmarshal(respBody, &errResponse) - if err != nil { - var errsResp []GeneralErrorResponse - err = sonic.Unmarshal(respBody, &errsResp) - if err != nil { - ErrorWithStatusCode.Error.Message = conv.BytesToString(respBody) - return ErrorWithStatusCode - } - if len(errsResp) > 0 { - errResponse = errsResp[0] - } - } - - if errResponse.Error.Message != "" { - // OpenAI format error, so we override the default one - ErrorWithStatusCode.Error = errResponse.Error - } else { - ErrorWithStatusCode.Error.Message = errResponse.ToMessage() - } - if ErrorWithStatusCode.Error.Message == "" { - ErrorWithStatusCode.Error.Message = fmt.Sprintf("bad response status code %d", resp.StatusCode) - } - - if code, ok := ErrorWithStatusCode.Error.Code.(int64); ok && code >= 400 && code < 600 { - ErrorWithStatusCode.StatusCode = int(code) - } - - return ErrorWithStatusCode -} diff --git a/service/aiproxy/relay/adaptor/openai/helper.go b/service/aiproxy/relay/adaptor/openai/helper.go deleted file mode 100644 index 1c4b1f5e4..000000000 --- a/service/aiproxy/relay/adaptor/openai/helper.go +++ /dev/null @@ -1,26 +0,0 @@ -package openai - -import ( - "fmt" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func ResponseText2Usage(responseText string, modeName string, promptTokens int) *model.Usage { - usage := &model.Usage{ - PromptTokens: promptTokens, - CompletionTokens: CountTokenText(responseText, modeName), - } - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - return usage -} - -func GetFullRequestURL(baseURL string, requestURL string) string { - fullRequestURL := fmt.Sprintf("%s%s", baseURL, requestURL) - - if strings.HasPrefix(baseURL, "https://gateway.ai.cloudflare.com") { - fullRequestURL = fmt.Sprintf("%s%s", baseURL, strings.TrimPrefix(requestURL, "/openai/deployments")) - } - return fullRequestURL -} diff --git a/service/aiproxy/relay/adaptor/openai/image.go b/service/aiproxy/relay/adaptor/openai/image.go deleted file mode 100644 index 3292de249..000000000 --- a/service/aiproxy/relay/adaptor/openai/image.go +++ /dev/null @@ -1,91 +0,0 @@ -package openai - -import ( - "bytes" - "errors" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/bytedance/sonic/ast" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/image" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func ConvertImageRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - node, err := common.UnmarshalBody2Node(req) - if err != nil { - return "", nil, nil, err - } - responseFormat, err := node.Get("response_format").String() - if err != nil && !errors.Is(err, ast.ErrNotExist) { - return "", nil, nil, err - } - meta.Set(MetaResponseFormat, responseFormat) - - _, err = node.Set("model", ast.NewString(meta.ActualModel)) - if err != nil { - return "", nil, nil, err - } - - jsonData, err := node.MarshalJSON() - if err != nil { - return "", nil, nil, err - } - - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} - -func ImageHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseFormat := meta.GetString(MetaResponseFormat) - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - var imageResponse ImageResponse - err = sonic.Unmarshal(responseBody, &imageResponse) - if err != nil { - return nil, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - usage := &model.Usage{ - PromptTokens: len(imageResponse.Data), - TotalTokens: len(imageResponse.Data), - } - - if responseFormat == "b64_json" { - for _, data := range imageResponse.Data { - if len(data.B64Json) > 0 { - continue - } - _, data.B64Json, err = image.GetImageFromURL(c.Request.Context(), data.URL) - if err != nil { - return usage, ErrorWrapper(err, "get_image_from_url_failed", http.StatusInternalServerError) - } - } - } - - data, err := sonic.Marshal(imageResponse) - if err != nil { - return usage, ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - - _, err = c.Writer.Write(data) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return usage, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/main.go b/service/aiproxy/relay/adaptor/openai/main.go deleted file mode 100644 index bac32fae1..000000000 --- a/service/aiproxy/relay/adaptor/openai/main.go +++ /dev/null @@ -1,444 +0,0 @@ -package openai - -import ( - "bufio" - "bytes" - "errors" - "io" - "net/http" - "slices" - "strings" - "sync" - - "github.com/bytedance/sonic" - "github.com/bytedance/sonic/ast" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/common/render" - "github.com/labring/sealos/service/aiproxy/common/splitter" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -const ( - DataPrefix = "data:" - Done = "[DONE]" - DataPrefixLength = len(DataPrefix) -) - -var ( - DataPrefixBytes = conv.StringToBytes(DataPrefix) - DoneBytes = conv.StringToBytes(Done) -) - -type UsageAndChoicesResponse struct { - Usage *model.Usage - Choices []*ChatCompletionsStreamResponseChoice -} - -const scannerBufferSize = 2 * bufio.MaxScanTokenSize - -var scannerBufferPool = sync.Pool{ - New: func() any { - buf := make([]byte, scannerBufferSize) - return &buf - }, -} - -//nolint:forcetypeassert -func getScannerBuffer() *[]byte { - return scannerBufferPool.Get().(*[]byte) -} - -func putScannerBuffer(buf *[]byte) { - if cap(*buf) != scannerBufferSize { - return - } - scannerBufferPool.Put(buf) -} - -func GetUsageAndChoicesResponseFromNode(node *ast.Node) (*UsageAndChoicesResponse, error) { - var usage *model.Usage - usageNode, err := node.Get("usage").Raw() - if err != nil { - if !errors.Is(err, ast.ErrNotExist) { - return nil, err - } - } else { - err = sonic.UnmarshalString(usageNode, &usage) - if err != nil { - return nil, err - } - } - - var choices []*ChatCompletionsStreamResponseChoice - choicesNode, err := node.Get("choices").Raw() - if err != nil { - if !errors.Is(err, ast.ErrNotExist) { - return nil, err - } - } else { - err = sonic.UnmarshalString(choicesNode, &choices) - if err != nil { - return nil, err - } - } - return &UsageAndChoicesResponse{ - Usage: usage, - Choices: choices, - }, nil -} - -func StreamHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseText := strings.Builder{} - - scanner := bufio.NewScanner(resp.Body) - buf := getScannerBuffer() - defer putScannerBuffer(buf) - scanner.Buffer(*buf, cap(*buf)) - - var usage *model.Usage - - common.SetEventStreamHeaders(c) - - hasReasoningContent := false - var thinkSplitter *splitter.Splitter - if meta.ChannelConfig.SplitThink { - thinkSplitter = splitter.NewThinkSplitter() - } - - for scanner.Scan() { - data := scanner.Bytes() - if len(data) < DataPrefixLength { // ignore blank line or wrong format - continue - } - if !slices.Equal(data[:DataPrefixLength], DataPrefixBytes) { - continue - } - data = bytes.TrimSpace(data[DataPrefixLength:]) - if slices.Equal(data, DoneBytes) { - break - } - - switch meta.Mode { - case relaymode.ChatCompletions: - node, err := sonic.Get(data) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - streamResponse, err := GetUsageAndChoicesResponseFromNode(&node) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - if streamResponse.Usage != nil { - usage = streamResponse.Usage - responseText.Reset() - } - for _, choice := range streamResponse.Choices { - if usage == nil { - responseText.WriteString(choice.Delta.StringContent()) - } - if choice.Delta.ReasoningContent != "" { - hasReasoningContent = true - } - } - - _, err = node.Set("model", ast.NewString(meta.OriginModel)) - if err != nil { - log.Error("error set model: " + err.Error()) - } - - if meta.ChannelConfig.SplitThink && !hasReasoningContent { - respMap, err := node.Map() - if err != nil { - log.Error("error get node map: " + err.Error()) - continue - } - StreamSplitThink(respMap, thinkSplitter, func(data map[string]any) { - _ = render.ObjectData(c, data) - }) - continue - } - - _ = render.ObjectData(c, &node) - case relaymode.Completions: - node, err := sonic.Get(data) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - streamResponse, err := GetUsageAndChoicesResponseFromNode(&node) - if err != nil { - log.Error("error unmarshalling stream response: " + err.Error()) - continue - } - if streamResponse.Usage != nil { - usage = streamResponse.Usage - responseText.Reset() - } else { - for _, choice := range streamResponse.Choices { - responseText.WriteString(choice.Text) - } - } - _, err = node.Set("model", ast.NewString(meta.OriginModel)) - if err != nil { - log.Error("error set model: " + err.Error()) - } - _ = render.ObjectData(c, &node) - } - } - - if err := scanner.Err(); err != nil { - log.Error("error reading stream: " + err.Error()) - } - - render.Done(c) - - if usage == nil || (usage.TotalTokens == 0 && responseText.Len() > 0) { - usage = ResponseText2Usage(responseText.String(), meta.ActualModel, meta.InputTokens) - } - - if usage.TotalTokens != 0 && usage.PromptTokens == 0 { // some channels don't return prompt tokens & completion tokens - usage.PromptTokens = meta.InputTokens - usage.CompletionTokens = usage.TotalTokens - meta.InputTokens - } - - return usage, nil -} - -// renderCallback maybe reuse data, so don't modify data -func StreamSplitThink(data map[string]any, thinkSplitter *splitter.Splitter, renderCallback func(data map[string]any)) { - choices, ok := data["choices"].([]any) - // only support one choice - if !ok || len(choices) != 1 { - renderCallback(data) - return - } - choice := choices[0] - choiceMap, ok := choice.(map[string]any) - if !ok { - renderCallback(data) - return - } - delta, ok := choiceMap["delta"].(map[string]any) - if !ok { - renderCallback(data) - return - } - content, ok := delta["content"].(string) - if !ok { - renderCallback(data) - return - } - think, remaining := thinkSplitter.Process(conv.StringToBytes(content)) - if len(think) == 0 && len(remaining) == 0 { - delta["content"] = "" - delete(delta, "reasoning_content") - renderCallback(data) - return - } - if len(think) > 0 { - delta["content"] = "" - delta["reasoning_content"] = conv.BytesToString(think) - renderCallback(data) - } - if len(remaining) > 0 { - delta["content"] = conv.BytesToString(remaining) - delete(delta, "reasoning_content") - renderCallback(data) - } -} - -func StreamSplitThinkModeld(data *ChatCompletionsStreamResponse, thinkSplitter *splitter.Splitter, renderCallback func(data *ChatCompletionsStreamResponse)) { - choices := data.Choices - // only support one choice - if len(data.Choices) != 1 { - renderCallback(data) - return - } - choice := choices[0] - content, ok := choice.Delta.Content.(string) - if !ok { - renderCallback(data) - return - } - think, remaining := thinkSplitter.Process(conv.StringToBytes(content)) - if len(think) == 0 && len(remaining) == 0 { - choice.Delta.Content = "" - choice.Delta.ReasoningContent = "" - renderCallback(data) - return - } - if len(think) > 0 { - choice.Delta.Content = "" - choice.Delta.ReasoningContent = conv.BytesToString(think) - renderCallback(data) - } - if len(remaining) > 0 { - choice.Delta.Content = conv.BytesToString(remaining) - choice.Delta.ReasoningContent = "" - renderCallback(data) - } -} - -func SplitThink(data map[string]any) { - choices, ok := data["choices"].([]any) - if !ok { - return - } - for _, choice := range choices { - choiceMap, ok := choice.(map[string]any) - if !ok { - continue - } - message, ok := choiceMap["message"].(map[string]any) - if !ok { - continue - } - content, ok := message["content"].(string) - if !ok { - continue - } - think, remaining := splitter.NewThinkSplitter().Process(conv.StringToBytes(content)) - message["reasoning_content"] = conv.BytesToString(think) - message["content"] = conv.BytesToString(remaining) - } -} - -func SplitThinkModeld(data *TextResponse) { - choices := data.Choices - for _, choice := range choices { - content, ok := choice.Message.Content.(string) - if !ok { - continue - } - think, remaining := splitter.NewThinkSplitter().Process(conv.StringToBytes(content)) - choice.Message.ReasoningContent = conv.BytesToString(think) - choice.Message.Content = conv.BytesToString(remaining) - } -} - -func GetSlimTextResponseFromNode(node *ast.Node) (*SlimTextResponse, error) { - var e model.Error - errorNode, err := node.Get("error").Raw() - if err != nil { - if !errors.Is(err, ast.ErrNotExist) { - return nil, err - } - } else { - err = sonic.UnmarshalString(errorNode, &e) - if err != nil { - return nil, err - } - } - var choices []*TextResponseChoice - choicesNode, err := node.Get("choices").Raw() - if err != nil { - if !errors.Is(err, ast.ErrNotExist) { - return nil, err - } - } else { - err = sonic.UnmarshalString(choicesNode, &choices) - if err != nil { - return nil, err - } - } - var usage model.Usage - usageNode, err := node.Get("usage").Raw() - if err != nil { - if !errors.Is(err, ast.ErrNotExist) { - return nil, err - } - } else { - err = sonic.UnmarshalString(usageNode, &usage) - if err != nil { - return nil, err - } - } - return &SlimTextResponse{ - Error: e, - Choices: choices, - Usage: usage, - }, nil -} - -func Handler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - - node, err := sonic.Get(responseBody) - if err != nil { - return nil, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - textResponse, err := GetSlimTextResponseFromNode(&node) - if err != nil { - return nil, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - if textResponse.Error.Type != "" { - return nil, ErrorWrapperWithMessage(textResponse.Error.Message, textResponse.Error.Code, http.StatusBadRequest) - } - - if textResponse.Usage.TotalTokens == 0 || (textResponse.Usage.PromptTokens == 0 && textResponse.Usage.CompletionTokens == 0) { - completionTokens := 0 - for _, choice := range textResponse.Choices { - if choice.Text != "" { - completionTokens += CountTokenText(choice.Text, meta.ActualModel) - continue - } - completionTokens += CountTokenText(choice.Message.StringContent(), meta.ActualModel) - } - textResponse.Usage = model.Usage{ - PromptTokens: meta.InputTokens, - CompletionTokens: completionTokens, - } - } - textResponse.Usage.TotalTokens = textResponse.Usage.PromptTokens + textResponse.Usage.CompletionTokens - - _, err = node.Set("model", ast.NewString(meta.OriginModel)) - if err != nil { - return &textResponse.Usage, ErrorWrapper(err, "set_model_failed", http.StatusInternalServerError) - } - - if meta.ChannelConfig.SplitThink { - respMap, err := node.Map() - if err != nil { - return &textResponse.Usage, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - SplitThink(respMap) - } - - newData, err := sonic.Marshal(&node) - if err != nil { - return &textResponse.Usage, ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - - _, err = c.Writer.Write(newData) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return &textResponse.Usage, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/model.go b/service/aiproxy/relay/adaptor/openai/model.go deleted file mode 100644 index 906a5c0ad..000000000 --- a/service/aiproxy/relay/adaptor/openai/model.go +++ /dev/null @@ -1,155 +0,0 @@ -package openai - -import ( - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type TextContent struct { - Type string `json:"type,omitempty"` - Text string `json:"text,omitempty"` -} - -type ImageContent struct { - ImageURL *model.ImageURL `json:"image_url,omitempty"` - Type string `json:"type,omitempty"` -} - -type ChatRequest struct { - Model string `json:"model"` - Messages []*model.Message `json:"messages"` - MaxTokens int `json:"max_tokens"` -} - -type TextRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - Messages []*model.Message `json:"messages"` - MaxTokens int `json:"max_tokens"` -} - -// ImageRequest docs: https://platform.openai.com/docs/api-reference/images/create -type ImageRequest struct { - Model string `json:"model"` - Prompt string `binding:"required" json:"prompt"` - Size string `json:"size,omitempty"` - Quality string `json:"quality,omitempty"` - ResponseFormat string `json:"response_format,omitempty"` - Style string `json:"style,omitempty"` - User string `json:"user,omitempty"` - N int `json:"n,omitempty"` -} - -type WhisperJSONResponse struct { - Text string `json:"text,omitempty"` -} - -type WhisperVerboseJSONResponse struct { - Task string `json:"task,omitempty"` - Language string `json:"language,omitempty"` - Text string `json:"text,omitempty"` - Segments []*Segment `json:"segments,omitempty"` - Duration float64 `json:"duration,omitempty"` -} - -type Segment struct { - Text string `json:"text"` - Tokens []int `json:"tokens"` - ID int `json:"id"` - Seek int `json:"seek"` - Start float64 `json:"start"` - End float64 `json:"end"` - Temperature float64 `json:"temperature"` - AvgLogprob float64 `json:"avg_logprob"` - CompressionRatio float64 `json:"compression_ratio"` - NoSpeechProb float64 `json:"no_speech_prob"` -} - -type UsageOrResponseText struct { - *model.Usage - ResponseText string -} - -type SlimTextResponse struct { - Error model.Error `json:"error"` - Choices []*TextResponseChoice `json:"choices"` - Usage model.Usage `json:"usage"` -} - -type SlimRerankResponse struct { - Meta model.RerankMeta `json:"meta"` -} - -type TextResponseChoice struct { - FinishReason string `json:"finish_reason"` - Message model.Message `json:"message"` - Index int `json:"index"` - Text string `json:"text"` -} - -type TextResponse struct { - ID string `json:"id"` - Model string `json:"model,omitempty"` - Object string `json:"object"` - Choices []*TextResponseChoice `json:"choices"` - model.Usage `json:"usage"` - Created int64 `json:"created"` -} - -type EmbeddingResponseItem struct { - Object string `json:"object"` - Embedding []float64 `json:"embedding"` - Index int `json:"index"` -} - -type EmbeddingResponse struct { - Object string `json:"object"` - Model string `json:"model"` - Data []*EmbeddingResponseItem `json:"data"` - model.Usage `json:"usage"` -} - -type ImageData struct { - URL string `json:"url,omitempty"` - B64Json string `json:"b64_json,omitempty"` - RevisedPrompt string `json:"revised_prompt,omitempty"` -} - -type ImageResponse struct { - Data []*ImageData `json:"data"` - Created int64 `json:"created"` -} - -type ChatCompletionsStreamResponseChoice struct { - FinishReason *string `json:"finish_reason,omitempty"` - Delta model.Message `json:"delta"` - Index int `json:"index"` - Text string `json:"text,omitempty"` -} - -type ChatCompletionsStreamResponse struct { - Usage *model.Usage `json:"usage,omitempty"` - ID string `json:"id"` - Object string `json:"object"` - Model string `json:"model"` - Choices []*ChatCompletionsStreamResponseChoice `json:"choices"` - Created int64 `json:"created"` -} - -type SubscriptionResponse struct { - Object string `json:"object"` - HasPaymentMethod bool `json:"has_payment_method"` - SoftLimitUSD float64 `json:"soft_limit_usd"` - HardLimitUSD float64 `json:"hard_limit_usd"` - SystemHardLimitUSD float64 `json:"system_hard_limit_usd"` - AccessUntil int64 `json:"access_until"` -} - -type UsageResponse struct { - Object string `json:"object"` - // DailyCosts []OpenAIUsageDailyCost `json:"daily_costs"` - TotalUsage float64 `json:"total_usage"` // unit: 0.01 dollar -} - -type ErrorResp struct { - Error model.Error `json:"error"` -} diff --git a/service/aiproxy/relay/adaptor/openai/moderations.go b/service/aiproxy/relay/adaptor/openai/moderations.go deleted file mode 100644 index 29fe98933..000000000 --- a/service/aiproxy/relay/adaptor/openai/moderations.go +++ /dev/null @@ -1,62 +0,0 @@ -package openai - -import ( - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func ModerationsHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - - var respMap map[string]any - err = sonic.Unmarshal(body, &respMap) - if err != nil { - return nil, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - if _, ok := respMap["error"]; ok { - var errorResp ErrorResp - err = sonic.Unmarshal(body, &errorResp) - if err != nil { - return nil, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - return nil, ErrorWrapperWithMessage(errorResp.Error.Message, errorResp.Error.Code, http.StatusBadRequest) - } - - if _, ok := respMap["model"]; ok && meta.OriginModel != "" { - respMap["model"] = meta.OriginModel - } - - usage := &model.Usage{ - PromptTokens: meta.InputTokens, - TotalTokens: meta.InputTokens, - } - - newData, err := sonic.Marshal(respMap) - if err != nil { - return usage, ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError) - } - - _, err = c.Writer.Write(newData) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return usage, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/rerank.go b/service/aiproxy/relay/adaptor/openai/rerank.go deleted file mode 100644 index 1bdd644c4..000000000 --- a/service/aiproxy/relay/adaptor/openai/rerank.go +++ /dev/null @@ -1,76 +0,0 @@ -package openai - -import ( - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/bytedance/sonic/ast" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func ConvertRerankRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - node, err := common.UnmarshalBody2Node(req) - if err != nil { - return "", nil, nil, err - } - - _, err = node.Set("model", ast.NewString(meta.ActualModel)) - if err != nil { - return "", nil, nil, err - } - - jsonData, err := node.MarshalJSON() - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} - -func RerankHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - var rerankResponse SlimRerankResponse - err = sonic.Unmarshal(responseBody, &rerankResponse) - if err != nil { - return nil, ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError) - } - - c.Writer.WriteHeader(resp.StatusCode) - - _, err = c.Writer.Write(responseBody) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - - if rerankResponse.Meta.Tokens == nil { - return &model.Usage{ - PromptTokens: meta.InputTokens, - CompletionTokens: 0, - TotalTokens: meta.InputTokens, - }, nil - } - if rerankResponse.Meta.Tokens.InputTokens <= 0 { - rerankResponse.Meta.Tokens.InputTokens = meta.InputTokens - } - return &model.Usage{ - PromptTokens: rerankResponse.Meta.Tokens.InputTokens, - CompletionTokens: rerankResponse.Meta.Tokens.OutputTokens, - TotalTokens: rerankResponse.Meta.Tokens.InputTokens + rerankResponse.Meta.Tokens.OutputTokens, - }, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/stt.go b/service/aiproxy/relay/adaptor/openai/stt.go deleted file mode 100644 index 3c779386b..000000000 --- a/service/aiproxy/relay/adaptor/openai/stt.go +++ /dev/null @@ -1,185 +0,0 @@ -package openai - -import ( - "bufio" - "bytes" - "fmt" - "io" - "mime/multipart" - "net/http" - "strings" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func ConvertSTTRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) { - err := request.ParseMultipartForm(1024 * 1024 * 4) - if err != nil { - return "", nil, nil, err - } - - multipartBody := &bytes.Buffer{} - multipartWriter := multipart.NewWriter(multipartBody) - - for key, values := range request.MultipartForm.Value { - if len(values) == 0 { - continue - } - value := values[0] - if key == "model" { - err = multipartWriter.WriteField(key, meta.ActualModel) - if err != nil { - return "", nil, nil, err - } - continue - } - if key == "response_format" { - meta.Set(MetaResponseFormat, value) - continue - } - err = multipartWriter.WriteField(key, value) - if err != nil { - return "", nil, nil, err - } - } - - for key, files := range request.MultipartForm.File { - if len(files) == 0 { - continue - } - fileHeader := files[0] - file, err := fileHeader.Open() - if err != nil { - return "", nil, nil, err - } - w, err := multipartWriter.CreateFormFile(key, fileHeader.Filename) - if err != nil { - file.Close() - return "", nil, nil, err - } - _, err = io.Copy(w, file) - file.Close() - if err != nil { - return "", nil, nil, err - } - } - - multipartWriter.Close() - ContentType := multipartWriter.FormDataContentType() - return http.MethodPost, http.Header{ - "Content-Type": {ContentType}, - }, multipartBody, nil -} - -func STTHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*model.Usage, *model.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - responseFormat := meta.GetString(MetaResponseFormat) - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - return nil, ErrorWrapper(err, "read_response_body_failed", http.StatusInternalServerError) - } - - var openAIErr SlimTextResponse - if err = sonic.Unmarshal(responseBody, &openAIErr); err == nil { - if openAIErr.Error.Message != "" { - return nil, ErrorWrapper(fmt.Errorf("type %s, code %v, message %s", openAIErr.Error.Type, openAIErr.Error.Code, openAIErr.Error.Message), "request_error", http.StatusInternalServerError) - } - } - - var text string - switch responseFormat { - case "text": - text = getTextFromText(responseBody) - case "srt": - text, err = getTextFromSRT(responseBody) - case "verbose_json": - text, err = getTextFromVerboseJSON(responseBody) - case "vtt": - text, err = getTextFromVTT(responseBody) - case "json": - fallthrough - default: - text, err = getTextFromJSON(responseBody) - } - if err != nil { - return nil, ErrorWrapper(err, "get_text_from_body_err", http.StatusInternalServerError) - } - var promptTokens int - if meta.InputTokens > 0 { - promptTokens = meta.InputTokens - } else { - promptTokens = CountTokenText(text, meta.ActualModel) - } - - for k, v := range resp.Header { - c.Writer.Header().Set(k, v[0]) - } - _, err = c.Writer.Write(responseBody) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - - return &model.Usage{ - PromptTokens: promptTokens, - CompletionTokens: 0, - TotalTokens: promptTokens, - }, nil -} - -func getTextFromVTT(body []byte) (string, error) { - return getTextFromSRT(body) -} - -func getTextFromVerboseJSON(body []byte) (string, error) { - var whisperResponse WhisperVerboseJSONResponse - if err := sonic.Unmarshal(body, &whisperResponse); err != nil { - return "", fmt.Errorf("unmarshal_response_body_failed err :%w", err) - } - return whisperResponse.Text, nil -} - -func getTextFromSRT(body []byte) (string, error) { - scanner := bufio.NewScanner(bytes.NewReader(body)) - var builder strings.Builder - var textLine bool - for scanner.Scan() { - line := scanner.Text() - if textLine { - builder.WriteString(line) - textLine = false - continue - } else if strings.Contains(line, "-->") { - textLine = true - continue - } - } - if err := scanner.Err(); err != nil { - return "", err - } - return builder.String(), nil -} - -func getTextFromText(body []byte) string { - return strings.TrimSuffix(conv.BytesToString(body), "\n") -} - -func getTextFromJSON(body []byte) (string, error) { - var whisperResponse WhisperJSONResponse - if err := sonic.Unmarshal(body, &whisperResponse); err != nil { - return "", fmt.Errorf("unmarshal_response_body_failed err :%w", err) - } - return whisperResponse.Text, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/token.go b/service/aiproxy/relay/adaptor/openai/token.go deleted file mode 100644 index 60ece2172..000000000 --- a/service/aiproxy/relay/adaptor/openai/token.go +++ /dev/null @@ -1,240 +0,0 @@ -package openai - -import ( - "errors" - "math" - "strings" - "sync" - "unicode/utf8" - - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/image" - "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/pkoukk/tiktoken-go" - log "github.com/sirupsen/logrus" -) - -// tokenEncoderMap won't grow after initialization -var ( - tokenEncoderMap = map[string]*tiktoken.Tiktoken{} - defaultTokenEncoder *tiktoken.Tiktoken - tokenEncoderLock sync.RWMutex -) - -func init() { - gpt35TokenEncoder, err := tiktoken.EncodingForModel("gpt-3.5-turbo") - if err != nil { - log.Fatal("failed to get gpt-3.5-turbo token encoder: " + err.Error()) - } - defaultTokenEncoder = gpt35TokenEncoder -} - -func getTokenEncoder(model string) *tiktoken.Tiktoken { - tokenEncoderLock.RLock() - tokenEncoder, ok := tokenEncoderMap[model] - tokenEncoderLock.RUnlock() - if ok { - return tokenEncoder - } - - tokenEncoderLock.Lock() - defer tokenEncoderLock.Unlock() - if tokenEncoder, ok := tokenEncoderMap[model]; ok { - return tokenEncoder - } - - tokenEncoder, err := tiktoken.EncodingForModel(model) - if err != nil { - log.Warnf("failed to get token encoder for model %s: %v, using encoder for gpt-3.5-turbo", model, err) - tokenEncoder = defaultTokenEncoder - } - tokenEncoderMap[model] = tokenEncoder - return tokenEncoder -} - -func getTokenNum(tokenEncoder *tiktoken.Tiktoken, text string) int { - return len(tokenEncoder.Encode(text, nil, nil)) -} - -func CountTokenMessages(messages []*model.Message, model string) int { - if !config.GetBillingEnabled() { - return 0 - } - tokenEncoder := getTokenEncoder(model) - // Reference: - // https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb - // https://github.com/pkoukk/tiktoken-go/issues/6 - // - // Every message follows <|start|>{role/name}\n{content}<|end|>\n - var tokensPerMessage int - var tokensPerName int - if model == "gpt-3.5-turbo-0301" { - tokensPerMessage = 4 - tokensPerName = -1 // If there's a name, the role is omitted - } else { - tokensPerMessage = 3 - tokensPerName = 1 - } - tokenNum := 0 - for _, message := range messages { - tokenNum += tokensPerMessage - switch v := message.Content.(type) { - case string: - tokenNum += getTokenNum(tokenEncoder, v) - case []any: - for _, it := range v { - m, ok := it.(map[string]any) - if !ok { - continue - } - switch m["type"] { - case "text": - if textValue, ok := m["text"]; ok { - if textString, ok := textValue.(string); ok { - tokenNum += getTokenNum(tokenEncoder, textString) - } - } - case "image_url": - imageURL, ok := m["image_url"].(map[string]any) - if ok { - url, ok := imageURL["url"].(string) - if !ok { - continue - } - detail := "" - if imageURL["detail"] != nil { - detail, ok = imageURL["detail"].(string) - if !ok { - continue - } - } - imageTokens, err := countImageTokens(url, detail, model) - if err != nil { - log.Error("error counting image tokens: " + err.Error()) - } else { - tokenNum += imageTokens - } - } - } - } - } - tokenNum += getTokenNum(tokenEncoder, message.Role) - if message.Name != nil { - tokenNum += tokensPerName - tokenNum += getTokenNum(tokenEncoder, *message.Name) - } - } - tokenNum += 3 // Every reply is primed with <|start|>assistant<|message|> - return tokenNum -} - -const ( - lowDetailCost = 85 - highDetailCostPerTile = 170 - additionalCost = 85 - // gpt-4o-mini cost higher than other model - gpt4oMiniLowDetailCost = 2833 - gpt4oMiniHighDetailCost = 5667 - gpt4oMiniAdditionalCost = 2833 -) - -// https://platform.openai.com/docs/guides/vision/calculating-costs -// https://github.com/openai/openai-cookbook/blob/05e3f9be4c7a2ae7ecf029a7c32065b024730ebe/examples/How_to_count_tokens_with_tiktoken.ipynb -func countImageTokens(url string, detail string, model string) (_ int, err error) { - fetchSize := true - var width, height int - // Reference: https://platform.openai.com/docs/guides/vision/low-or-high-fidelity-image-understanding - // detail == "auto" is undocumented on how it works, it just said the model will use the auto setting which will look at the image input size and decide if it should use the low or high setting. - // According to the official guide, "low" disable the high-res model, - // and only receive low-res 512px x 512px version of the image, indicating - // that image is treated as low-res when size is smaller than 512px x 512px, - // then we can assume that image size larger than 512px x 512px is treated - // as high-res. Then we have the following logic: - // if detail == "" || detail == "auto" { - // width, height, err = image.GetImageSize(url) - // if err != nil { - // return 0, err - // } - // fetchSize = false - // // not sure if this is correct - // if width > 512 || height > 512 { - // detail = "high" - // } else { - // detail = "low" - // } - // } - - // However, in my test, it seems to be always the same as "high". - // The following image, which is 125x50, is still treated as high-res, taken - // 255 tokens in the response of non-stream chat completion api. - // https://upload.wikimedia.org/wikipedia/commons/1/10/18_Infantry_Division_Messina.jpg - if detail == "" || detail == "auto" { - // assume by test, not sure if this is correct - detail = "high" - } - switch detail { - case "low": - if strings.HasPrefix(model, "gpt-4o-mini") { - return gpt4oMiniLowDetailCost, nil - } - return lowDetailCost, nil - case "high": - if fetchSize { - width, height, err = image.GetImageSize(url) - if err != nil { - return 0, err - } - } - if width > 2048 || height > 2048 { // max(width, height) > 2048 - ratio := float64(2048) / math.Max(float64(width), float64(height)) - width = int(float64(width) * ratio) - height = int(float64(height) * ratio) - } - if width > 768 && height > 768 { // min(width, height) > 768 - ratio := float64(768) / math.Min(float64(width), float64(height)) - width = int(float64(width) * ratio) - height = int(float64(height) * ratio) - } - numSquares := int(math.Ceil(float64(width)/512) * math.Ceil(float64(height)/512)) - if strings.HasPrefix(model, "gpt-4o-mini") { - return numSquares*gpt4oMiniHighDetailCost + gpt4oMiniAdditionalCost, nil - } - result := numSquares*highDetailCostPerTile + additionalCost - return result, nil - default: - return 0, errors.New("invalid detail option") - } -} - -func CountTokenInput(input any, model string) int { - if !config.GetBillingEnabled() { - return 0 - } - switch v := input.(type) { - case string: - return CountTokenText(v, model) - case []any: - num := 0 - for _, s := range v { - num += CountTokenInput(s, model) - } - return num - case []string: - text := "" - for _, s := range v { - text += s - } - return CountTokenText(text, model) - } - return 0 -} - -func CountTokenText(text string, model string) int { - if !config.GetBillingEnabled() { - return 0 - } - if strings.HasPrefix(model, "tts") { - return utf8.RuneCountInString(text) - } - return getTokenNum(getTokenEncoder(model), text) -} diff --git a/service/aiproxy/relay/adaptor/openai/tts.go b/service/aiproxy/relay/adaptor/openai/tts.go deleted file mode 100644 index 54264d877..000000000 --- a/service/aiproxy/relay/adaptor/openai/tts.go +++ /dev/null @@ -1,79 +0,0 @@ -package openai - -import ( - "bytes" - "errors" - "io" - "net/http" - - "github.com/bytedance/sonic/ast" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func ConvertTTSRequest(meta *meta.Meta, req *http.Request, defaultVoice string) (string, http.Header, io.Reader, error) { - node, err := common.UnmarshalBody2Node(req) - if err != nil { - return "", nil, nil, err - } - - input, err := node.Get("input").String() - if err != nil { - if errors.Is(err, ast.ErrNotExist) { - return "", nil, nil, errors.New("input is required") - } - return "", nil, nil, err - } - if len(input) > 4096 { - return "", nil, nil, errors.New("input is too long (over 4096 characters)") - } - - voice, err := node.Get("voice").String() - if err != nil && !errors.Is(err, ast.ErrNotExist) { - return "", nil, nil, err - } - if voice == "" && defaultVoice != "" { - _, err = node.Set("voice", ast.NewString(defaultVoice)) - if err != nil { - return "", nil, nil, err - } - } - - _, err = node.Set("model", ast.NewString(meta.ActualModel)) - if err != nil { - return "", nil, nil, err - } - - jsonData, err := node.MarshalJSON() - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(jsonData), nil -} - -func TTSHandler(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - if resp.StatusCode != http.StatusOK { - return nil, ErrorHanlder(resp) - } - - defer resp.Body.Close() - - log := middleware.GetLogger(c) - - for k, v := range resp.Header { - c.Writer.Header().Set(k, v[0]) - } - - _, err := io.Copy(c.Writer, resp.Body) - if err != nil { - log.Warnf("write response body failed: %v", err) - } - return &relaymodel.Usage{ - PromptTokens: meta.InputTokens, - CompletionTokens: 0, - TotalTokens: meta.InputTokens, - }, nil -} diff --git a/service/aiproxy/relay/adaptor/openai/util.go b/service/aiproxy/relay/adaptor/openai/util.go deleted file mode 100644 index 1b4c7d8ff..000000000 --- a/service/aiproxy/relay/adaptor/openai/util.go +++ /dev/null @@ -1,42 +0,0 @@ -package openai - -import ( - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -func ErrorWrapper(err error, code any, statusCode int) *relaymodel.ErrorWithStatusCode { - return &relaymodel.ErrorWithStatusCode{ - Error: relaymodel.Error{ - Message: err.Error(), - Type: middleware.ErrorTypeAIPROXY, - Code: code, - }, - StatusCode: statusCode, - } -} - -func ErrorWrapperWithMessage(message string, code any, statusCode int) *relaymodel.ErrorWithStatusCode { - return &relaymodel.ErrorWithStatusCode{ - Error: relaymodel.Error{ - Message: message, - Type: middleware.ErrorTypeAIPROXY, - Code: code, - }, - StatusCode: statusCode, - } -} - -func GetPromptTokens(meta *meta.Meta, textRequest *relaymodel.GeneralOpenAIRequest) int { - switch meta.Mode { - case relaymode.ChatCompletions: - return CountTokenMessages(textRequest.Messages, textRequest.Model) - case relaymode.Completions: - return CountTokenInput(textRequest.Prompt, textRequest.Model) - case relaymode.Moderations: - return CountTokenInput(textRequest.Input, textRequest.Model) - } - return 0 -} diff --git a/service/aiproxy/relay/adaptor/siliconflow/adaptor.go b/service/aiproxy/relay/adaptor/siliconflow/adaptor.go deleted file mode 100644 index c814250f5..000000000 --- a/service/aiproxy/relay/adaptor/siliconflow/adaptor.go +++ /dev/null @@ -1,50 +0,0 @@ -package siliconflow - -import ( - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var _ adaptor.Adaptor = (*Adaptor)(nil) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.siliconflow.cn/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "siliconflow" -} - -//nolint:gocritic -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - usage, err := a.Adaptor.DoResponse(meta, c, resp) - if err != nil { - return nil, err - } - switch meta.Mode { - case relaymode.AudioSpeech: - size := c.Writer.Size() - usage = &relaymodel.Usage{ - CompletionTokens: size, - TotalTokens: size, - } - } - return usage, nil -} diff --git a/service/aiproxy/relay/adaptor/siliconflow/balance.go b/service/aiproxy/relay/adaptor/siliconflow/balance.go deleted file mode 100644 index e588f5185..000000000 --- a/service/aiproxy/relay/adaptor/siliconflow/balance.go +++ /dev/null @@ -1,65 +0,0 @@ -package siliconflow - -import ( - "context" - "fmt" - "net/http" - "strconv" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.Balancer = (*Adaptor)(nil) - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - u := channel.BaseURL - if u == "" { - u = baseURL - } - url := u + "/user/info" - req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", "Bearer "+channel.Key) - res, err := http.DefaultClient.Do(req) - if err != nil { - return 0, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return 0, fmt.Errorf("status code: %d", res.StatusCode) - } - response := UsageResponse{} - err = sonic.ConfigDefault.NewDecoder(res.Body).Decode(&response) - if err != nil { - return 0, err - } - balance, err := strconv.ParseFloat(response.Data.Balance, 64) - if err != nil { - return 0, err - } - return balance, nil -} - -type UsageResponse struct { - Message string `json:"message"` - Data struct { - ID string `json:"id"` - Name string `json:"name"` - Image string `json:"image"` - Email string `json:"email"` - Balance string `json:"balance"` - Status string `json:"status"` - Introduction string `json:"introduction"` - Role string `json:"role"` - ChargeBalance string `json:"chargeBalance"` - TotalBalance string `json:"totalBalance"` - Category string `json:"category"` - IsAdmin bool `json:"isAdmin"` - } `json:"data"` - Code int `json:"code"` - Status bool `json:"status"` -} diff --git a/service/aiproxy/relay/adaptor/siliconflow/constants.go b/service/aiproxy/relay/adaptor/siliconflow/constants.go deleted file mode 100644 index a3e3c9429..000000000 --- a/service/aiproxy/relay/adaptor/siliconflow/constants.go +++ /dev/null @@ -1,79 +0,0 @@ -package siliconflow - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://docs.siliconflow.cn/docs/getting-started - -var ModelList = []*model.ModelConfig{ - { - Model: "BAAI/bge-reranker-v2-m3", - Type: relaymode.Rerank, - Owner: model.ModelOwnerBAAI, - InputPrice: 0.0009, - OutputPrice: 0, - RPM: 2000, - }, - - { - Model: "BAAI/bge-large-zh-v1.5", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerBAAI, - InputPrice: 0.0005, - RPM: 2000, - }, - - { - Model: "fishaudio/fish-speech-1.4", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerFishAudio, - OutputPrice: 0.105, - Config: map[model.ModelConfigKey]any{ - model.ModelConfigSupportVoicesKey: []string{ - "fishaudio/fish-speech-1.4:alex", - "fishaudio/fish-speech-1.4:benjamin", - "fishaudio/fish-speech-1.4:charles", - "fishaudio/fish-speech-1.4:david", - "fishaudio/fish-speech-1.4:anna", - "fishaudio/fish-speech-1.4:bella", - "fishaudio/fish-speech-1.4:claire", - "fishaudio/fish-speech-1.4:diana", - }, - }, - }, - - { - Model: "FunAudioLLM/SenseVoiceSmall", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerFunAudioLLM, - }, - - { - Model: "stabilityai/stable-diffusion-3-5-large", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - ImagePrices: map[string]float64{ - "1024x1024": 0, - "512x1024": 0, - "768x512": 0, - "768x1024": 0, - "1024x576": 0, - "576x1024": 0, - }, - }, - { - Model: "stabilityai/stable-diffusion-3-5-large-turbo", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStabilityAI, - ImagePrices: map[string]float64{ - "1024x1024": 0, - "512x1024": 0, - "768x512": 0, - "768x1024": 0, - "1024x576": 0, - "576x1024": 0, - }, - }, -} diff --git a/service/aiproxy/relay/adaptor/siliconflow/image.go b/service/aiproxy/relay/adaptor/siliconflow/image.go deleted file mode 100644 index 5172c6d0d..000000000 --- a/service/aiproxy/relay/adaptor/siliconflow/image.go +++ /dev/null @@ -1,54 +0,0 @@ -package siliconflow - -import ( - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" -) - -type ImageRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - NegativePrompt string `json:"negative_prompt"` - ImageSize string `json:"image_size"` - BatchSize int `json:"batch_size"` - Seed int64 `json:"seed"` - NumInferenceSteps int `json:"num_inference_steps"` - GuidanceScale int `json:"guidance_scale"` - PromptEnhancement bool `json:"prompt_enhancement"` -} - -func ConvertImageRequest(meta *meta.Meta, request *http.Request) (http.Header, io.Reader, error) { - var reqMap map[string]any - err := common.UnmarshalBodyReusable(request, &reqMap) - if err != nil { - return nil, nil, err - } - - meta.Set(openai.MetaResponseFormat, reqMap["response_format"]) - - reqMap["model"] = meta.ActualModel - reqMap["batch_size"] = reqMap["n"] - delete(reqMap, "n") - if _, ok := reqMap["steps"]; ok { - reqMap["num_inference_steps"] = reqMap["steps"] - delete(reqMap, "steps") - } - if _, ok := reqMap["scale"]; ok { - reqMap["guidance_scale"] = reqMap["scale"] - delete(reqMap, "scale") - } - reqMap["image_size"] = reqMap["size"] - delete(reqMap, "size") - - data, err := sonic.Marshal(&reqMap) - if err != nil { - return nil, nil, err - } - return http.Header{}, bytes.NewReader(data), nil -} diff --git a/service/aiproxy/relay/adaptor/stepfun/adaptor.go b/service/aiproxy/relay/adaptor/stepfun/adaptor.go deleted file mode 100644 index d0847ed9b..000000000 --- a/service/aiproxy/relay/adaptor/stepfun/adaptor.go +++ /dev/null @@ -1,43 +0,0 @@ -package stepfun - -import ( - "io" - "net/http" - - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.stepfun.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - switch meta.Mode { - case relaymode.AudioSpeech: - return openai.ConvertTTSRequest(meta, req, "cixingnansheng") - default: - return a.Adaptor.ConvertRequest(meta, req) - } -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "stepfun" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/stepfun/constants.go b/service/aiproxy/relay/adaptor/stepfun/constants.go deleted file mode 100644 index 09ad93613..000000000 --- a/service/aiproxy/relay/adaptor/stepfun/constants.go +++ /dev/null @@ -1,165 +0,0 @@ -package stepfun - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "step-1-8k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.005, - OutputPrice: 0.02, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "step-1-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.015, - OutputPrice: 0.07, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "step-1-128k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.04, - OutputPrice: 0.2, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(128000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "step-1-256k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.95, - OutputPrice: 0.3, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(256000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "step-1-flash", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.001, - OutputPrice: 0.004, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "step-2-16k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.038, - OutputPrice: 0.12, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(16000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "step-1v-8k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.005, - OutputPrice: 0.02, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8000), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "step-1v-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.015, - OutputPrice: 0.07, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - { - Model: "step-1.5v-mini", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.008, - OutputPrice: 0.035, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32000), - model.WithModelConfigToolChoice(true), - model.WithModelConfigVision(true), - ), - }, - - { - Model: "step-tts-mini", - Type: relaymode.AudioSpeech, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.09, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(1000), - model.WithModelConfigSupportFormats([]string{"opus", "wav", "flac", "mp3"}), - model.WithModelConfigSupportVoices([]string{ - "cixingnansheng", "zhengpaiqingnian", "yuanqinansheng", - "qingniandaxuesheng", "boyinnansheng", "ruyananshi", - "shenchennanyin", "qinqienvsheng", "wenrounvsheng", - "jilingshaonv", "yuanqishaonv", "ruanmengnvsheng", - "youyanvsheng", "lengyanyujie", "shuangkuaijiejie", - "wenjingxuejie", "linjiajiejie", "linjiameimei", - "zhixingjiejie", - }), - ), - }, - - { - Model: "step-asr", - Type: relaymode.AudioTranscription, - Owner: model.ModelOwnerStepFun, - InputPrice: 0.09, - RPM: 60, - }, - - { - Model: "step-1x-medium", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerStepFun, - RPM: 60, - ImageMaxBatchSize: 1, - ImagePrices: map[string]float64{ - "256x256": 0.1, - "512x512": 0.1, - "768x768": 0.1, - "1024x1024": 0.1, - "1280x800": 0.1, - "800x1280": 0.1, - }, - }, -} diff --git a/service/aiproxy/relay/adaptor/tencent/adaptor.go b/service/aiproxy/relay/adaptor/tencent/adaptor.go deleted file mode 100644 index b1860a8cc..000000000 --- a/service/aiproxy/relay/adaptor/tencent/adaptor.go +++ /dev/null @@ -1,31 +0,0 @@ -package tencent - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -// https://cloud.tencent.com/document/api/1729/101837 - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.hunyuan.cloud.tencent.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "tencent" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/tencent/constants.go b/service/aiproxy/relay/adaptor/tencent/constants.go deleted file mode 100644 index 84bea4faf..000000000 --- a/service/aiproxy/relay/adaptor/tencent/constants.go +++ /dev/null @@ -1,191 +0,0 @@ -package tencent - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://cloud.tencent.com/document/product/1729/104753 - -var ModelList = []*model.ModelConfig{ - { - Model: "hunyuan-lite", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(262144), - model.WithModelConfigMaxInputTokens(256000), - model.WithModelConfigMaxOutputTokens(6144), - ), - }, - { - Model: "hunyuan-turbo-latest", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.015, - OutputPrice: 0.05, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(28672), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "hunyuan-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.015, - OutputPrice: 0.05, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(28672), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "hunyuan-pro", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.03, - OutputPrice: 0.10, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(28672), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "hunyuan-large", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.004, - OutputPrice: 0.012, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxInputTokens(28672), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "hunyuan-large-longcontext", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.006, - OutputPrice: 0.018, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(6144), - ), - }, - { - Model: "hunyuan-standard", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.0008, - OutputPrice: 0.002, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - // { - // Model: "hunyuan-standard-256K", - // Type: relaymode.ChatCompletions, - // Owner: model.ModelOwnerTencent, - // InputPrice: 0.0005, - // OutputPrice: 0.002, - // }, - { - Model: "hunyuan-role", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.004, - OutputPrice: 0.008, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "hunyuan-functioncall", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.004, - OutputPrice: 0.008, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "hunyuan-code", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.004, - OutputPrice: 0.008, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigMaxInputTokens(4096), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "hunyuan-turbo-vision", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.08, - OutputPrice: 0.08, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "hunyuan-vision", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerTencent, - InputPrice: 0.018, - OutputPrice: 0.018, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigMaxInputTokens(6144), - model.WithModelConfigMaxOutputTokens(2048), - model.WithModelConfigVision(true), - model.WithModelConfigToolChoice(true), - ), - }, - - { - Model: "hunyuan-embedding", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerTencent, - InputPrice: 0.0007, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(1024), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/vertexai/adaptor.go b/service/aiproxy/relay/adaptor/vertexai/adaptor.go deleted file mode 100644 index 64882637b..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/adaptor.go +++ /dev/null @@ -1,117 +0,0 @@ -package vertexai - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - channelhelper "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -var _ channelhelper.Adaptor = new(Adaptor) - -const channelName = "vertexai" - -type Adaptor struct{} - -func (a *Adaptor) GetBaseURL() string { - return "" -} - -type Config struct { - Region string - ProjectID string - ADCJSON string -} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) { - adaptor := GetAdaptor(meta.ActualModel) - if adaptor == nil { - return "", nil, nil, errors.New("adaptor not found") - } - - return adaptor.ConvertRequest(meta, request) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - adaptor := GetAdaptor(meta.ActualModel) - if adaptor == nil { - return nil, openai.ErrorWrapperWithMessage(meta.ActualModel+" adaptor not found", "adaptor_not_found", http.StatusInternalServerError) - } - return adaptor.DoResponse(meta, c, resp) -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return modelList -} - -func (a *Adaptor) GetChannelName() string { - return channelName -} - -func (a *Adaptor) GetRequestURL(meta *meta.Meta) (string, error) { - var suffix string - if strings.HasPrefix(meta.ActualModel, "gemini") { - if meta.GetBool("stream") { - suffix = "streamGenerateContent?alt=sse" - } else { - suffix = "generateContent" - } - } else { - if meta.GetBool("stream") { - suffix = "streamRawPredict?alt=sse" - } else { - suffix = "rawPredict" - } - } - - config, err := getConfigFromKey(meta.Channel.Key) - if err != nil { - return "", err - } - - if meta.Channel.BaseURL != "" { - return fmt.Sprintf( - "%s/v1/projects/%s/locations/%s/publishers/google/models/%s:%s", - meta.Channel.BaseURL, - config.ProjectID, - config.Region, - meta.ActualModel, - suffix, - ), nil - } - return fmt.Sprintf( - "https://%s-aiplatform.googleapis.com/v1/projects/%s/locations/%s/publishers/google/models/%s:%s", - config.Region, - config.ProjectID, - config.Region, - meta.ActualModel, - suffix, - ), nil -} - -func (a *Adaptor) SetupRequestHeader(meta *meta.Meta, _ *gin.Context, req *http.Request) error { - config, err := getConfigFromKey(meta.Channel.Key) - if err != nil { - return err - } - token, err := getToken(context.Background(), meta.Channel.ID, config.ADCJSON) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+token) - return nil -} - -func (a *Adaptor) DoRequest(_ *meta.Meta, _ *gin.Context, req *http.Request) (*http.Response, error) { - return utils.DoRequest(req) -} diff --git a/service/aiproxy/relay/adaptor/vertexai/claude/adapter.go b/service/aiproxy/relay/adaptor/vertexai/claude/adapter.go deleted file mode 100644 index 6fef4659a..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/claude/adapter.go +++ /dev/null @@ -1,92 +0,0 @@ -package vertexai - -import ( - "bytes" - "io" - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/anthropic" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" - "github.com/pkg/errors" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "claude-3-haiku@20240307", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - { - Model: "claude-3-sonnet@20240229", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - { - Model: "claude-3-opus@20240229", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - { - Model: "claude-3-5-sonnet@20240620", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - { - Model: "claude-3-5-sonnet-v2@20241022", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, - { - Model: "claude-3-5-haiku@20241022", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerAnthropic, - }, -} - -const anthropicVersion = "vertex-2023-10-16" - -type Adaptor struct{} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) { - if request == nil { - return "", nil, nil, errors.New("request is nil") - } - - claudeReq, err := anthropic.ConvertRequest(meta, request) - if err != nil { - return "", nil, nil, err - } - meta.Set("stream", claudeReq.Stream) - req := Request{ - AnthropicVersion: anthropicVersion, - // Model: claudeReq.Model, - Messages: claudeReq.Messages, - System: claudeReq.System, - MaxTokens: claudeReq.MaxTokens, - Temperature: claudeReq.Temperature, - TopP: claudeReq.TopP, - TopK: claudeReq.TopK, - Stream: claudeReq.Stream, - Tools: claudeReq.Tools, - } - data, err := sonic.Marshal(req) - if err != nil { - return "", nil, nil, err - } - return http.MethodPost, nil, bytes.NewReader(data), nil -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - if utils.IsStreamResponse(resp) { - usage, err = anthropic.StreamHandler(meta, c, resp) - } else { - usage, err = anthropic.Handler(meta, c, resp) - } - return -} diff --git a/service/aiproxy/relay/adaptor/vertexai/claude/model.go b/service/aiproxy/relay/adaptor/vertexai/claude/model.go deleted file mode 100644 index eda799ec8..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/claude/model.go +++ /dev/null @@ -1,17 +0,0 @@ -package vertexai - -import "github.com/labring/sealos/service/aiproxy/relay/adaptor/anthropic" - -type Request struct { - ToolChoice any `json:"tool_choice,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - AnthropicVersion string `json:"anthropic_version"` - System string `json:"system,omitempty"` - Messages []anthropic.Message `json:"messages"` - StopSequences []string `json:"stop_sequences,omitempty"` - Tools []anthropic.Tool `json:"tools,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopK int `json:"top_k,omitempty"` - Stream bool `json:"stream,omitempty"` -} diff --git a/service/aiproxy/relay/adaptor/vertexai/gemini/adapter.go b/service/aiproxy/relay/adaptor/vertexai/gemini/adapter.go deleted file mode 100644 index 88957903a..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/gemini/adapter.go +++ /dev/null @@ -1,57 +0,0 @@ -package vertexai - -import ( - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/gemini" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "gemini-1.5-pro-001", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "gemini-1.5-flash-001", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "gemini-pro", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, - { - Model: "gemini-pro-vision", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerGoogle, - }, -} - -type Adaptor struct{} - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) { - return gemini.ConvertRequest(meta, request) -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.Embeddings: - usage, err = gemini.EmbeddingHandler(c, resp) - default: - if utils.IsStreamResponse(resp) { - usage, err = gemini.StreamHandler(meta, c, resp) - } else { - usage, err = gemini.Handler(meta, c, resp) - } - } - return -} diff --git a/service/aiproxy/relay/adaptor/vertexai/key.go b/service/aiproxy/relay/adaptor/vertexai/key.go deleted file mode 100644 index 955c9b9ca..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/key.go +++ /dev/null @@ -1,39 +0,0 @@ -package vertexai - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - _, err := getConfigFromKey(key) - if err != nil { - return err - } - return nil -} - -func (a *Adaptor) KeyHelp() string { - return "region|projectID|adcJSON" -} - -// region|projectID|adcJSON -func getConfigFromKey(key string) (Config, error) { - region, after, ok := strings.Cut(key, "|") - if !ok { - return Config{}, errors.New("invalid key format") - } - projectID, adcJSON, ok := strings.Cut(after, "|") - if !ok { - return Config{}, errors.New("invalid key format") - } - return Config{ - Region: region, - ProjectID: projectID, - ADCJSON: adcJSON, - }, nil -} diff --git a/service/aiproxy/relay/adaptor/vertexai/registry.go b/service/aiproxy/relay/adaptor/vertexai/registry.go deleted file mode 100644 index 7253a16e0..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/registry.go +++ /dev/null @@ -1,54 +0,0 @@ -package vertexai - -import ( - "io" - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - claude "github.com/labring/sealos/service/aiproxy/relay/adaptor/vertexai/claude" - gemini "github.com/labring/sealos/service/aiproxy/relay/adaptor/vertexai/gemini" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type ModelType int - -const ( - VerterAIClaude ModelType = iota + 1 - VerterAIGemini -) - -var ( - modelMapping = map[string]ModelType{} - modelList = []*model.ModelConfig{} -) - -func init() { - for _, model := range claude.ModelList { - modelMapping[model.Model] = VerterAIClaude - modelList = append(modelList, model) - } - - for _, model := range gemini.ModelList { - modelMapping[model.Model] = VerterAIGemini - modelList = append(modelList, model) - } -} - -type innerAIAdapter interface { - ConvertRequest(meta *meta.Meta, request *http.Request) (string, http.Header, io.Reader, error) - DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) -} - -func GetAdaptor(model string) innerAIAdapter { - adaptorType := modelMapping[model] - switch adaptorType { - case VerterAIClaude: - return &claude.Adaptor{} - case VerterAIGemini: - return &gemini.Adaptor{} - default: - return nil - } -} diff --git a/service/aiproxy/relay/adaptor/vertexai/token.go b/service/aiproxy/relay/adaptor/vertexai/token.go deleted file mode 100644 index 63cd7478d..000000000 --- a/service/aiproxy/relay/adaptor/vertexai/token.go +++ /dev/null @@ -1,63 +0,0 @@ -package vertexai - -import ( - "context" - "fmt" - "time" - - credentials "cloud.google.com/go/iam/credentials/apiv1" - "cloud.google.com/go/iam/credentials/apiv1/credentialspb" - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/patrickmn/go-cache" - "google.golang.org/api/option" -) - -type ApplicationDefaultCredentials struct { - Type string `json:"type"` - ProjectID string `json:"project_id"` - PrivateKeyID string `json:"private_key_id"` - PrivateKey string `json:"private_key"` - ClientEmail string `json:"client_email"` - ClientID string `json:"client_id"` - AuthURI string `json:"auth_uri"` - TokenURI string `json:"token_uri"` - AuthProviderX509CertURL string `json:"auth_provider_x509_cert_url"` - ClientX509CertURL string `json:"client_x509_cert_url"` - UniverseDomain string `json:"universe_domain"` -} - -var Cache = cache.New(50*time.Minute, 55*time.Minute) - -const defaultScope = "https://www.googleapis.com/auth/cloud-platform" - -func getToken(ctx context.Context, channelID int, adcJSON string) (string, error) { - cacheKey := fmt.Sprintf("vertexai-token-%d", channelID) - if token, found := Cache.Get(cacheKey); found { - return token.(string), nil - } - adc := &ApplicationDefaultCredentials{} - if err := sonic.UnmarshalString(adcJSON, adc); err != nil { - return "", fmt.Errorf("failed to decode credentials file: %w", err) - } - - c, err := credentials.NewIamCredentialsClient(ctx, option.WithCredentialsJSON(conv.StringToBytes(adcJSON))) - if err != nil { - return "", fmt.Errorf("failed to create client: %w", err) - } - defer c.Close() - - req := &credentialspb.GenerateAccessTokenRequest{ - // See https://pkg.go.dev/cloud.google.com/go/iam/credentials/apiv1/credentialspb#GenerateAccessTokenRequest. - Name: "projects/-/serviceAccounts/" + adc.ClientEmail, - Scope: []string{defaultScope}, - } - resp, err := c.GenerateAccessToken(ctx, req) - if err != nil { - return "", fmt.Errorf("failed to generate access token: %w", err) - } - _ = resp - - Cache.Set(cacheKey, resp.GetAccessToken(), cache.DefaultExpiration) - return resp.GetAccessToken(), nil -} diff --git a/service/aiproxy/relay/adaptor/xai/adaptor.go b/service/aiproxy/relay/adaptor/xai/adaptor.go deleted file mode 100644 index 2dd1d8332..000000000 --- a/service/aiproxy/relay/adaptor/xai/adaptor.go +++ /dev/null @@ -1,24 +0,0 @@ -package xai - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://api.xai.com/v1" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "xai" -} diff --git a/service/aiproxy/relay/adaptor/xai/constants.go b/service/aiproxy/relay/adaptor/xai/constants.go deleted file mode 100644 index 087d79e7a..000000000 --- a/service/aiproxy/relay/adaptor/xai/constants.go +++ /dev/null @@ -1,90 +0,0 @@ -package xai - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "grok-3", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.002, - OutputPrice: 0.01, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1000000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "grok-3-deepsearch", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.01, - OutputPrice: 0.05, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1000000), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "grok-3-reasoner", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.004, - OutputPrice: 0.02, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1000000), - ), - }, - { - Model: "grok-2-1212", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.002, - OutputPrice: 0.01, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, - { - Model: "grok-2-vision-1212", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.06, - OutputPrice: 0.06, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigVision(true), - ), - }, - { - Model: "grok-beta", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.03, - OutputPrice: 0.12, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, - { - Model: "grok-vision-beta", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXAI, - InputPrice: 0.06, - OutputPrice: 0.06, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigVision(true), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/xunfei/adaptor.go b/service/aiproxy/relay/adaptor/xunfei/adaptor.go deleted file mode 100644 index 2ac141317..000000000 --- a/service/aiproxy/relay/adaptor/xunfei/adaptor.go +++ /dev/null @@ -1,50 +0,0 @@ -package xunfei - -import ( - "io" - "net/http" - - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" -) - -type Adaptor struct { - openai.Adaptor -} - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -const baseURL = "https://spark-api-open.xf-yun.com/v1" - -func (a *Adaptor) ConvertRequest(meta *meta.Meta, req *http.Request) (string, http.Header, io.Reader, error) { - domain, err := getXunfeiDomain(meta.ActualModel) - if err != nil { - return "", nil, nil, err - } - model := meta.ActualModel - meta.ActualModel = domain - defer func() { - meta.ActualModel = model - }() - method, h, body, err := a.Adaptor.ConvertRequest(meta, req) - if err != nil { - return "", nil, nil, err - } - return method, h, body, nil -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "xunfei" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/xunfei/constants.go b/service/aiproxy/relay/adaptor/xunfei/constants.go deleted file mode 100644 index 6937e3b23..000000000 --- a/service/aiproxy/relay/adaptor/xunfei/constants.go +++ /dev/null @@ -1,80 +0,0 @@ -package xunfei - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -// https://www.xfyun.cn/doc/spark/HTTP%E8%B0%83%E7%94%A8%E6%96%87%E6%A1%A3.html#_1-%E6%8E%A5%E5%8F%A3%E8%AF%B4%E6%98%8E - -var ModelList = []*model.ModelConfig{ - { - Model: "SparkDesk-4.0-Ultra", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXunfei, - InputPrice: 0.14, - OutputPrice: 0.14, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "SparkDesk-Lite", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXunfei, - InputPrice: 0.001, - OutputPrice: 0.001, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - ), - }, - { - Model: "SparkDesk-Max", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXunfei, - InputPrice: 0.06, - OutputPrice: 0.06, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "SparkDesk-Max-32k", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXunfei, - InputPrice: 0.09, - OutputPrice: 0.09, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(32768), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "SparkDesk-Pro", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXunfei, - InputPrice: 0.014, - OutputPrice: 0.014, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, - { - Model: "SparkDesk-Pro-128K", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerXunfei, - InputPrice: 0.026, - OutputPrice: 0.026, - RPM: 120, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/xunfei/key.go b/service/aiproxy/relay/adaptor/xunfei/key.go deleted file mode 100644 index 55ce0deb4..000000000 --- a/service/aiproxy/relay/adaptor/xunfei/key.go +++ /dev/null @@ -1,21 +0,0 @@ -package xunfei - -import ( - "errors" - "strings" - - "github.com/labring/sealos/service/aiproxy/relay/adaptor" -) - -var _ adaptor.KeyValidator = (*Adaptor)(nil) - -func (a *Adaptor) ValidateKey(key string) error { - if strings.Contains(key, ":") { - return nil - } - return errors.New("invalid key format") -} - -func (a *Adaptor) KeyHelp() string { - return "xxx:xxx" -} diff --git a/service/aiproxy/relay/adaptor/xunfei/main.go b/service/aiproxy/relay/adaptor/xunfei/main.go deleted file mode 100644 index 1d6004dc5..000000000 --- a/service/aiproxy/relay/adaptor/xunfei/main.go +++ /dev/null @@ -1,31 +0,0 @@ -package xunfei - -import ( - "errors" - "strings" -) - -// https://console.xfyun.cn/services/cbm -// https://www.xfyun.cn/doc/spark/HTTP%E8%B0%83%E7%94%A8%E6%96%87%E6%A1%A3.html - -func getXunfeiDomain(modelName string) (string, error) { - _, s, ok := strings.Cut(modelName, "-") - if !ok { - return "", errors.New("invalid model name") - } - switch strings.ToLower(s) { - case "lite": - return "lite", nil - case "pro": - return "generalv3", nil - case "pro-128k": - return "pro-128k", nil - case "max": - return "generalv3.5", nil - case "max-32k": - return "max-32k", nil - case "4.0-ultra": - return "4.0Ultra", nil - } - return "", errors.New("invalid model name") -} diff --git a/service/aiproxy/relay/adaptor/zhipu/adaptor.go b/service/aiproxy/relay/adaptor/zhipu/adaptor.go deleted file mode 100644 index 2fbb82ea5..000000000 --- a/service/aiproxy/relay/adaptor/zhipu/adaptor.go +++ /dev/null @@ -1,45 +0,0 @@ -package zhipu - -import ( - "net/http" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type Adaptor struct { - openai.Adaptor -} - -const baseURL = "https://open.bigmodel.cn/api/paas/v4" - -func (a *Adaptor) GetBaseURL() string { - return baseURL -} - -func (a *Adaptor) DoResponse(meta *meta.Meta, c *gin.Context, resp *http.Response) (usage *relaymodel.Usage, err *relaymodel.ErrorWithStatusCode) { - switch meta.Mode { - case relaymode.Embeddings: - err, usage = EmbeddingsHandler(c, resp) - default: - usage, err = openai.DoResponse(meta, c, resp) - } - return -} - -func (a *Adaptor) GetModelList() []*model.ModelConfig { - return ModelList -} - -func (a *Adaptor) GetChannelName() string { - return "zhipu" -} - -func (a *Adaptor) GetBalance(channel *model.Channel) (float64, error) { - return 0, adaptor.ErrGetBalanceNotImplemented -} diff --git a/service/aiproxy/relay/adaptor/zhipu/constants.go b/service/aiproxy/relay/adaptor/zhipu/constants.go deleted file mode 100644 index bd501e73d..000000000 --- a/service/aiproxy/relay/adaptor/zhipu/constants.go +++ /dev/null @@ -1,230 +0,0 @@ -package zhipu - -import ( - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -var ModelList = []*model.ModelConfig{ - { - Model: "glm-3-turbo", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.001, - OutputPrice: 0.001, - RPM: 300, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - { - Model: "glm-4", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.1, - OutputPrice: 0.1, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.05, - OutputPrice: 0.05, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4-air", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.001, - OutputPrice: 0.001, - RPM: 900, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4-airx", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.01, - OutputPrice: 0.01, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(8192), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4-long", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.001, - OutputPrice: 0.001, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(1024000), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4-flashx", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4-flash", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 1800, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - model.WithModelConfigToolChoice(true), - ), - }, - { - Model: "glm-4v-flash", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - model.WithModelConfigMaxOutputTokens(1024), - model.WithModelConfigVision(true), - ), - }, - { - Model: "glm-4v", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.05, - OutputPrice: 0.05, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(2048), - model.WithModelConfigMaxOutputTokens(1024), - model.WithModelConfigVision(true), - ), - }, - { - Model: "glm-4v-plus", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.01, - OutputPrice: 0.01, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - model.WithModelConfigMaxOutputTokens(1024), - model.WithModelConfigVision(true), - ), - }, - - { - Model: "charglm-4", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.001, - OutputPrice: 0.001, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(4096), - model.WithModelConfigMaxOutputTokens(2048), - ), - }, - { - Model: "codegeex-4", - Type: relaymode.ChatCompletions, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.0001, - OutputPrice: 0.0001, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxContextTokens(131072), - model.WithModelConfigMaxOutputTokens(4096), - ), - }, - - { - Model: "embedding-2", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.0005, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - ), - }, - { - Model: "embedding-3", - Type: relaymode.Embeddings, - Owner: model.ModelOwnerChatGLM, - InputPrice: 0.0005, - RPM: 600, - Config: model.NewModelConfig( - model.WithModelConfigMaxInputTokens(8192), - ), - }, - - { - Model: "cogview-3", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerChatGLM, - ImageMaxBatchSize: 1, - ImagePrices: map[string]float64{ - "1024x1024": 0.1, - }, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxOutputTokens(1024), - ), - }, - { - Model: "cogview-3-plus", - Type: relaymode.ImagesGenerations, - Owner: model.ModelOwnerChatGLM, - ImageMaxBatchSize: 1, - ImagePrices: map[string]float64{ - "1024x1024": 0.06, - "768x1344": 0.06, - "864x1152": 0.06, - "1344x768": 0.06, - "1152x864": 0.06, - "1440x720": 0.06, - "720x1440": 0.06, - }, - RPM: 60, - Config: model.NewModelConfig( - model.WithModelConfigMaxOutputTokens(1024), - ), - }, -} diff --git a/service/aiproxy/relay/adaptor/zhipu/main.go b/service/aiproxy/relay/adaptor/zhipu/main.go deleted file mode 100644 index 78610ad60..000000000 --- a/service/aiproxy/relay/adaptor/zhipu/main.go +++ /dev/null @@ -1,56 +0,0 @@ -package zhipu - -import ( - "net/http" - - "github.com/bytedance/sonic" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -// https://open.bigmodel.cn/doc/api#chatglm_std -// chatglm_std, chatglm_lite -// https://open.bigmodel.cn/api/paas/v3/model-api/chatglm_std/invoke -// https://open.bigmodel.cn/api/paas/v3/model-api/chatglm_std/sse-invoke - -func EmbeddingsHandler(c *gin.Context, resp *http.Response) (*model.ErrorWithStatusCode, *model.Usage) { - defer resp.Body.Close() - - var zhipuResponse EmbeddingResponse - err := sonic.ConfigDefault.NewDecoder(resp.Body).Decode(&zhipuResponse) - if err != nil { - return openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError), nil - } - fullTextResponse := embeddingResponseZhipu2OpenAI(&zhipuResponse) - jsonResponse, err := sonic.Marshal(fullTextResponse) - if err != nil { - return openai.ErrorWrapper(err, "marshal_response_body_failed", http.StatusInternalServerError), nil - } - c.Writer.Header().Set("Content-Type", "application/json") - c.Writer.WriteHeader(resp.StatusCode) - _, _ = c.Writer.Write(jsonResponse) - return nil, &fullTextResponse.Usage -} - -func embeddingResponseZhipu2OpenAI(response *EmbeddingResponse) *openai.EmbeddingResponse { - openAIEmbeddingResponse := openai.EmbeddingResponse{ - Object: "list", - Data: make([]*openai.EmbeddingResponseItem, 0, len(response.Embeddings)), - Model: response.Model, - Usage: model.Usage{ - PromptTokens: response.PromptTokens, - CompletionTokens: response.CompletionTokens, - TotalTokens: response.Usage.TotalTokens, - }, - } - - for _, item := range response.Embeddings { - openAIEmbeddingResponse.Data = append(openAIEmbeddingResponse.Data, &openai.EmbeddingResponseItem{ - Object: `embedding`, - Index: item.Index, - Embedding: item.Embedding, - }) - } - return &openAIEmbeddingResponse -} diff --git a/service/aiproxy/relay/adaptor/zhipu/model.go b/service/aiproxy/relay/adaptor/zhipu/model.go deleted file mode 100644 index 42148fa18..000000000 --- a/service/aiproxy/relay/adaptor/zhipu/model.go +++ /dev/null @@ -1,37 +0,0 @@ -package zhipu - -import ( - "github.com/labring/sealos/service/aiproxy/relay/model" -) - -type Request struct { - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - RequestID string `json:"request_id,omitempty"` - Prompt []*model.Message `json:"prompt"` - Incremental bool `json:"incremental,omitempty"` -} - -type EmbeddingRequest struct { - Input any `json:"input"` - Model string `json:"model"` -} - -type EmbeddingResponse struct { - Model string `json:"model"` - Object string `json:"object"` - Embeddings []EmbeddingData `json:"data"` - model.Usage `json:"usage"` -} - -type EmbeddingData struct { - Object string `json:"object"` - Embedding []float64 `json:"embedding"` - Index int `json:"index"` -} - -type ImageRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - UserID string `json:"user_id,omitempty"` -} diff --git a/service/aiproxy/relay/channeltype/define.go b/service/aiproxy/relay/channeltype/define.go deleted file mode 100644 index c6f46591d..000000000 --- a/service/aiproxy/relay/channeltype/define.go +++ /dev/null @@ -1,112 +0,0 @@ -package channeltype - -import ( - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/ai360" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/ali" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/anthropic" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/aws" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/azure" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/baichuan" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/baidu" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/baiduv2" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/cloudflare" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/cohere" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/coze" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/deepseek" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/doc2x" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/doubao" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/doubaoaudio" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/gemini" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/geminiopenai" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/groq" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/lingyiwanwu" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/minimax" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/mistral" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/moonshot" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/novita" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/ollama" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/siliconflow" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/stepfun" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/tencent" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/vertexai" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/xai" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/xunfei" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/zhipu" -) - -var ChannelAdaptor = map[int]adaptor.Adaptor{ - 1: &openai.Adaptor{}, - 3: &azure.Adaptor{}, - 12: &geminiopenai.Adaptor{}, - 13: &baiduv2.Adaptor{}, - 14: &anthropic.Adaptor{}, - 15: &baidu.Adaptor{}, - 16: &zhipu.Adaptor{}, - 17: &ali.Adaptor{}, - 18: &xunfei.Adaptor{}, - 19: &ai360.Adaptor{}, - 23: &tencent.Adaptor{}, - 24: &gemini.Adaptor{}, - 25: &moonshot.Adaptor{}, - 26: &baichuan.Adaptor{}, - 27: &minimax.Adaptor{}, - 28: &mistral.Adaptor{}, - 29: &groq.Adaptor{}, - 30: &ollama.Adaptor{}, - 31: &lingyiwanwu.Adaptor{}, - 32: &stepfun.Adaptor{}, - 33: &aws.Adaptor{}, - 34: &coze.Adaptor{}, - 35: &cohere.Adaptor{}, - 36: &deepseek.Adaptor{}, - 37: &cloudflare.Adaptor{}, - 40: &doubao.Adaptor{}, - 41: &novita.Adaptor{}, - 42: &vertexai.Adaptor{}, - 43: &siliconflow.Adaptor{}, - 44: &doubaoaudio.Adaptor{}, - 45: &xai.Adaptor{}, - 46: &doc2x.Adaptor{}, -} - -func GetAdaptor(channel int) (adaptor.Adaptor, bool) { - a, ok := ChannelAdaptor[channel] - return a, ok -} - -type AdaptorMeta struct { - Name string `json:"name"` - KeyHelp string `json:"keyHelp"` - DefaultBaseURL string `json:"defaultBaseUrl"` -} - -var ( - ChannelNames = map[int]string{} - ChannelMetas = map[int]AdaptorMeta{} -) - -func init() { - names := make(map[string]struct{}) - for i, adaptor := range ChannelAdaptor { - name := adaptor.GetChannelName() - if _, ok := names[name]; ok { - panic("duplicate channel name: " + name) - } - names[name] = struct{}{} - ChannelMetas[i] = AdaptorMeta{ - Name: name, - KeyHelp: getAdaptorKeyHelp(adaptor), - DefaultBaseURL: adaptor.GetBaseURL(), - } - ChannelNames[i] = name - } -} - -func getAdaptorKeyHelp(a adaptor.Adaptor) string { - if keyValidator, ok := a.(adaptor.KeyValidator); ok { - return keyValidator.KeyHelp() - } - return "" -} diff --git a/service/aiproxy/relay/constant/common.go b/service/aiproxy/relay/constant/common.go deleted file mode 100644 index 03544fd3c..000000000 --- a/service/aiproxy/relay/constant/common.go +++ /dev/null @@ -1,7 +0,0 @@ -package constant - -var ( - StopFinishReason = "stop" - StreamObject = "chat.completion.chunk" - NonStreamObject = "chat.completion" -) diff --git a/service/aiproxy/relay/constant/finishreason/define.go b/service/aiproxy/relay/constant/finishreason/define.go deleted file mode 100644 index 1ed9c4255..000000000 --- a/service/aiproxy/relay/constant/finishreason/define.go +++ /dev/null @@ -1,5 +0,0 @@ -package finishreason - -const ( - Stop = "stop" -) diff --git a/service/aiproxy/relay/constant/role/define.go b/service/aiproxy/relay/constant/role/define.go deleted file mode 100644 index 972488c5c..000000000 --- a/service/aiproxy/relay/constant/role/define.go +++ /dev/null @@ -1,5 +0,0 @@ -package role - -const ( - Assistant = "assistant" -) diff --git a/service/aiproxy/relay/controller/consume.go b/service/aiproxy/relay/controller/consume.go deleted file mode 100644 index 94fa56b47..000000000 --- a/service/aiproxy/relay/controller/consume.go +++ /dev/null @@ -1,54 +0,0 @@ -package controller - -import ( - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/balance" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - "github.com/shopspring/decimal" -) - -type PreCheckGroupBalanceReq struct { - InputTokens int - MaxTokens int - InputPrice float64 - OutputPrice float64 -} - -func getPreConsumedAmount(req *PreCheckGroupBalanceReq) float64 { - if req == nil || req.InputPrice == 0 || (req.InputTokens == 0 && req.MaxTokens == 0) { - return 0 - } - preConsumedTokens := int64(req.InputTokens) - if req.MaxTokens != 0 { - preConsumedTokens += int64(req.MaxTokens) - } - return decimal. - NewFromInt(preConsumedTokens). - Mul(decimal.NewFromFloat(req.InputPrice)). - Div(decimal.NewFromInt(model.PriceUnit)). - InexactFloat64() -} - -func checkGroupBalance(req *PreCheckGroupBalanceReq, meta *meta.Meta, groupRemainBalance float64) bool { - if meta.IsChannelTest { - return true - } - - preConsumedAmount := getPreConsumedAmount(req) - - return groupRemainBalance > preConsumedAmount -} - -func getGroupBalance(ctx *gin.Context, meta *meta.Meta) (float64, balance.PostGroupConsumer, error) { - if meta.IsChannelTest { - return 0, nil, nil - } - - gbc, err := middleware.GetGroupBalanceConsumer(ctx, meta.Group) - if err != nil { - return 0, nil, err - } - return gbc.GroupBalance, gbc.Consumer, nil -} diff --git a/service/aiproxy/relay/controller/dohelper.go b/service/aiproxy/relay/controller/dohelper.go deleted file mode 100644 index 26b306c0a..000000000 --- a/service/aiproxy/relay/controller/dohelper.go +++ /dev/null @@ -1,243 +0,0 @@ -package controller - -import ( - "bytes" - "context" - "errors" - "io" - "net/http" - "sync" - "time" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/conv" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" - log "github.com/sirupsen/logrus" -) - -const ( - // 0.5MB - maxBufferSize = 512 * 1024 -) - -type responseWriter struct { - gin.ResponseWriter - body *bytes.Buffer -} - -func (rw *responseWriter) Write(b []byte) (int, error) { - if total := rw.body.Len() + len(b); total <= maxBufferSize { - rw.body.Write(b) - } else { - rw.body.Write(b[:maxBufferSize-rw.body.Len()]) - } - return rw.ResponseWriter.Write(b) -} - -func (rw *responseWriter) WriteString(s string) (int, error) { - if total := rw.body.Len() + len(s); total <= maxBufferSize { - rw.body.WriteString(s) - } else { - rw.body.WriteString(s[:maxBufferSize-rw.body.Len()]) - } - return rw.ResponseWriter.WriteString(s) -} - -var bufferPool = sync.Pool{ - New: func() interface{} { - return bytes.NewBuffer(make([]byte, 0, maxBufferSize)) - }, -} - -func getBuffer() *bytes.Buffer { - return bufferPool.Get().(*bytes.Buffer) -} - -func putBuffer(buf *bytes.Buffer) { - buf.Reset() - if buf.Cap() > maxBufferSize { - return - } - bufferPool.Put(buf) -} - -func DoHelper( - a adaptor.Adaptor, - c *gin.Context, - meta *meta.Meta, -) ( - *relaymodel.Usage, - *model.RequestDetail, - *relaymodel.ErrorWithStatusCode, -) { - log := middleware.GetLogger(c) - detail := model.RequestDetail{} - - // 1. Get request body - if err := getRequestBody(meta, c, &detail); err != nil { - return nil, nil, err - } - - // 2. Convert and prepare request - resp, err := prepareAndDoRequest(a, c, meta) - if err != nil { - return nil, &detail, err - } - - // 3. Handle error response - if resp == nil { - relayErr := openai.ErrorWrapperWithMessage("response is nil", openai.ErrorCodeBadResponse, http.StatusInternalServerError) - detail.ResponseBody = relayErr.JSONOrEmpty() - return nil, &detail, relayErr - } - - // 4. Handle success response - usage, relayErr := handleResponse(a, c, meta, resp, &detail) - if relayErr != nil { - return nil, &detail, relayErr - } - - // 5. Update usage metrics - updateUsageMetrics(usage, meta, log) - - return usage, &detail, nil -} - -func getRequestBody(meta *meta.Meta, c *gin.Context, detail *model.RequestDetail) *relaymodel.ErrorWithStatusCode { - switch meta.Mode { - case relaymode.AudioTranscription, relaymode.AudioTranslation: - return nil - default: - reqBody, err := common.GetRequestBody(c.Request) - if err != nil { - return openai.ErrorWrapperWithMessage("get request body failed: "+err.Error(), "get_request_body_failed", http.StatusBadRequest) - } - detail.RequestBody = conv.BytesToString(reqBody) - return nil - } -} - -func prepareAndDoRequest(a adaptor.Adaptor, c *gin.Context, meta *meta.Meta) (*http.Response, *relaymodel.ErrorWithStatusCode) { - method, header, body, err := a.ConvertRequest(meta, c.Request) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("convert request failed: "+err.Error(), "convert_request_failed", http.StatusBadRequest) - } - if closer, ok := body.(io.Closer); ok { - defer closer.Close() - } - - if meta.Channel.BaseURL == "" { - meta.Channel.BaseURL = a.GetBaseURL() - } - - fullRequestURL, err := a.GetRequestURL(meta) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("get request url failed: "+err.Error(), "get_request_url_failed", http.StatusBadRequest) - } - - log.Debugf("request url: %s %s", method, fullRequestURL) - - ctx := context.Background() - if timeout := config.GetTimeoutWithModelType()[int(meta.Mode)]; timeout > 0 { - // donot use c.Request.Context() because it will be canceled by the client - // which will cause the usage of non-streaming requests to be unable to be recorded - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second) - defer cancel() - } - - req, err := http.NewRequestWithContext(ctx, method, fullRequestURL, body) - if err != nil { - return nil, openai.ErrorWrapperWithMessage("new request failed: "+err.Error(), "new_request_failed", http.StatusBadRequest) - } - - if err := setupRequestHeader(a, c, meta, req, header); err != nil { - return nil, err - } - - return doRequest(a, c, meta, req) -} - -func setupRequestHeader(a adaptor.Adaptor, c *gin.Context, meta *meta.Meta, req *http.Request, header http.Header) *relaymodel.ErrorWithStatusCode { - contentType := req.Header.Get("Content-Type") - if contentType == "" { - contentType = "application/json; charset=utf-8" - } - req.Header.Set("Content-Type", contentType) - for key, value := range header { - req.Header[key] = value - } - if err := a.SetupRequestHeader(meta, c, req); err != nil { - return openai.ErrorWrapperWithMessage("setup request header failed: "+err.Error(), "setup_request_header_failed", http.StatusBadRequest) - } - return nil -} - -func doRequest(a adaptor.Adaptor, c *gin.Context, meta *meta.Meta, req *http.Request) (*http.Response, *relaymodel.ErrorWithStatusCode) { - resp, err := a.DoRequest(meta, c, req) - if err != nil { - if errors.Is(err, context.Canceled) { - return nil, openai.ErrorWrapperWithMessage("do request failed: request canceled by client", "request_canceled", http.StatusBadRequest) - } - if errors.Is(err, context.DeadlineExceeded) { - return nil, openai.ErrorWrapperWithMessage("do request failed: request timeout", "request_timeout", http.StatusGatewayTimeout) - } - if errors.Is(err, io.EOF) { - return nil, openai.ErrorWrapperWithMessage("do request failed: "+err.Error(), "request_failed", http.StatusServiceUnavailable) - } - if errors.Is(err, io.ErrUnexpectedEOF) { - return nil, openai.ErrorWrapperWithMessage("do request failed: "+err.Error(), "request_failed", http.StatusInternalServerError) - } - return nil, openai.ErrorWrapperWithMessage("do request failed: "+err.Error(), "request_failed", http.StatusBadRequest) - } - return resp, nil -} - -func handleResponse(a adaptor.Adaptor, c *gin.Context, meta *meta.Meta, resp *http.Response, detail *model.RequestDetail) (*relaymodel.Usage, *relaymodel.ErrorWithStatusCode) { - buf := getBuffer() - defer putBuffer(buf) - - rw := &responseWriter{ - ResponseWriter: c.Writer, - body: buf, - } - rawWriter := c.Writer - defer func() { c.Writer = rawWriter }() - c.Writer = rw - - c.Header("Content-Type", resp.Header.Get("Content-Type")) - - usage, relayErr := a.DoResponse(meta, c, resp) - if relayErr != nil { - detail.ResponseBody = relayErr.JSONOrEmpty() - } else { - // copy body buffer - // do not use bytes conv - detail.ResponseBody = rw.body.String() - } - - return usage, relayErr -} - -func updateUsageMetrics(usage *relaymodel.Usage, meta *meta.Meta, log *log.Entry) { - if usage == nil { - usage = &relaymodel.Usage{ - PromptTokens: meta.InputTokens, - TotalTokens: meta.InputTokens, - } - } - if usage.TotalTokens == 0 { - usage.TotalTokens = usage.PromptTokens + usage.CompletionTokens - } - log.Data["t_input"] = usage.PromptTokens - log.Data["t_output"] = usage.CompletionTokens - log.Data["t_total"] = usage.TotalTokens -} diff --git a/service/aiproxy/relay/controller/handle.go b/service/aiproxy/relay/controller/handle.go deleted file mode 100644 index 268c83c2e..000000000 --- a/service/aiproxy/relay/controller/handle.go +++ /dev/null @@ -1,144 +0,0 @@ -package controller - -import ( - "fmt" - "net/http" - "strconv" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/common/consume" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/channeltype" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func Handle(meta *meta.Meta, c *gin.Context, preProcess func() (*PreCheckGroupBalanceReq, error)) *relaymodel.ErrorWithStatusCode { - log := middleware.GetLogger(c) - - // 1. Get adaptor - adaptor, ok := channeltype.GetAdaptor(meta.Channel.Type) - if !ok { - log.Errorf("invalid (%s[%d]) channel type: %d", meta.Channel.Name, meta.Channel.ID, meta.Channel.Type) - return openai.ErrorWrapperWithMessage( - "invalid channel error", "invalid_channel_type", http.StatusInternalServerError) - } - - // 2. Get group balance - groupRemainBalance, postGroupConsumer, err := getGroupBalance(c, meta) - if err != nil { - log.Errorf("get group (%s) balance failed: %v", meta.Group.ID, err) - errMsg := fmt.Sprintf("get group (%s) balance failed", meta.Group.ID) - consume.AsyncConsume( - nil, - http.StatusInternalServerError, - nil, - meta, - 0, - 0, - errMsg, - c.ClientIP(), - meta.RetryTimes, - nil, - ) - return openai.ErrorWrapperWithMessage( - errMsg, - "get_group_quota_failed", - http.StatusInternalServerError, - ) - } - - if !meta.IsChannelTest && groupRemainBalance <= 0 { - return openai.ErrorWrapperWithMessage(fmt.Sprintf("group (%s) balance not enough", meta.Group.ID), "insufficient_group_balance", http.StatusForbidden) - } - - // 3. Pre-process request - preCheckReq, err := preProcess() - if err != nil { - log.Errorf("pre-process request failed: %s", err.Error()) - detail := &model.RequestDetail{} - if err := getRequestBody(meta, c, detail); err != nil { - log.Errorf("get request body failed: %v", err.Error) - } - consume.AsyncConsume( - nil, - http.StatusBadRequest, - nil, - meta, - 0, - 0, - err.Error(), - c.ClientIP(), - meta.RetryTimes, - detail, - ) - return openai.ErrorWrapper(err, "invalid_request", http.StatusBadRequest) - } - - // 4. Pre-check balance - ok = checkGroupBalance(preCheckReq, meta, groupRemainBalance) - if !ok { - return openai.ErrorWrapperWithMessage(fmt.Sprintf("group (%s) balance is not enough", meta.Group.ID), "insufficient_group_balance", http.StatusForbidden) - } - - meta.InputTokens = preCheckReq.InputTokens - - // 5. Do request - usage, detail, respErr := DoHelper(adaptor, c, meta) - if respErr != nil { - var logDetail *model.RequestDetail - if detail != nil && config.DebugEnabled { - logDetail = detail - log.Errorf( - "handle failed: %+v\nrequest detail:\n%s\nresponse detail:\n%s", - respErr, - logDetail.RequestBody, - logDetail.ResponseBody, - ) - } else { - log.Errorf("handle failed: %+v", respErr) - } - - consume.AsyncConsume( - postGroupConsumer, - respErr.StatusCode, - usage, - meta, - preCheckReq.InputPrice, - preCheckReq.OutputPrice, - respErr.Error.JSONOrEmpty(), - c.ClientIP(), - meta.RetryTimes, - detail, - ) - return respErr - } - - amount := consume.CalculateAmount(usage, preCheckReq.InputPrice, preCheckReq.OutputPrice) - if amount > 0 { - log.Data["amount"] = strconv.FormatFloat(amount, 'f', -1, 64) - } - - if !config.GetSaveAllLogDetail() { - detail = nil - } - - // 6. Post consume - consume.AsyncConsume( - postGroupConsumer, - http.StatusOK, - usage, - meta, - preCheckReq.InputPrice, - preCheckReq.OutputPrice, - "", - c.ClientIP(), - meta.RetryTimes, - detail, - ) - - return nil -} diff --git a/service/aiproxy/relay/controller/image.go b/service/aiproxy/relay/controller/image.go deleted file mode 100644 index 3acaec388..000000000 --- a/service/aiproxy/relay/controller/image.go +++ /dev/null @@ -1,69 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func validateImageMaxBatchSize(modelConfig *model.ModelConfig, batchSize int) error { - if batchSize <= 1 { - return nil - } - if modelConfig.ImageMaxBatchSize <= 0 { - return nil - } - if batchSize > modelConfig.ImageMaxBatchSize { - return fmt.Errorf("batch size %d is greater than the maximum batch size %d", batchSize, modelConfig.ImageMaxBatchSize) - } - return nil -} - -func getImageRequest(meta *meta.Meta, c *gin.Context) (*relaymodel.ImageRequest, error) { - imageRequest, err := utils.UnmarshalImageRequest(c.Request) - if err != nil { - return nil, err - } - if imageRequest.Prompt == "" { - return nil, errors.New("prompt is required") - } - if imageRequest.Size == "" { - return nil, errors.New("size is required") - } - if imageRequest.N == 0 { - imageRequest.N = 1 - } - if err := validateImageMaxBatchSize(meta.ModelConfig, imageRequest.N); err != nil { - return nil, err - } - return imageRequest, nil -} - -func RelayImageHelper(meta *meta.Meta, c *gin.Context) *relaymodel.ErrorWithStatusCode { - return Handle(meta, c, func() (*PreCheckGroupBalanceReq, error) { - if !config.GetBillingEnabled() { - return &PreCheckGroupBalanceReq{}, nil - } - - imageRequest, err := getImageRequest(meta, c) - if err != nil { - return nil, err - } - - imageCostPrice, ok := GetImageSizePrice(meta.ModelConfig, imageRequest.Size) - if !ok { - return nil, fmt.Errorf("invalid image size: %s", imageRequest.Size) - } - - return &PreCheckGroupBalanceReq{ - InputTokens: imageRequest.N, - InputPrice: imageCostPrice, - }, nil - }) -} diff --git a/service/aiproxy/relay/controller/pdf.go b/service/aiproxy/relay/controller/pdf.go deleted file mode 100644 index 47afe7811..000000000 --- a/service/aiproxy/relay/controller/pdf.go +++ /dev/null @@ -1,13 +0,0 @@ -package controller - -import ( - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func RelayParsePdfHelper(meta *meta.Meta, c *gin.Context) *relaymodel.ErrorWithStatusCode { - return Handle(meta, c, func() (*PreCheckGroupBalanceReq, error) { - return &PreCheckGroupBalanceReq{}, nil - }) -} diff --git a/service/aiproxy/relay/controller/price.go b/service/aiproxy/relay/controller/price.go deleted file mode 100644 index 91ae7974e..000000000 --- a/service/aiproxy/relay/controller/price.go +++ /dev/null @@ -1,17 +0,0 @@ -package controller - -import ( - "github.com/labring/sealos/service/aiproxy/model" -) - -func GetModelPrice(modelConfig *model.ModelConfig) (float64, float64, bool) { - return modelConfig.InputPrice, modelConfig.OutputPrice, true -} - -func GetImageSizePrice(modelConfig *model.ModelConfig, size string) (float64, bool) { - if len(modelConfig.ImagePrices) == 0 { - return 0, true - } - price, ok := modelConfig.ImagePrices[size] - return price, ok -} diff --git a/service/aiproxy/relay/controller/rerank.go b/service/aiproxy/relay/controller/rerank.go deleted file mode 100644 index b1325d772..000000000 --- a/service/aiproxy/relay/controller/rerank.go +++ /dev/null @@ -1,59 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "strings" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func RerankHelper(meta *meta.Meta, c *gin.Context) *relaymodel.ErrorWithStatusCode { - return Handle(meta, c, func() (*PreCheckGroupBalanceReq, error) { - if !config.GetBillingEnabled() { - return &PreCheckGroupBalanceReq{}, nil - } - - inputPrice, outputPrice, ok := GetModelPrice(meta.ModelConfig) - if !ok { - return nil, fmt.Errorf("model price not found: %s", meta.OriginModel) - } - - rerankRequest, err := getRerankRequest(c) - if err != nil { - return nil, err - } - - return &PreCheckGroupBalanceReq{ - InputTokens: rerankPromptTokens(rerankRequest), - InputPrice: inputPrice, - OutputPrice: outputPrice, - }, nil - }) -} - -func getRerankRequest(c *gin.Context) (*relaymodel.RerankRequest, error) { - rerankRequest, err := utils.UnmarshalRerankRequest(c.Request) - if err != nil { - return nil, err - } - if rerankRequest.Model == "" { - return nil, errors.New("model parameter must be provided") - } - if rerankRequest.Query == "" { - return nil, errors.New("query must not be empty") - } - if len(rerankRequest.Documents) == 0 { - return nil, errors.New("document list must not be empty") - } - - return rerankRequest, nil -} - -func rerankPromptTokens(rerankRequest *relaymodel.RerankRequest) int { - return len(rerankRequest.Query) + len(strings.Join(rerankRequest.Documents, "")) -} diff --git a/service/aiproxy/relay/controller/stt.go b/service/aiproxy/relay/controller/stt.go deleted file mode 100644 index c93eb0ad6..000000000 --- a/service/aiproxy/relay/controller/stt.go +++ /dev/null @@ -1,106 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "math" - "mime/multipart" - "os" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/audio" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func RelaySTTHelper(meta *meta.Meta, c *gin.Context) *relaymodel.ErrorWithStatusCode { - return Handle(meta, c, func() (*PreCheckGroupBalanceReq, error) { - if !config.GetBillingEnabled() { - return &PreCheckGroupBalanceReq{}, nil - } - - inputPrice, outputPrice, ok := GetModelPrice(meta.ModelConfig) - if !ok { - return nil, fmt.Errorf("model price not found: %s", meta.OriginModel) - } - - audioFile, err := c.FormFile("file") - if err != nil { - return nil, fmt.Errorf("failed to get audio file: %w", err) - } - - duration, err := getAudioDuration(audioFile) - if err != nil { - return nil, err - } - - durationInt := int(math.Ceil(duration)) - log := middleware.GetLogger(c) - log.Data["duration"] = durationInt - - return &PreCheckGroupBalanceReq{ - InputTokens: durationInt, - InputPrice: inputPrice, - OutputPrice: outputPrice, - }, nil - }) -} - -func getAudioDuration(audioFile *multipart.FileHeader) (float64, error) { - // Try to get duration directly from audio data - audioData, err := audioFile.Open() - if err != nil { - return 0, fmt.Errorf("failed to open audio file: %w", err) - } - defer audioData.Close() - - // If it's already an os.File, use file path method - if osFile, ok := audioData.(*os.File); ok { - duration, err := audio.GetAudioDurationFromFilePath(osFile.Name()) - if err != nil { - return 0, fmt.Errorf("failed to get audio duration from temp file: %w", err) - } - return duration, nil - } - - // Try to get duration from audio data - duration, err := audio.GetAudioDuration(audioData) - if err == nil { - return duration, nil - } - - // If duration is NaN, create temp file and try again - if errors.Is(err, audio.ErrAudioDurationNAN) { - return getDurationFromTempFile(audioFile) - } - - return 0, fmt.Errorf("failed to get audio duration: %w", err) -} - -func getDurationFromTempFile(audioFile *multipart.FileHeader) (float64, error) { - tempFile, err := os.CreateTemp("", "audio") - if err != nil { - return 0, fmt.Errorf("failed to create temp file: %w", err) - } - defer os.Remove(tempFile.Name()) - defer tempFile.Close() - - newAudioData, err := audioFile.Open() - if err != nil { - return 0, fmt.Errorf("failed to open audio file: %w", err) - } - defer newAudioData.Close() - - if _, err = tempFile.ReadFrom(newAudioData); err != nil { - return 0, fmt.Errorf("failed to read from temp file: %w", err) - } - - duration, err := audio.GetAudioDurationFromFilePath(tempFile.Name()) - if err != nil { - return 0, fmt.Errorf("failed to get audio duration from temp file: %w", err) - } - - return duration, nil -} diff --git a/service/aiproxy/relay/controller/text.go b/service/aiproxy/relay/controller/text.go deleted file mode 100644 index d94b74474..000000000 --- a/service/aiproxy/relay/controller/text.go +++ /dev/null @@ -1,37 +0,0 @@ -package controller - -import ( - "fmt" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func RelayTextHelper(meta *meta.Meta, c *gin.Context) *relaymodel.ErrorWithStatusCode { - return Handle(meta, c, func() (*PreCheckGroupBalanceReq, error) { - if !config.GetBillingEnabled() { - return &PreCheckGroupBalanceReq{}, nil - } - - inputPrice, outputPrice, ok := GetModelPrice(meta.ModelConfig) - if !ok { - return nil, fmt.Errorf("model price not found: %s", meta.OriginModel) - } - - textRequest, err := utils.UnmarshalGeneralOpenAIRequest(c.Request) - if err != nil { - return nil, err - } - - return &PreCheckGroupBalanceReq{ - InputTokens: openai.GetPromptTokens(meta, textRequest), - MaxTokens: textRequest.MaxTokens, - InputPrice: inputPrice, - OutputPrice: outputPrice, - }, nil - }) -} diff --git a/service/aiproxy/relay/controller/tts.go b/service/aiproxy/relay/controller/tts.go deleted file mode 100644 index 2298ab923..000000000 --- a/service/aiproxy/relay/controller/tts.go +++ /dev/null @@ -1,36 +0,0 @@ -package controller - -import ( - "fmt" - - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/config" - "github.com/labring/sealos/service/aiproxy/relay/adaptor/openai" - "github.com/labring/sealos/service/aiproxy/relay/meta" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/utils" -) - -func RelayTTSHelper(meta *meta.Meta, c *gin.Context) *relaymodel.ErrorWithStatusCode { - return Handle(meta, c, func() (*PreCheckGroupBalanceReq, error) { - if !config.GetBillingEnabled() { - return &PreCheckGroupBalanceReq{}, nil - } - - inputPrice, outputPrice, ok := GetModelPrice(meta.ModelConfig) - if !ok { - return nil, fmt.Errorf("model price not found: %s", meta.OriginModel) - } - - ttsRequest, err := utils.UnmarshalTTSRequest(c.Request) - if err != nil { - return nil, err - } - - return &PreCheckGroupBalanceReq{ - InputTokens: openai.CountTokenText(ttsRequest.Input, meta.ActualModel), - InputPrice: inputPrice, - OutputPrice: outputPrice, - }, nil - }) -} diff --git a/service/aiproxy/relay/meta/meta.go b/service/aiproxy/relay/meta/meta.go deleted file mode 100644 index 438206276..000000000 --- a/service/aiproxy/relay/meta/meta.go +++ /dev/null @@ -1,171 +0,0 @@ -package meta - -import ( - "fmt" - "time" - - "github.com/labring/sealos/service/aiproxy/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type ChannelMeta struct { - Name string - BaseURL string - Key string - ID int - Type int -} - -type Meta struct { - values map[string]any - Channel *ChannelMeta - ChannelConfig model.ChannelConfig - Group *model.GroupCache - Token *model.TokenCache - ModelConfig *model.ModelConfig - - Endpoint string - RequestAt time.Time - RequestID string - OriginModel string - ActualModel string - Mode relaymode.Mode - InputTokens int - IsChannelTest bool - RetryTimes int -} - -type Option func(meta *Meta) - -func WithEndpoint(endpoint string) Option { - return func(meta *Meta) { - meta.Endpoint = endpoint - } -} - -func WithChannelTest(isChannelTest bool) Option { - return func(meta *Meta) { - meta.IsChannelTest = isChannelTest - } -} - -func WithRequestID(requestID string) Option { - return func(meta *Meta) { - meta.RequestID = requestID - } -} - -func WithRequestAt(requestAt time.Time) Option { - return func(meta *Meta) { - meta.RequestAt = requestAt - } -} - -func WithGroup(group *model.GroupCache) Option { - return func(meta *Meta) { - meta.Group = group - } -} - -func WithToken(token *model.TokenCache) Option { - return func(meta *Meta) { - meta.Token = token - } -} - -func WithRetryTimes(retryTimes int) Option { - return func(meta *Meta) { - meta.RetryTimes = retryTimes - } -} - -func NewMeta( - channel *model.Channel, - mode relaymode.Mode, - modelName string, - modelConfig *model.ModelConfig, - opts ...Option, -) *Meta { - meta := Meta{ - values: make(map[string]any), - Mode: mode, - OriginModel: modelName, - ActualModel: modelName, - RequestAt: time.Now(), - ModelConfig: modelConfig, - } - - for _, opt := range opts { - opt(&meta) - } - - if channel != nil { - meta.Channel = &ChannelMeta{ - Name: channel.Name, - BaseURL: channel.BaseURL, - Key: channel.Key, - ID: channel.ID, - Type: channel.Type, - } - if channel.Config != nil { - meta.ChannelConfig = *channel.Config - } - meta.ActualModel, _ = GetMappedModelName(modelName, channel.ModelMapping) - } - - return &meta -} - -func (m *Meta) ClearValues() { - clear(m.values) -} - -func (m *Meta) Set(key string, value any) { - m.values[key] = value -} - -func (m *Meta) Get(key string) (any, bool) { - v, ok := m.values[key] - return v, ok -} - -func (m *Meta) Delete(key string) { - delete(m.values, key) -} - -func (m *Meta) MustGet(key string) any { - v, ok := m.Get(key) - if !ok { - panic(fmt.Sprintf("meta key %s not found", key)) - } - return v -} - -func (m *Meta) GetString(key string) string { - v, ok := m.Get(key) - if !ok { - return "" - } - s, _ := v.(string) - return s -} - -func (m *Meta) GetBool(key string) bool { - v, ok := m.Get(key) - if !ok { - return false - } - b, _ := v.(bool) - return b -} - -func GetMappedModelName(modelName string, mapping map[string]string) (string, bool) { - if len(modelName) == 0 { - return modelName, false - } - mappedModelName := mapping[modelName] - if mappedModelName != "" { - return mappedModelName, true - } - return modelName, false -} diff --git a/service/aiproxy/relay/model/constant.go b/service/aiproxy/relay/model/constant.go deleted file mode 100644 index c9d6d645c..000000000 --- a/service/aiproxy/relay/model/constant.go +++ /dev/null @@ -1,7 +0,0 @@ -package model - -const ( - ContentTypeText = "text" - ContentTypeImageURL = "image_url" - ContentTypeInputAudio = "input_audio" -) diff --git a/service/aiproxy/relay/model/general.go b/service/aiproxy/relay/model/general.go deleted file mode 100644 index 502502385..000000000 --- a/service/aiproxy/relay/model/general.go +++ /dev/null @@ -1,91 +0,0 @@ -package model - -type ResponseFormat struct { - JSONSchema *JSONSchema `json:"json_schema,omitempty"` - Type string `json:"type,omitempty"` -} - -type JSONSchema struct { - Schema map[string]interface{} `json:"schema,omitempty"` - Strict *bool `json:"strict,omitempty"` - Description string `json:"description,omitempty"` - Name string `json:"name"` -} - -type Audio struct { - Voice string `json:"voice,omitempty"` - Format string `json:"format,omitempty"` -} - -type StreamOptions struct { - IncludeUsage bool `json:"include_usage,omitempty"` -} - -type GeneralOpenAIRequest struct { - Prediction any `json:"prediction,omitempty"` - Prompt any `json:"prompt,omitempty"` - Input any `json:"input,omitempty"` - Metadata any `json:"metadata,omitempty"` - Functions any `json:"functions,omitempty"` - LogitBias any `json:"logit_bias,omitempty"` - FunctionCall any `json:"function_call,omitempty"` - ToolChoice any `json:"tool_choice,omitempty"` - Stop any `json:"stop,omitempty"` - MaxCompletionTokens *int `json:"max_completion_tokens,omitempty"` - TopLogprobs *int `json:"top_logprobs,omitempty"` - Style *string `json:"style,omitempty"` - Quality *string `json:"quality,omitempty"` - Audio *Audio `json:"audio,omitempty"` - PresencePenalty *float64 `json:"presence_penalty,omitempty"` - ResponseFormat *ResponseFormat `json:"response_format,omitempty"` - Store *bool `json:"store,omitempty"` - ServiceTier *string `json:"service_tier,omitempty"` - FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` - Logprobs *bool `json:"logprobs,omitempty"` - StreamOptions *StreamOptions `json:"stream_options,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - TopP *float64 `json:"top_p,omitempty"` - ParallelTooCalls *bool `json:"parallel_tool_calls,omitempty"` - EncodingFormat string `json:"encoding_format,omitempty"` - Model string `json:"model,omitempty"` - Instruction string `json:"instruction,omitempty"` - User string `json:"user,omitempty"` - Size string `json:"size,omitempty"` - Modalities []string `json:"modalities,omitempty"` - Messages []*Message `json:"messages,omitempty"` - Tools []*Tool `json:"tools,omitempty"` - N int `json:"n,omitempty"` - Dimensions int `json:"dimensions,omitempty"` - Seed float64 `json:"seed,omitempty"` - MaxTokens int `json:"max_tokens,omitempty"` - TopK int `json:"top_k,omitempty"` - NumCtx int `json:"num_ctx,omitempty"` - Stream bool `json:"stream,omitempty"` -} - -func (r GeneralOpenAIRequest) ParseInput() []string { - if r.Input == nil { - return nil - } - var input []string - switch v := r.Input.(type) { - case string: - input = []string{v} - case []any: - input = make([]string, 0, len(v)) - for _, item := range v { - if str, ok := item.(string); ok { - input = append(input, str) - } - } - } - return input -} - -type TextToSpeechRequest struct { - Model string `binding:"required" json:"model"` - Input string `binding:"required" json:"input"` - Voice string `binding:"required" json:"voice"` - ResponseFormat string `json:"response_format"` - Speed float64 `json:"speed"` -} diff --git a/service/aiproxy/relay/model/image.go b/service/aiproxy/relay/model/image.go deleted file mode 100644 index 945e67bb5..000000000 --- a/service/aiproxy/relay/model/image.go +++ /dev/null @@ -1,12 +0,0 @@ -package model - -type ImageRequest struct { - Model string `json:"model"` - Prompt string `json:"prompt"` - Size string `json:"size,omitempty"` - Quality string `json:"quality,omitempty"` - ResponseFormat string `json:"response_format,omitempty"` - Style string `json:"style,omitempty"` - User string `json:"user,omitempty"` - N int `json:"n,omitempty"` -} diff --git a/service/aiproxy/relay/model/message.go b/service/aiproxy/relay/model/message.go deleted file mode 100644 index fab5e7942..000000000 --- a/service/aiproxy/relay/model/message.go +++ /dev/null @@ -1,106 +0,0 @@ -package model - -import "strings" - -type Message struct { - Content any `json:"content,omitempty"` - ReasoningContent string `json:"reasoning_content,omitempty"` - Name *string `json:"name,omitempty"` - Role string `json:"role,omitempty"` - ToolCallID string `json:"tool_call_id,omitempty"` - ToolCalls []*Tool `json:"tool_calls,omitempty"` -} - -func (m *Message) IsStringContent() bool { - _, ok := m.Content.(string) - return ok -} - -func (m *Message) ToStringContentMessage() { - if m.IsStringContent() { - return - } - m.Content = m.StringContent() -} - -func (m *Message) StringContent() string { - if m.ReasoningContent != "" { - return m.ReasoningContent - } - - content, ok := m.Content.(string) - if ok { - return content - } - contentList, ok := m.Content.([]any) - if !ok { - return "" - } - - var strBuilder strings.Builder - for _, contentItem := range contentList { - contentMap, ok := contentItem.(map[string]any) - if !ok { - continue - } - if contentMap["type"] == ContentTypeText { - if subStr, ok := contentMap["text"].(string); ok { - strBuilder.WriteString(subStr) - strBuilder.WriteString("\n") - } - } - } - return strBuilder.String() -} - -func (m *Message) ParseContent() []MessageContent { - var contentList []MessageContent - content, ok := m.Content.(string) - if ok { - contentList = append(contentList, MessageContent{ - Type: ContentTypeText, - Text: content, - }) - return contentList - } - anyList, ok := m.Content.([]any) - if ok { - for _, contentItem := range anyList { - contentMap, ok := contentItem.(map[string]any) - if !ok { - continue - } - switch contentMap["type"] { - case ContentTypeText: - if subStr, ok := contentMap["text"].(string); ok { - contentList = append(contentList, MessageContent{ - Type: ContentTypeText, - Text: subStr, - }) - } - case ContentTypeImageURL: - if subObj, ok := contentMap["image_url"].(map[string]any); ok { - contentList = append(contentList, MessageContent{ - Type: ContentTypeImageURL, - ImageURL: &ImageURL{ - URL: subObj["url"].(string), - }, - }) - } - } - } - return contentList - } - return nil -} - -type ImageURL struct { - URL string `json:"url,omitempty"` - Detail string `json:"detail,omitempty"` -} - -type MessageContent struct { - ImageURL *ImageURL `json:"image_url,omitempty"` - Type string `json:"type,omitempty"` - Text string `json:"text"` -} diff --git a/service/aiproxy/relay/model/misc.go b/service/aiproxy/relay/model/misc.go deleted file mode 100644 index 5330b103b..000000000 --- a/service/aiproxy/relay/model/misc.go +++ /dev/null @@ -1,50 +0,0 @@ -package model - -import ( - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/common/conv" -) - -type Usage struct { - PromptTokens int `json:"prompt_tokens"` - CompletionTokens int `json:"completion_tokens"` - TotalTokens int `json:"total_tokens"` -} - -type Error struct { - Code any `json:"code,omitempty"` - Message string `json:"message,omitempty"` - Type string `json:"type,omitempty"` - Param string `json:"param,omitempty"` -} - -func (e *Error) IsEmpty() bool { - return e == nil || (e.Code == nil && e.Message == "" && e.Type == "" && e.Param == "") -} - -func (e *Error) JSONOrEmpty() string { - if e.IsEmpty() { - return "" - } - jsonBuf, err := sonic.Marshal(e) - if err != nil { - return "" - } - return conv.BytesToString(jsonBuf) -} - -type ErrorWithStatusCode struct { - Error Error `json:"error,omitempty"` - StatusCode int `json:"-"` -} - -func (e *ErrorWithStatusCode) JSONOrEmpty() string { - if e.StatusCode == 0 && e.Error.IsEmpty() { - return "" - } - jsonBuf, err := sonic.MarshalString(e) - if err != nil { - return "" - } - return jsonBuf -} diff --git a/service/aiproxy/relay/model/pdf.go b/service/aiproxy/relay/model/pdf.go deleted file mode 100644 index 63d775cda..000000000 --- a/service/aiproxy/relay/model/pdf.go +++ /dev/null @@ -1,10 +0,0 @@ -package model - -type ParsePdfResponse struct { - Pages int `json:"pages"` - Markdown string `json:"markdown"` -} - -type ParsePdfListResponse struct { - Markdowns []string `json:"markdowns"` -} diff --git a/service/aiproxy/relay/model/rerank.go b/service/aiproxy/relay/model/rerank.go deleted file mode 100644 index 1bbdf5dc5..000000000 --- a/service/aiproxy/relay/model/rerank.go +++ /dev/null @@ -1,37 +0,0 @@ -package model - -type RerankRequest struct { - TopN *int `json:"top_n,omitempty"` - MaxChunksPerDoc *int `json:"max_chunks_per_doc,omitempty"` - ReturnDocuments *bool `json:"return_documents,omitempty"` - OverlapTokens *int `json:"overlap_tokens,omitempty"` - Model string `json:"model"` - Query string `json:"query"` - Documents []string `json:"documents"` -} - -type Document struct { - Text string `json:"text"` -} - -type RerankResult struct { - Document *Document `json:"document,omitempty"` - Index int `json:"index"` - RelevanceScore float64 `json:"relevance_score"` -} - -type RerankMetaTokens struct { - InputTokens int `json:"input_tokens"` - OutputTokens int `json:"output_tokens"` -} - -type RerankMeta struct { - Tokens *RerankMetaTokens `json:"tokens,omitempty"` - Model string `json:"model,omitempty"` -} - -type RerankResponse struct { - Meta RerankMeta `json:"meta"` - ID string `json:"id"` - Result []*RerankResult `json:"result"` -} diff --git a/service/aiproxy/relay/model/tool.go b/service/aiproxy/relay/model/tool.go deleted file mode 100644 index 5a25e419d..000000000 --- a/service/aiproxy/relay/model/tool.go +++ /dev/null @@ -1,14 +0,0 @@ -package model - -type Tool struct { - ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` // when splicing claude tools stream messages, it is empty - Function Function `json:"function"` -} - -type Function struct { - Parameters any `json:"parameters,omitempty"` - Arguments string `json:"arguments,omitempty"` - Description string `json:"description,omitempty"` - Name string `json:"name,omitempty"` -} diff --git a/service/aiproxy/relay/relaymode/define.go b/service/aiproxy/relay/relaymode/define.go deleted file mode 100644 index 931ab0675..000000000 --- a/service/aiproxy/relay/relaymode/define.go +++ /dev/null @@ -1,51 +0,0 @@ -package relaymode - -import "fmt" - -type Mode int - -func (m Mode) String() string { - switch m { - case Unknown: - return "Unknown" - case ChatCompletions: - return "ChatCompletions" - case Completions: - return "Completions" - case Embeddings: - return "Embeddings" - case Moderations: - return "Moderations" - case ImagesGenerations: - return "ImagesGenerations" - case Edits: - return "Edits" - case AudioSpeech: - return "AudioSpeech" - case AudioTranscription: - return "AudioTranscription" - case AudioTranslation: - return "AudioTranslation" - case Rerank: - return "Rerank" - case ParsePdf: - return "ParsePdf" - default: - return fmt.Sprintf("Mode(%d)", m) - } -} - -const ( - Unknown Mode = iota - ChatCompletions - Completions - Embeddings - Moderations - ImagesGenerations - Edits - AudioSpeech - AudioTranscription - AudioTranslation - Rerank - ParsePdf -) diff --git a/service/aiproxy/relay/utils/testreq.go b/service/aiproxy/relay/utils/testreq.go deleted file mode 100644 index 33f8e83a8..000000000 --- a/service/aiproxy/relay/utils/testreq.go +++ /dev/null @@ -1,161 +0,0 @@ -package utils - -import ( - "bytes" - "fmt" - "io" - - "github.com/bytedance/sonic" - "github.com/labring/sealos/service/aiproxy/model" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -type UnsupportedModelTypeError struct { - ModelType string -} - -func (e *UnsupportedModelTypeError) Error() string { - return fmt.Sprintf("model type '%s' not supported", e.ModelType) -} - -func NewErrUnsupportedModelType(modelType string) *UnsupportedModelTypeError { - return &UnsupportedModelTypeError{ModelType: modelType} -} - -func BuildRequest(modelConfig *model.ModelConfig) (io.Reader, relaymode.Mode, error) { - switch modelConfig.Type { - case relaymode.ChatCompletions: - body, err := BuildChatCompletionRequest(modelConfig.Model) - if err != nil { - return nil, relaymode.Unknown, err - } - return body, relaymode.ChatCompletions, nil - case relaymode.Completions: - return nil, relaymode.Unknown, NewErrUnsupportedModelType("completions") - case relaymode.Embeddings: - body, err := BuildEmbeddingsRequest(modelConfig.Model) - if err != nil { - return nil, relaymode.Unknown, err - } - return body, relaymode.Embeddings, nil - case relaymode.Moderations: - body, err := BuildModerationsRequest(modelConfig.Model) - if err != nil { - return nil, relaymode.Unknown, err - } - return body, relaymode.Moderations, nil - case relaymode.ImagesGenerations: - body, err := BuildImagesGenerationsRequest(modelConfig) - if err != nil { - return nil, relaymode.Unknown, err - } - return body, relaymode.ImagesGenerations, nil - case relaymode.Edits: - return nil, relaymode.Unknown, NewErrUnsupportedModelType("edits") - case relaymode.AudioSpeech: - body, err := BuildAudioSpeechRequest(modelConfig.Model) - if err != nil { - return nil, relaymode.Unknown, err - } - return body, relaymode.AudioSpeech, nil - case relaymode.AudioTranscription: - return nil, relaymode.Unknown, NewErrUnsupportedModelType("audio transcription") - case relaymode.AudioTranslation: - return nil, relaymode.Unknown, NewErrUnsupportedModelType("audio translation") - case relaymode.Rerank: - body, err := BuildRerankRequest(modelConfig.Model) - if err != nil { - return nil, relaymode.Unknown, err - } - return body, relaymode.Rerank, nil - case relaymode.ParsePdf: - return nil, relaymode.Unknown, NewErrUnsupportedModelType("parse pdf") - default: - return nil, relaymode.Unknown, NewErrUnsupportedModelType(modelConfig.Type.String()) - } -} - -func BuildChatCompletionRequest(model string) (io.Reader, error) { - testRequest := &relaymodel.GeneralOpenAIRequest{ - MaxTokens: 2, - Model: model, - Messages: []*relaymodel.Message{ - { - Role: "user", - Content: "hi", - }, - }, - } - jsonBytes, err := sonic.Marshal(testRequest) - if err != nil { - return nil, err - } - return bytes.NewReader(jsonBytes), nil -} - -func BuildEmbeddingsRequest(model string) (io.Reader, error) { - embeddingsRequest := &relaymodel.GeneralOpenAIRequest{ - Model: model, - Input: "hi", - } - jsonBytes, err := sonic.Marshal(embeddingsRequest) - if err != nil { - return nil, err - } - return bytes.NewReader(jsonBytes), nil -} - -func BuildModerationsRequest(model string) (io.Reader, error) { - moderationsRequest := &relaymodel.GeneralOpenAIRequest{ - Model: model, - Input: "hi", - } - jsonBytes, err := sonic.Marshal(moderationsRequest) - if err != nil { - return nil, err - } - return bytes.NewReader(jsonBytes), nil -} - -func BuildImagesGenerationsRequest(modelConfig *model.ModelConfig) (io.Reader, error) { - imagesGenerationsRequest := &relaymodel.GeneralOpenAIRequest{ - Model: modelConfig.Model, - Prompt: "hi", - Size: "1024x1024", - } - for size := range modelConfig.ImagePrices { - imagesGenerationsRequest.Size = size - break - } - jsonBytes, err := sonic.Marshal(imagesGenerationsRequest) - if err != nil { - return nil, err - } - return bytes.NewReader(jsonBytes), nil -} - -func BuildAudioSpeechRequest(model string) (io.Reader, error) { - audioSpeechRequest := &relaymodel.GeneralOpenAIRequest{ - Model: model, - Input: "hi", - } - jsonBytes, err := sonic.Marshal(audioSpeechRequest) - if err != nil { - return nil, err - } - return bytes.NewReader(jsonBytes), nil -} - -func BuildRerankRequest(model string) (io.Reader, error) { - rerankRequest := &relaymodel.RerankRequest{ - Model: model, - Query: "hi", - Documents: []string{"hi"}, - } - jsonBytes, err := sonic.Marshal(rerankRequest) - if err != nil { - return nil, err - } - return bytes.NewReader(jsonBytes), nil -} diff --git a/service/aiproxy/relay/utils/utils.go b/service/aiproxy/relay/utils/utils.go deleted file mode 100644 index 74b3d1570..000000000 --- a/service/aiproxy/relay/utils/utils.go +++ /dev/null @@ -1,73 +0,0 @@ -package utils - -import ( - "net/http" - "strings" - - "github.com/labring/sealos/service/aiproxy/common" - relaymodel "github.com/labring/sealos/service/aiproxy/relay/model" -) - -func UnmarshalGeneralOpenAIRequest(req *http.Request) (*relaymodel.GeneralOpenAIRequest, error) { - var request relaymodel.GeneralOpenAIRequest - err := common.UnmarshalBodyReusable(req, &request) - if err != nil { - return nil, err - } - return &request, nil -} - -func UnmarshalImageRequest(req *http.Request) (*relaymodel.ImageRequest, error) { - var request relaymodel.ImageRequest - err := common.UnmarshalBodyReusable(req, &request) - if err != nil { - return nil, err - } - return &request, nil -} - -func UnmarshalRerankRequest(req *http.Request) (*relaymodel.RerankRequest, error) { - var request relaymodel.RerankRequest - err := common.UnmarshalBodyReusable(req, &request) - if err != nil { - return nil, err - } - return &request, nil -} - -func UnmarshalTTSRequest(req *http.Request) (*relaymodel.TextToSpeechRequest, error) { - var request relaymodel.TextToSpeechRequest - err := common.UnmarshalBodyReusable(req, &request) - if err != nil { - return nil, err - } - return &request, nil -} - -func UnmarshalMap(req *http.Request) (map[string]any, error) { - var request map[string]any - err := common.UnmarshalBodyReusable(req, &request) - if err != nil { - return nil, err - } - return request, nil -} - -var defaultClient = &http.Client{} - -func DoRequest(req *http.Request) (*http.Response, error) { - resp, err := defaultClient.Do(req) - if err != nil { - return nil, err - } - return resp, nil -} - -func IsStreamResponse(resp *http.Response) bool { - contentType := resp.Header.Get("Content-Type") - if contentType == "" { - return false - } - return strings.Contains(contentType, "event-stream") || - strings.Contains(contentType, "x-ndjson") -} diff --git a/service/aiproxy/router/api.go b/service/aiproxy/router/api.go deleted file mode 100644 index b554a036e..000000000 --- a/service/aiproxy/router/api.go +++ /dev/null @@ -1,173 +0,0 @@ -package router - -import ( - "github.com/gin-contrib/gzip" - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/common/env" - "github.com/labring/sealos/service/aiproxy/controller" - "github.com/labring/sealos/service/aiproxy/middleware" -) - -func SetAPIRouter(router *gin.Engine) { - api := router.Group("/api") - if env.Bool("GZIP_ENABLED", false) { - api.Use(gzip.Gzip(gzip.DefaultCompression)) - } - - healthRouter := api.Group("") - healthRouter.GET("/status", controller.GetStatus) - - apiRouter := api.Group("") - apiRouter.Use(middleware.AdminAuth) - { - modelsRoute := apiRouter.Group("/models") - { - modelsRoute.GET("/builtin", controller.BuiltinModels) - modelsRoute.GET("/builtin/channel", controller.ChannelBuiltinModels) - modelsRoute.GET("/builtin/channel/:type", controller.ChannelBuiltinModelsByType) - modelsRoute.GET("/enabled", controller.EnabledModels) - modelsRoute.GET("/enabled/channel", controller.ChannelEnabledModels) - modelsRoute.GET("/enabled/channel/:type", controller.ChannelEnabledModelsByType) - modelsRoute.GET("/default", controller.ChannelDefaultModelsAndMapping) - modelsRoute.GET("/default/:type", controller.ChannelDefaultModelsAndMappingByType) - } - - dashboardRoute := apiRouter.Group("/dashboard") - { - dashboardRoute.GET("/", controller.GetDashboard) - dashboardRoute.GET("/:group", controller.GetGroupDashboard) - dashboardRoute.GET("/:group/models", controller.GetGroupDashboardModels) - } - - modelCostRankRoute := apiRouter.Group("/model_cost_rank") - { - modelCostRankRoute.GET("/", controller.GetModelCostRank) - modelCostRankRoute.GET("/:group", controller.GetGroupModelCostRank) - } - - groupsRoute := apiRouter.Group("/groups") - { - groupsRoute.GET("/", controller.GetGroups) - groupsRoute.GET("/search", controller.SearchGroups) - groupsRoute.POST("/batch_delete", controller.DeleteGroups) - } - groupRoute := apiRouter.Group("/group") - { - groupRoute.POST("/:group", controller.CreateGroup) - groupRoute.PUT("/:group", controller.UpdateGroup) - groupRoute.GET("/:group", controller.GetGroup) - groupRoute.DELETE("/:group", controller.DeleteGroup) - groupRoute.POST("/:group/status", controller.UpdateGroupStatus) - groupRoute.POST("/:group/rpm_ratio", controller.UpdateGroupRPMRatio) - groupRoute.POST("/:group/rpm", controller.UpdateGroupRPM) - groupRoute.POST("/:group/tpm_ratio", controller.UpdateGroupTPMRatio) - groupRoute.POST("/:group/tpm", controller.UpdateGroupTPM) - } - - optionRoute := apiRouter.Group("/option") - { - optionRoute.GET("/", controller.GetOptions) - optionRoute.GET("/:key", controller.GetOption) - optionRoute.PUT("/", controller.UpdateOption) - optionRoute.PUT("/batch", controller.UpdateOptions) - } - - channelsRoute := apiRouter.Group("/channels") - { - channelsRoute.GET("/", controller.GetChannels) - channelsRoute.GET("/all", controller.GetAllChannels) - channelsRoute.GET("/type_names", controller.ChannelTypeNames) - channelsRoute.GET("/type_metas", controller.ChannelTypeMetas) - channelsRoute.POST("/", controller.AddChannels) - channelsRoute.GET("/search", controller.SearchChannels) - channelsRoute.GET("/update_balance", controller.UpdateAllChannelsBalance) - channelsRoute.POST("/batch_delete", controller.DeleteChannels) - channelsRoute.GET("/test", controller.TestAllChannels) - - importRoute := channelsRoute.Group("/import") - { - importRoute.POST("/oneapi", controller.ImportChannelFromOneAPI) - } - } - channelRoute := apiRouter.Group("/channel") - { - channelRoute.GET("/:id", controller.GetChannel) - channelRoute.POST("/", controller.AddChannel) - channelRoute.PUT("/:id", controller.UpdateChannel) - channelRoute.POST("/:id/status", controller.UpdateChannelStatus) - channelRoute.DELETE("/:id", controller.DeleteChannel) - channelRoute.GET("/:id/test", controller.TestChannelModels) - channelRoute.GET("/:id/test/:model", controller.TestChannel) - channelRoute.GET("/:id/update_balance", controller.UpdateChannelBalance) - } - - tokensRoute := apiRouter.Group("/tokens") - { - tokensRoute.GET("/", controller.GetTokens) - tokensRoute.GET("/:id", controller.GetToken) - tokensRoute.PUT("/:id", controller.UpdateToken) - tokensRoute.POST("/:id/status", controller.UpdateTokenStatus) - tokensRoute.POST("/:id/name", controller.UpdateTokenName) - tokensRoute.DELETE("/:id", controller.DeleteToken) - tokensRoute.GET("/search", controller.SearchTokens) - tokensRoute.POST("/batch_delete", controller.DeleteTokens) - } - tokenRoute := apiRouter.Group("/token") - { - tokenRoute.GET("/:group/search", controller.SearchGroupTokens) - tokenRoute.POST("/:group/batch_delete", controller.DeleteGroupTokens) - tokenRoute.GET("/:group", controller.GetGroupTokens) - tokenRoute.GET("/:group/:id", controller.GetGroupToken) - tokenRoute.POST("/:group", controller.AddGroupToken) - tokenRoute.PUT("/:group/:id", controller.UpdateGroupToken) - tokenRoute.POST("/:group/:id/status", controller.UpdateGroupTokenStatus) - tokenRoute.POST("/:group/:id/name", controller.UpdateGroupTokenName) - tokenRoute.DELETE("/:group/:id", controller.DeleteGroupToken) - } - - logsRoute := apiRouter.Group("/logs") - { - logsRoute.GET("/", controller.GetLogs) - logsRoute.DELETE("/", controller.DeleteHistoryLogs) - logsRoute.GET("/search", controller.SearchLogs) - logsRoute.GET("/consume_error", controller.SearchConsumeError) - logsRoute.GET("/detail/:log_id", controller.GetLogDetail) - logsRoute.GET("/used/models", controller.GetUsedModels) - } - logRoute := apiRouter.Group("/log") - { - logRoute.GET("/:group", controller.GetGroupLogs) - logRoute.GET("/:group/search", controller.SearchGroupLogs) - logRoute.GET("/:group/detail/:log_id", controller.GetGroupLogDetail) - logRoute.GET("/:group/used/models", controller.GetGroupUsedModels) - logRoute.GET("/:group/used/token_names", controller.GetGroupUsedTokenNames) - } - - modelConfigsRoute := apiRouter.Group("/model_configs") - { - modelConfigsRoute.GET("/", controller.GetModelConfigs) - modelConfigsRoute.GET("/search", controller.SearchModelConfigs) - modelConfigsRoute.GET("/all", controller.GetAllModelConfigs) - modelConfigsRoute.POST("/contains", controller.GetModelConfigsByModelsContains) - modelConfigsRoute.POST("/", controller.SaveModelConfigs) - modelConfigsRoute.POST("/batch_delete", controller.DeleteModelConfigs) - } - modelConfigRoute := apiRouter.Group("/model_config") - { - modelConfigRoute.GET("/:model", controller.GetModelConfig) - modelConfigRoute.POST("/", controller.SaveModelConfig) - modelConfigRoute.DELETE("/:model", controller.DeleteModelConfig) - } - - monitorRoute := apiRouter.Group("/monitor") - { - monitorRoute.GET("/", controller.GetAllChannelModelErrorRates) - monitorRoute.GET("/:id", controller.GetChannelModelErrorRates) - monitorRoute.DELETE("/", controller.ClearAllModelErrors) - monitorRoute.DELETE("/:id", controller.ClearChannelAllModelErrors) - monitorRoute.DELETE("/:id/:model", controller.ClearChannelModelErrors) - monitorRoute.GET("/models", controller.GetModelsErrorRate) - monitorRoute.GET("/banned_channels", controller.GetAllBannedModelChannels) - } - } -} diff --git a/service/aiproxy/router/main.go b/service/aiproxy/router/main.go deleted file mode 100644 index a704ab8ec..000000000 --- a/service/aiproxy/router/main.go +++ /dev/null @@ -1,10 +0,0 @@ -package router - -import ( - "github.com/gin-gonic/gin" -) - -func SetRouter(router *gin.Engine) { - SetAPIRouter(router) - SetRelayRouter(router) -} diff --git a/service/aiproxy/router/relay.go b/service/aiproxy/router/relay.go deleted file mode 100644 index 3f03f9233..000000000 --- a/service/aiproxy/router/relay.go +++ /dev/null @@ -1,129 +0,0 @@ -package router - -import ( - "github.com/gin-gonic/gin" - "github.com/labring/sealos/service/aiproxy/controller" - "github.com/labring/sealos/service/aiproxy/middleware" - "github.com/labring/sealos/service/aiproxy/relay/relaymode" -) - -func SetRelayRouter(router *gin.Engine) { - // https://platform.openai.com/docs/api-reference/introduction - v1Router := router.Group("/v1") - v1Router.Use(middleware.TokenAuth) - - modelsRouter := v1Router.Group("/models") - { - modelsRouter.GET("", controller.ListModels) - modelsRouter.GET("/:model", controller.RetrieveModel) - } - dashboardRouter := v1Router.Group("/dashboard") - { - dashboardRouter.GET("/billing/subscription", controller.GetSubscription) - dashboardRouter.GET("/billing/usage", controller.GetUsage) - } - relayRouter := v1Router.Group("") - { - relayRouter.POST( - "/completions", - middleware.NewDistribute(relaymode.Completions), - controller.NewRelay(relaymode.Completions), - ) - - relayRouter.POST( - "/chat/completions", - middleware.NewDistribute(relaymode.ChatCompletions), - controller.NewRelay(relaymode.ChatCompletions), - ) - relayRouter.POST( - "/edits", - middleware.NewDistribute(relaymode.Edits), - controller.NewRelay(relaymode.Edits), - ) - relayRouter.POST( - "/images/generations", - middleware.NewDistribute(relaymode.ImagesGenerations), - controller.NewRelay(relaymode.ImagesGenerations), - ) - relayRouter.POST( - "/embeddings", - middleware.NewDistribute(relaymode.Embeddings), - controller.NewRelay(relaymode.Embeddings), - ) - relayRouter.POST( - "/engines/:model/embeddings", - middleware.NewDistribute(relaymode.Embeddings), - controller.NewRelay(relaymode.Embeddings), - ) - relayRouter.POST( - "/audio/transcriptions", - middleware.NewDistribute(relaymode.AudioTranscription), - controller.NewRelay(relaymode.AudioTranscription), - ) - relayRouter.POST( - "/audio/translations", - middleware.NewDistribute(relaymode.AudioTranslation), - controller.NewRelay(relaymode.AudioTranslation), - ) - relayRouter.POST( - "/audio/speech", - middleware.NewDistribute(relaymode.AudioSpeech), - controller.NewRelay(relaymode.AudioSpeech), - ) - relayRouter.POST( - "/rerank", - middleware.NewDistribute(relaymode.Rerank), - controller.NewRelay(relaymode.Rerank), - ) - relayRouter.POST( - "/moderations", - middleware.NewDistribute(relaymode.Moderations), - controller.NewRelay(relaymode.Moderations), - ) - relayRouter.POST( - "/parse/pdf", - middleware.NewDistribute(relaymode.ParsePdf), - controller.NewRelay(relaymode.ParsePdf), - ) - - relayRouter.POST("/images/edits", controller.RelayNotImplemented) - relayRouter.POST("/images/variations", controller.RelayNotImplemented) - relayRouter.GET("/files", controller.RelayNotImplemented) - relayRouter.POST("/files", controller.RelayNotImplemented) - relayRouter.DELETE("/files/:id", controller.RelayNotImplemented) - relayRouter.GET("/files/:id", controller.RelayNotImplemented) - relayRouter.GET("/files/:id/content", controller.RelayNotImplemented) - relayRouter.POST("/fine_tuning/jobs", controller.RelayNotImplemented) - relayRouter.GET("/fine_tuning/jobs", controller.RelayNotImplemented) - relayRouter.GET("/fine_tuning/jobs/:id", controller.RelayNotImplemented) - relayRouter.POST("/fine_tuning/jobs/:id/cancel", controller.RelayNotImplemented) - relayRouter.GET("/fine_tuning/jobs/:id/events", controller.RelayNotImplemented) - relayRouter.DELETE("/models/:model", controller.RelayNotImplemented) - relayRouter.POST("/assistants", controller.RelayNotImplemented) - relayRouter.GET("/assistants/:id", controller.RelayNotImplemented) - relayRouter.POST("/assistants/:id", controller.RelayNotImplemented) - relayRouter.DELETE("/assistants/:id", controller.RelayNotImplemented) - relayRouter.GET("/assistants", controller.RelayNotImplemented) - relayRouter.POST("/assistants/:id/files", controller.RelayNotImplemented) - relayRouter.GET("/assistants/:id/files/:fileId", controller.RelayNotImplemented) - relayRouter.DELETE("/assistants/:id/files/:fileId", controller.RelayNotImplemented) - relayRouter.GET("/assistants/:id/files", controller.RelayNotImplemented) - relayRouter.POST("/threads", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id", controller.RelayNotImplemented) - relayRouter.DELETE("/threads/:id", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id/messages", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/messages/:messageId", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id/messages/:messageId", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/messages/:messageId/files/:filesId", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/messages/:messageId/files", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id/runs", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/runs/:runsId", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id/runs/:runsId", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/runs", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id/runs/:runsId/submit_tool_outputs", controller.RelayNotImplemented) - relayRouter.POST("/threads/:id/runs/:runsId/cancel", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/runs/:runsId/steps/:stepId", controller.RelayNotImplemented) - relayRouter.GET("/threads/:id/runs/:runsId/steps", controller.RelayNotImplemented) - } -} diff --git a/service/go.sum b/service/go.sum index 42d19600f..464c08394 100644 --- a/service/go.sum +++ b/service/go.sum @@ -49,6 +49,7 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -87,6 +88,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -120,6 +122,7 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/service/go.work b/service/go.work index eaab1dd40..b5c1193cc 100644 --- a/service/go.work +++ b/service/go.work @@ -3,7 +3,6 @@ go 1.22.7 use ( . ./account - ./aiproxy ./database ./exceptionmonitor ./launchpad diff --git a/service/go.work.sum b/service/go.work.sum index c2539a051..21f5e8bc2 100644 --- a/service/go.work.sum +++ b/service/go.work.sum @@ -953,6 +953,8 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -1777,6 +1779,9 @@ golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808 h1:+Kc94D8UVEVxJnLXp/+ golang.org/x/telemetry v0.0.0-20240208230135-b75ee8823808/go.mod h1:KG1lNk5ZFNssSZLrpVb4sMXKMpGwGXOxSG3rnu2gZQQ= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=