diff --git a/backend/internal/service/idempotency.go b/backend/internal/service/idempotency.go index 2a86bd6038..4241468789 100644 --- a/backend/internal/service/idempotency.go +++ b/backend/internal/service/idempotency.go @@ -11,6 +11,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/util/logredact" @@ -454,11 +455,21 @@ func (c *IdempotencyCoordinator) marshalStoredResponse(data any) (string, error) } redacted := logredact.RedactText(string(raw)) if c.cfg.MaxStoredResponseLen > 0 && len(redacted) > c.cfg.MaxStoredResponseLen { - redacted = redacted[:c.cfg.MaxStoredResponseLen] + "...(truncated)" + redacted = truncateUTF8(redacted, c.cfg.MaxStoredResponseLen) + "...(truncated)" } return redacted, nil } +func truncateUTF8(s string, maxBytes int) string { + if maxBytes <= 0 || len(s) <= maxBytes { + return s + } + for maxBytes > 0 && !utf8.ValidString(s[:maxBytes]) { + maxBytes-- + } + return s[:maxBytes] +} + func (c *IdempotencyCoordinator) decodeStoredResponse(stored *string) (any, error) { if stored == nil || strings.TrimSpace(*stored) == "" { return map[string]any{}, nil diff --git a/backend/internal/service/idempotency_test.go b/backend/internal/service/idempotency_test.go index 6ff75d1c3e..e11c9e75f4 100644 --- a/backend/internal/service/idempotency_test.go +++ b/backend/internal/service/idempotency_test.go @@ -3,10 +3,12 @@ package service import ( "context" "errors" + "strings" "sync" "sync/atomic" "testing" "time" + "unicode/utf8" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/stretchr/testify/require" @@ -441,6 +443,51 @@ func TestIdempotencyCoordinator_StoreUnavailableMetrics(t *testing.T) { require.GreaterOrEqual(t, GetIdempotencyMetricsSnapshot().StoreUnavailableTotal, uint64(1)) } +type utf8RejectingIdempotencyRepo struct { + inMemoryIdempotencyRepo +} + +func newUTF8RejectingIdempotencyRepo() *utf8RejectingIdempotencyRepo { + return &utf8RejectingIdempotencyRepo{inMemoryIdempotencyRepo: *newInMemoryIdempotencyRepo()} +} + +func (r *utf8RejectingIdempotencyRepo) MarkSucceeded(ctx context.Context, id int64, responseStatus int, responseBody string, expiresAt time.Time) error { + if !utf8.ValidString(responseBody) { + return errors.New(`pq: invalid byte sequence for encoding "UTF8": 0xe8 0xb4 0x2e`) + } + return r.inMemoryIdempotencyRepo.MarkSucceeded(ctx, id, responseStatus, responseBody, expiresAt) +} + +func TestIdempotencyCoordinator_TruncatedStoredResponseRemainsUTF8(t *testing.T) { + repo := newUTF8RejectingIdempotencyRepo() + cfg := DefaultIdempotencyConfig() + cfg.MaxStoredResponseLen = len(`{"message":"`) + 2 + coordinator := NewIdempotencyCoordinator(repo, cfg) + + opts := IdempotencyExecuteOptions{ + Scope: "test.scope.truncate_utf8", + Method: "POST", + Route: "/api/v1/accounts/import/codex-session", + ActorScope: "admin:1", + RequireKey: true, + IdempotencyKey: "truncate-utf8", + Payload: map[string]any{"content": "codex-session"}, + } + + result, err := coordinator.Execute(context.Background(), opts, func(ctx context.Context) (any, error) { + return map[string]any{"message": strings.Repeat("\u8d26", 8)}, nil + }) + require.NoError(t, err) + require.NotNil(t, result) + + stored, err := repo.GetByScopeAndKeyHash(context.Background(), opts.Scope, HashIdempotencyKey(opts.IdempotencyKey)) + require.NoError(t, err) + require.NotNil(t, stored) + require.NotNil(t, stored.ResponseBody) + require.True(t, utf8.ValidString(*stored.ResponseBody)) + require.Contains(t, *stored.ResponseBody, "...(truncated)") +} + func TestDefaultIdempotencyCoordinatorAndTTLs(t *testing.T) { SetDefaultIdempotencyCoordinator(nil) require.Nil(t, DefaultIdempotencyCoordinator())