mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: add chat debug retention purge (#24943)
> Mux is acting on Mike's behalf. Adds configurable retention for chat debug data, including the purge query, updated_at index, site config, experimental API, SDK types, frontend lifecycle setting, and docs. The purge deletes debug runs older than the configured retention window and relies on existing cascades to delete steps. The default retention is 30 days, and setting the value to 0 disables the purge.
This commit is contained in:
@@ -1216,6 +1216,8 @@ func New(options *Options) *API {
|
||||
r.Put("/workspace-ttl", api.putChatWorkspaceTTL)
|
||||
r.Get("/retention-days", api.getChatRetentionDays)
|
||||
r.Put("/retention-days", api.putChatRetentionDays)
|
||||
r.Get("/debug-retention-days", api.getChatDebugRetentionDays)
|
||||
r.Put("/debug-retention-days", api.putChatDebugRetentionDays)
|
||||
r.Get("/auto-archive-days", api.getChatAutoArchiveDays)
|
||||
r.Put("/auto-archive-days", api.putChatAutoArchiveDays)
|
||||
r.Get("/template-allowlist", api.getChatTemplateAllowlist)
|
||||
|
||||
@@ -2110,6 +2110,13 @@ func (q *querier) DeleteOldAuditLogs(ctx context.Context, arg database.DeleteOld
|
||||
return q.db.DeleteOldAuditLogs(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) DeleteOldChatDebugRuns(ctx context.Context, arg database.DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return q.db.DeleteOldChatDebugRuns(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) DeleteOldChatFiles(ctx context.Context, arg database.DeleteOldChatFilesParams) (int64, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil {
|
||||
return 0, err
|
||||
@@ -2682,6 +2689,17 @@ func (q *querier) GetChatDebugLoggingAllowUsers(ctx context.Context) (bool, erro
|
||||
return q.db.GetChatDebugLoggingAllowUsers(ctx)
|
||||
}
|
||||
|
||||
func (q *querier) GetChatDebugRetentionDays(ctx context.Context, defaultDebugRetentionDays int32) (int32, error) {
|
||||
// Chat debug retention is a deployment-wide config read by dbpurge.
|
||||
// Only requires a valid actor in context. The HTTP GET handler
|
||||
// allows any authenticated user; the PUT handler enforces admin
|
||||
// access (policy.ActionUpdate on ResourceDeploymentConfig).
|
||||
if _, ok := ActorFromContext(ctx); !ok {
|
||||
return 0, ErrNoActor
|
||||
}
|
||||
return q.db.GetChatDebugRetentionDays(ctx, defaultDebugRetentionDays)
|
||||
}
|
||||
|
||||
func (q *querier) GetChatDebugRunByID(ctx context.Context, id uuid.UUID) (database.ChatDebugRun, error) {
|
||||
run, err := q.db.GetChatDebugRunByID(ctx, id)
|
||||
if err != nil {
|
||||
@@ -7528,6 +7546,13 @@ func (q *querier) UpsertChatDebugLoggingAllowUsers(ctx context.Context, allowUse
|
||||
return q.db.UpsertChatDebugLoggingAllowUsers(ctx, allowUsers)
|
||||
}
|
||||
|
||||
func (q *querier) UpsertChatDebugRetentionDays(ctx context.Context, debugRetentionDays int32) error {
|
||||
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
return q.db.UpsertChatDebugRetentionDays(ctx, debugRetentionDays)
|
||||
}
|
||||
|
||||
func (q *querier) UpsertChatDesktopEnabled(ctx context.Context, enableDesktop bool) error {
|
||||
if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceDeploymentConfig); err != nil {
|
||||
return err
|
||||
|
||||
@@ -742,6 +742,10 @@ func (s *MethodTestSuite) TestChats() {
|
||||
dbm.EXPECT().GetChatFileMetadataByChatID(gomock.Any(), file.ID).Return(rows, nil).AnyTimes()
|
||||
check.Args(file.ID).Asserts(rbac.ResourceChat.WithOwner(file.OwnerID.String()).InOrg(file.OrganizationID).WithID(file.ID), policy.ActionRead).Returns(rows)
|
||||
}))
|
||||
s.Run("DeleteOldChatDebugRuns", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), database.DeleteOldChatDebugRunsParams{}).Return(int64(0), nil).AnyTimes()
|
||||
check.Args(database.DeleteOldChatDebugRunsParams{}).Asserts(rbac.ResourceSystem, policy.ActionDelete)
|
||||
}))
|
||||
s.Run("DeleteOldChatFiles", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().DeleteOldChatFiles(gomock.Any(), database.DeleteOldChatFilesParams{}).Return(int64(0), nil).AnyTimes()
|
||||
check.Args(database.DeleteOldChatFilesParams{}).Asserts(rbac.ResourceSystem, policy.ActionDelete)
|
||||
@@ -762,6 +766,14 @@ func (s *MethodTestSuite) TestChats() {
|
||||
dbm.EXPECT().GetChatAutoArchiveDays(gomock.Any(), gomock.Any()).Return(int32(90), nil).AnyTimes()
|
||||
check.Args(int32(90)).Asserts()
|
||||
}))
|
||||
s.Run("GetChatDebugRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().GetChatDebugRetentionDays(gomock.Any(), int32(7)).Return(int32(7), nil).AnyTimes()
|
||||
check.Args(int32(7)).Asserts().Returns(int32(7))
|
||||
}))
|
||||
s.Run("UpsertChatDebugRetentionDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().UpsertChatDebugRetentionDays(gomock.Any(), int32(7)).Return(nil).AnyTimes()
|
||||
check.Args(int32(7)).Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate)
|
||||
}))
|
||||
s.Run("UpsertChatAutoArchiveDays", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().UpsertChatAutoArchiveDays(gomock.Any(), int32(90)).Return(nil).AnyTimes()
|
||||
check.Args(int32(90)).Asserts(rbac.ResourceDeploymentConfig, policy.ActionUpdate)
|
||||
|
||||
@@ -632,6 +632,14 @@ func (m queryMetricsStore) DeleteOldAuditLogs(ctx context.Context, arg database.
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) DeleteOldChatDebugRuns(ctx context.Context, arg database.DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.DeleteOldChatDebugRuns(ctx, arg)
|
||||
m.queryLatencies.WithLabelValues("DeleteOldChatDebugRuns").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldChatDebugRuns").Inc()
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) DeleteOldChatFiles(ctx context.Context, arg database.DeleteOldChatFilesParams) (int64, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.DeleteOldChatFiles(ctx, arg)
|
||||
@@ -1200,6 +1208,14 @@ func (m queryMetricsStore) GetChatDebugLoggingAllowUsers(ctx context.Context) (b
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) GetChatDebugRetentionDays(ctx context.Context, defaultDebugRetentionDays int32) (int32, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.GetChatDebugRetentionDays(ctx, defaultDebugRetentionDays)
|
||||
m.queryLatencies.WithLabelValues("GetChatDebugRetentionDays").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetChatDebugRetentionDays").Inc()
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) GetChatDebugRunByID(ctx context.Context, id uuid.UUID) (database.ChatDebugRun, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.GetChatDebugRunByID(ctx, id)
|
||||
@@ -5384,6 +5400,14 @@ func (m queryMetricsStore) UpsertChatDebugLoggingAllowUsers(ctx context.Context,
|
||||
return r0
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) UpsertChatDebugRetentionDays(ctx context.Context, debugRetentionDays int32) error {
|
||||
start := time.Now()
|
||||
r0 := m.s.UpsertChatDebugRetentionDays(ctx, debugRetentionDays)
|
||||
m.queryLatencies.WithLabelValues("UpsertChatDebugRetentionDays").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpsertChatDebugRetentionDays").Inc()
|
||||
return r0
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) UpsertChatDesktopEnabled(ctx context.Context, enableDesktop bool) error {
|
||||
start := time.Now()
|
||||
r0 := m.s.UpsertChatDesktopEnabled(ctx, enableDesktop)
|
||||
|
||||
@@ -1058,6 +1058,21 @@ func (mr *MockStoreMockRecorder) DeleteOldAuditLogs(ctx, arg any) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldAuditLogs", reflect.TypeOf((*MockStore)(nil).DeleteOldAuditLogs), ctx, arg)
|
||||
}
|
||||
|
||||
// DeleteOldChatDebugRuns mocks base method.
|
||||
func (m *MockStore) DeleteOldChatDebugRuns(ctx context.Context, arg database.DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "DeleteOldChatDebugRuns", ctx, arg)
|
||||
ret0, _ := ret[0].(int64)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// DeleteOldChatDebugRuns indicates an expected call of DeleteOldChatDebugRuns.
|
||||
func (mr *MockStoreMockRecorder) DeleteOldChatDebugRuns(ctx, arg any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldChatDebugRuns", reflect.TypeOf((*MockStore)(nil).DeleteOldChatDebugRuns), ctx, arg)
|
||||
}
|
||||
|
||||
// DeleteOldChatFiles mocks base method.
|
||||
func (m *MockStore) DeleteOldChatFiles(ctx context.Context, arg database.DeleteOldChatFilesParams) (int64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -2207,6 +2222,21 @@ func (mr *MockStoreMockRecorder) GetChatDebugLoggingAllowUsers(ctx any) *gomock.
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatDebugLoggingAllowUsers", reflect.TypeOf((*MockStore)(nil).GetChatDebugLoggingAllowUsers), ctx)
|
||||
}
|
||||
|
||||
// GetChatDebugRetentionDays mocks base method.
|
||||
func (m *MockStore) GetChatDebugRetentionDays(ctx context.Context, defaultDebugRetentionDays int32) (int32, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetChatDebugRetentionDays", ctx, defaultDebugRetentionDays)
|
||||
ret0, _ := ret[0].(int32)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetChatDebugRetentionDays indicates an expected call of GetChatDebugRetentionDays.
|
||||
func (mr *MockStoreMockRecorder) GetChatDebugRetentionDays(ctx, defaultDebugRetentionDays any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetChatDebugRetentionDays", reflect.TypeOf((*MockStore)(nil).GetChatDebugRetentionDays), ctx, defaultDebugRetentionDays)
|
||||
}
|
||||
|
||||
// GetChatDebugRunByID mocks base method.
|
||||
func (m *MockStore) GetChatDebugRunByID(ctx context.Context, id uuid.UUID) (database.ChatDebugRun, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -10114,6 +10144,20 @@ func (mr *MockStoreMockRecorder) UpsertChatDebugLoggingAllowUsers(ctx, allowUser
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChatDebugLoggingAllowUsers", reflect.TypeOf((*MockStore)(nil).UpsertChatDebugLoggingAllowUsers), ctx, allowUsers)
|
||||
}
|
||||
|
||||
// UpsertChatDebugRetentionDays mocks base method.
|
||||
func (m *MockStore) UpsertChatDebugRetentionDays(ctx context.Context, debugRetentionDays int32) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "UpsertChatDebugRetentionDays", ctx, debugRetentionDays)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// UpsertChatDebugRetentionDays indicates an expected call of UpsertChatDebugRetentionDays.
|
||||
func (mr *MockStoreMockRecorder) UpsertChatDebugRetentionDays(ctx, debugRetentionDays any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertChatDebugRetentionDays", reflect.TypeOf((*MockStore)(nil).UpsertChatDebugRetentionDays), ctx, debugRetentionDays)
|
||||
}
|
||||
|
||||
// UpsertChatDesktopEnabled mocks base method.
|
||||
func (m *MockStore) UpsertChatDesktopEnabled(ctx context.Context, enableDesktop bool) error {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
@@ -45,10 +45,13 @@ const (
|
||||
// long enough to cover the maximum interval of a heartbeat event (currently
|
||||
// 1 hour) plus some buffer.
|
||||
maxTelemetryHeartbeatAge = 24 * time.Hour
|
||||
// Chat batch sizes stay smaller than audit/connection log batches because
|
||||
// chat_files rows carry bytea blobs.
|
||||
// Chat and chat file batch sizes stay smaller than audit/connection
|
||||
// log batches because chat_files rows carry bytea blobs.
|
||||
chatsBatchSize = 1000
|
||||
chatFilesBatchSize = 1000
|
||||
// Chat debug run deletions can cascade into steps with large JSONB
|
||||
// payloads, so they use the same conservative batch size.
|
||||
chatDebugRunsBatchSize = 1000
|
||||
// chatAutoArchiveDigestMaxChats bounds how many chat titles a
|
||||
// single digest body lists. Past the cap, surplus titles are
|
||||
// summarized as "...and N more". 25 is a readable email-friendly
|
||||
@@ -181,9 +184,11 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder
|
||||
// purge fails.
|
||||
func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.Time) error {
|
||||
// Read chat configs outside the tx so a corrupt value can't
|
||||
// poison subsequent queries. On error we log and stash, then
|
||||
// run unrelated purges best-effort and skip only chat work;
|
||||
// purgeTick returns chatConfigErr after the tx so the failed
|
||||
// poison subsequent queries. On config read errors, log and stash
|
||||
// the error, then run unrelated purges best-effort. Retention and
|
||||
// auto-archive errors skip only the conversation purge and
|
||||
// auto-archive work. Debug retention errors skip only the debug
|
||||
// purge. purgeTick returns chatConfigErr after the tx so the failed
|
||||
// iteration is operator-visible via metric and logs.
|
||||
chatRetentionDays, chatRetentionErr := db.GetChatRetentionDays(ctx)
|
||||
if chatRetentionErr != nil {
|
||||
@@ -195,7 +200,13 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
i.logger.Error(ctx, "failed to read chat auto-archive config: skipping chat purge and auto-archive this tick", slog.Error(chatAutoArchiveErr))
|
||||
}
|
||||
|
||||
chatConfigErr := errors.Join(chatRetentionErr, chatAutoArchiveErr)
|
||||
chatDebugRetentionDays, chatDebugRetentionErr := db.GetChatDebugRetentionDays(ctx, codersdk.DefaultChatDebugRetentionDays)
|
||||
if chatDebugRetentionErr != nil {
|
||||
i.logger.Error(ctx, "failed to read chat debug retention config: skipping chat debug purge this tick", slog.Error(chatDebugRetentionErr))
|
||||
}
|
||||
|
||||
chatRetentionConfigErr := errors.Join(chatRetentionErr, chatAutoArchiveErr)
|
||||
chatConfigErr := errors.Join(chatRetentionConfigErr, chatDebugRetentionErr)
|
||||
|
||||
// Populated inside the tx; dispatched post-commit.
|
||||
var archivedChats []database.AutoArchiveInactiveChatsRow
|
||||
@@ -304,13 +315,26 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
}
|
||||
}
|
||||
|
||||
var purgedChats, purgedChatFiles int64
|
||||
if chatConfigErr == nil {
|
||||
var purgedChats, purgedChatFiles, purgedChatDebugRuns int64
|
||||
if chatRetentionConfigErr == nil {
|
||||
purgedChats, purgedChatFiles, archivedChats, err = i.purgeChatsInTx(ctx, tx, start, chatRetentionDays, chatAutoArchiveDays)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to purge chats: %w", err)
|
||||
}
|
||||
}
|
||||
if chatDebugRetentionErr == nil && chatDebugRetentionDays > 0 {
|
||||
deleteChatDebugRunsBefore := start.Add(-time.Duration(chatDebugRetentionDays) * 24 * time.Hour)
|
||||
// updated_at is the retention clock, so the window starts after
|
||||
// the run stops being written to. There is intentionally no
|
||||
// finished_at guard, so abandoned in-flight rows can be purged.
|
||||
purgedChatDebugRuns, err = tx.DeleteOldChatDebugRuns(ctx, database.DeleteOldChatDebugRunsParams{
|
||||
BeforeTime: deleteChatDebugRunsBefore,
|
||||
LimitCount: chatDebugRunsBatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to delete old chat debug runs: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
i.logger.Debug(ctx, "purged old database entries",
|
||||
slog.F("workspace_agent_logs", purgedWorkspaceAgentLogs),
|
||||
@@ -320,14 +344,11 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
slog.F("audit_logs", purgedAuditLogs),
|
||||
slog.F("chats", purgedChats),
|
||||
slog.F("chat_files", purgedChatFiles),
|
||||
slog.F("chat_debug_runs", purgedChatDebugRuns),
|
||||
slog.F("auto_archived_chats", len(archivedChats)),
|
||||
slog.F("duration", i.clk.Since(start)),
|
||||
)
|
||||
|
||||
if i.iterationDuration != nil {
|
||||
duration := i.clk.Since(start)
|
||||
i.iterationDuration.WithLabelValues("true").Observe(duration.Seconds())
|
||||
}
|
||||
if i.recordsPurged != nil {
|
||||
i.recordsPurged.WithLabelValues("workspace_agent_logs").Add(float64(purgedWorkspaceAgentLogs))
|
||||
i.recordsPurged.WithLabelValues("expired_api_keys").Add(float64(expiredAPIKeys))
|
||||
@@ -335,9 +356,17 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
i.recordsPurged.WithLabelValues("connection_logs").Add(float64(purgedConnectionLogs))
|
||||
i.recordsPurged.WithLabelValues("audit_logs").Add(float64(purgedAuditLogs))
|
||||
i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats))
|
||||
i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns))
|
||||
i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles))
|
||||
}
|
||||
|
||||
// chatConfigErr is returned after the tx, so do not record this
|
||||
// iteration as successful when only the deferred config read failed.
|
||||
if i.iterationDuration != nil && chatConfigErr == nil {
|
||||
duration := i.clk.Since(start)
|
||||
i.iterationDuration.WithLabelValues("true").Observe(duration.Seconds())
|
||||
}
|
||||
|
||||
return nil
|
||||
}, database.DefaultTXOptions().WithID("db_purge"))
|
||||
if err != nil {
|
||||
|
||||
@@ -61,6 +61,7 @@ func TestPurge(t *testing.T) {
|
||||
mDB := dbmock.NewMockStore(gomock.NewController(t))
|
||||
mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatAutoArchiveDays(gomock.Any(), codersdk.DefaultChatAutoArchiveDays).Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays).Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).Return(nil).Times(2)
|
||||
purger := dbpurge.New(context.Background(), testutil.Logger(t), mDB, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
<-done // wait for doTick() to run.
|
||||
@@ -139,12 +140,57 @@ func TestMetrics(t *testing.T) {
|
||||
})
|
||||
require.GreaterOrEqual(t, chats, 0)
|
||||
|
||||
chatDebugRuns := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{
|
||||
"record_type": "chat_debug_runs",
|
||||
})
|
||||
require.GreaterOrEqual(t, chatDebugRuns, 0)
|
||||
|
||||
chatFiles := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{
|
||||
"record_type": "chat_files",
|
||||
})
|
||||
require.GreaterOrEqual(t, chatFiles, 0)
|
||||
})
|
||||
|
||||
t.Run("LockNotAcquiredSkipsIterationMetric", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
|
||||
defer cancel()
|
||||
|
||||
reg := prometheus.NewRegistry()
|
||||
clk := quartz.NewMock(t)
|
||||
now := clk.Now()
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
mDB := dbmock.NewMockStore(ctrl)
|
||||
mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatAutoArchiveDays(gomock.Any(), codersdk.DefaultChatAutoArchiveDays).
|
||||
Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays).
|
||||
Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().TryAcquireLock(gomock.Any(), int64(database.LockIDDBPurge)).Return(false, nil).AnyTimes()
|
||||
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
|
||||
DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error {
|
||||
return f(mDB)
|
||||
}).MinTimes(1)
|
||||
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, mDB, &codersdk.DeploymentValues{}, reg, nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
successHist := promhelp.MetricValue(t, reg, "coderd_dbpurge_iteration_duration_seconds", prometheus.Labels{
|
||||
"success": "true",
|
||||
})
|
||||
require.Nil(t, successHist, "lock contention should not record a successful purge iteration")
|
||||
|
||||
failedHist := promhelp.MetricValue(t, reg, "coderd_dbpurge_iteration_duration_seconds", prometheus.Labels{
|
||||
"success": "false",
|
||||
})
|
||||
require.Nil(t, failedHist, "lock contention should not record a failed purge iteration")
|
||||
})
|
||||
|
||||
t.Run("FailedIteration", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
|
||||
defer cancel()
|
||||
@@ -158,6 +204,8 @@ func TestMetrics(t *testing.T) {
|
||||
mDB := dbmock.NewMockStore(ctrl)
|
||||
mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatAutoArchiveDays(gomock.Any(), codersdk.DefaultChatAutoArchiveDays).Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays).
|
||||
Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
|
||||
Return(xerrors.New("simulated database error")).
|
||||
MinTimes(1)
|
||||
@@ -181,9 +229,9 @@ func TestMetrics(t *testing.T) {
|
||||
require.Nil(t, successHist, "should not have success=true metric on failure")
|
||||
})
|
||||
|
||||
// A failed retention read must not block unrelated purges,
|
||||
// but must skip the chat passes and surface as a failed
|
||||
// iteration via the metric.
|
||||
// A failed retention read must not block unrelated or chat debug
|
||||
// purges, but must skip the conversation purge and auto-archive
|
||||
// passes and surface as a failed iteration via the metric.
|
||||
t.Run("FailedChatRetentionRead", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
|
||||
defer cancel()
|
||||
@@ -198,12 +246,24 @@ func TestMetrics(t *testing.T) {
|
||||
mDB.EXPECT().GetChatRetentionDays(gomock.Any()).
|
||||
Return(int32(0), xerrors.New("simulated retention read error")).
|
||||
MinTimes(1)
|
||||
// Both reads happen before the bail; InTx still runs
|
||||
// so unrelated purges commit best-effort.
|
||||
// All reads happen before the bail; InTx still runs so unrelated
|
||||
// purges and chat debug purge commit best-effort.
|
||||
mDB.EXPECT().GetChatAutoArchiveDays(gomock.Any(), codersdk.DefaultChatAutoArchiveDays).
|
||||
Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays).
|
||||
Return(int32(7), nil).AnyTimes()
|
||||
mDB.EXPECT().TryAcquireLock(gomock.Any(), int64(database.LockIDDBPurge)).Return(true, nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldWorkspaceAgentStats(gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1)
|
||||
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
|
||||
Return(nil).MinTimes(1)
|
||||
DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error {
|
||||
return f(mDB)
|
||||
}).MinTimes(1)
|
||||
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
|
||||
@@ -242,6 +302,8 @@ func TestMetrics(t *testing.T) {
|
||||
mDB.EXPECT().GetChatAutoArchiveDays(gomock.Any(), codersdk.DefaultChatAutoArchiveDays).
|
||||
Return(int32(0), xerrors.New("simulated auto-archive read error")).
|
||||
MinTimes(1)
|
||||
mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays).
|
||||
Return(int32(0), nil).AnyTimes()
|
||||
// InTx still runs so unrelated purges commit; chat
|
||||
// passes inside the tx are skipped.
|
||||
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
|
||||
@@ -266,6 +328,59 @@ func TestMetrics(t *testing.T) {
|
||||
})
|
||||
require.Nil(t, successHist, "should not have success=true metric on auto-archive read failure")
|
||||
})
|
||||
|
||||
// Same contract as the other chat config reads, but debug retention
|
||||
// read failures skip only debug purging.
|
||||
t.Run("FailedChatDebugRetentionRead", func(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
|
||||
defer cancel()
|
||||
|
||||
reg := prometheus.NewRegistry()
|
||||
clk := quartz.NewMock(t)
|
||||
now := clk.Now()
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
ctrl := gomock.NewController(t)
|
||||
mDB := dbmock.NewMockStore(ctrl)
|
||||
mDB.EXPECT().GetChatRetentionDays(gomock.Any()).Return(int32(30), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatAutoArchiveDays(gomock.Any(), codersdk.DefaultChatAutoArchiveDays).
|
||||
Return(int32(0), nil).AnyTimes()
|
||||
mDB.EXPECT().GetChatDebugRetentionDays(gomock.Any(), codersdk.DefaultChatDebugRetentionDays).
|
||||
Return(int32(0), xerrors.New("simulated chat debug retention read error")).
|
||||
MinTimes(1)
|
||||
mDB.EXPECT().TryAcquireLock(gomock.Any(), int64(database.LockIDDBPurge)).Return(true, nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldWorkspaceAgentStats(gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldProvisionerDaemons(gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
|
||||
mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1)
|
||||
mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1)
|
||||
mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
|
||||
DoAndReturn(func(f func(database.Store) error, _ *database.TxOptions) error {
|
||||
return f(mDB)
|
||||
}).MinTimes(1)
|
||||
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, mDB, &codersdk.DeploymentValues{}, reg, nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
hist := promhelp.HistogramValue(t, reg, "coderd_dbpurge_iteration_duration_seconds", prometheus.Labels{
|
||||
"success": "false",
|
||||
})
|
||||
require.NotNil(t, hist)
|
||||
require.Greater(t, hist.GetSampleCount(), uint64(0),
|
||||
"failed chat debug retention read must record a failed iteration")
|
||||
|
||||
successHist := promhelp.MetricValue(t, reg, "coderd_dbpurge_iteration_duration_seconds", prometheus.Labels{
|
||||
"success": "true",
|
||||
})
|
||||
require.Nil(t, successHist, "should not have success=true metric on chat debug retention read failure")
|
||||
})
|
||||
}
|
||||
|
||||
//nolint:paralleltest // It uses LockIDDBPurge.
|
||||
@@ -1815,6 +1930,201 @@ func mockAuditorPtr(m *audit.MockAuditor) *atomic.Pointer[audit.Auditor] {
|
||||
return &p
|
||||
}
|
||||
|
||||
//nolint:paralleltest // It uses LockIDDBPurge.
|
||||
func TestPurgeChatDebugRuns(t *testing.T) {
|
||||
now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
type chatDebugDeps struct {
|
||||
user database.User
|
||||
org database.Organization
|
||||
modelConfig database.ChatModelConfig
|
||||
}
|
||||
// setupChatDebugDeps creates the user, organization, and chat model config dependencies needed for the chat debug retention test.
|
||||
setupChatDebugDeps := func(t *testing.T, db database.Store) chatDebugDeps {
|
||||
t.Helper()
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
org := dbgen.Organization(t, db, database.Organization{})
|
||||
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{
|
||||
UserID: user.ID,
|
||||
OrganizationID: org.ID,
|
||||
})
|
||||
_ = dbgen.ChatProvider(t, db, database.ChatProvider{
|
||||
Provider: "openai",
|
||||
DisplayName: "OpenAI",
|
||||
})
|
||||
modelConfig := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
||||
Provider: "openai",
|
||||
Model: "test-model",
|
||||
ContextLimit: 8192,
|
||||
})
|
||||
return chatDebugDeps{user: user, org: org, modelConfig: modelConfig}
|
||||
}
|
||||
createChat := func(ctx context.Context, t *testing.T, db database.Store, rawDB *sql.DB, deps chatDebugDeps, archived bool, updatedAt time.Time) database.Chat {
|
||||
t.Helper()
|
||||
chat := dbgen.Chat(t, db, database.Chat{
|
||||
OrganizationID: deps.org.ID,
|
||||
OwnerID: deps.user.ID,
|
||||
LastModelConfigID: deps.modelConfig.ID,
|
||||
Title: "debug-retention-test-chat",
|
||||
})
|
||||
if archived {
|
||||
_, err := db.ArchiveChatByID(ctx, chat.ID)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
_, err := rawDB.ExecContext(ctx, "UPDATE chats SET updated_at = $1 WHERE id = $2", updatedAt, chat.ID)
|
||||
require.NoError(t, err)
|
||||
return chat
|
||||
}
|
||||
createDebugRunWithStep := func(ctx context.Context, t *testing.T, db database.Store, chatID uuid.UUID, updatedAt time.Time, finished bool) database.ChatDebugRun {
|
||||
t.Helper()
|
||||
run, err := db.InsertChatDebugRun(ctx, database.InsertChatDebugRunParams{
|
||||
ChatID: chatID,
|
||||
Kind: string(codersdk.ChatDebugRunKindChatTurn),
|
||||
Status: string(codersdk.ChatDebugStatusInProgress),
|
||||
Provider: sql.NullString{String: "openai", Valid: true},
|
||||
Model: sql.NullString{String: "gpt-4o-mini", Valid: true},
|
||||
StartedAt: sql.NullTime{Time: updatedAt.Add(-time.Minute), Valid: true},
|
||||
UpdatedAt: sql.NullTime{Time: updatedAt, Valid: true},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = db.InsertChatDebugStep(ctx, database.InsertChatDebugStepParams{
|
||||
RunID: run.ID,
|
||||
ChatID: run.ChatID,
|
||||
StepNumber: 1,
|
||||
Operation: string(codersdk.ChatDebugStepOperationStream),
|
||||
Status: string(codersdk.ChatDebugStatusCompleted),
|
||||
StartedAt: sql.NullTime{Time: updatedAt.Add(-time.Minute), Valid: true},
|
||||
UpdatedAt: sql.NullTime{Time: updatedAt, Valid: true},
|
||||
FinishedAt: sql.NullTime{Time: updatedAt, Valid: true},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
if finished {
|
||||
run, err = db.UpdateChatDebugRun(ctx, database.UpdateChatDebugRunParams{
|
||||
Status: sql.NullString{String: string(codersdk.ChatDebugStatusCompleted), Valid: true},
|
||||
FinishedAt: sql.NullTime{Time: updatedAt, Valid: true},
|
||||
Now: updatedAt,
|
||||
ID: run.ID,
|
||||
ChatID: run.ChatID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
return run
|
||||
}
|
||||
countDebugSteps := func(ctx context.Context, t *testing.T, rawDB *sql.DB, runID uuid.UUID) int {
|
||||
t.Helper()
|
||||
var count int
|
||||
err := rawDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM chat_debug_steps WHERE run_id = $1", runID).Scan(&count)
|
||||
require.NoError(t, err)
|
||||
return count
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(t *testing.T)
|
||||
}{
|
||||
{
|
||||
name: "DeletesOldRunsAndCascadedSteps",
|
||||
run: func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
clk := quartz.NewMock(t)
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure())
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
reg := prometheus.NewRegistry()
|
||||
deps := setupChatDebugDeps(t, db)
|
||||
require.NoError(t, db.UpsertChatDebugRetentionDays(ctx, int32(7)))
|
||||
|
||||
chat := createChat(ctx, t, db, rawDB, deps, false, now)
|
||||
oldRun := createDebugRunWithStep(ctx, t, db, chat.ID, now.Add(-8*24*time.Hour), true)
|
||||
recentRun := createDebugRunWithStep(ctx, t, db, chat.ID, now.Add(-6*24*time.Hour), true)
|
||||
unfinishedOldRun := createDebugRunWithStep(ctx, t, db, chat.ID, now.Add(-9*24*time.Hour), false)
|
||||
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, reg, nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
chatDebugRuns := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{
|
||||
"record_type": "chat_debug_runs",
|
||||
})
|
||||
require.Greater(t, chatDebugRuns, 0, "chat debug purge counter should record deleted runs")
|
||||
|
||||
_, err := db.GetChatDebugRunByID(ctx, oldRun.ID)
|
||||
require.ErrorIs(t, err, sql.ErrNoRows, "old finished run should be deleted")
|
||||
require.Zero(t, countDebugSteps(ctx, t, rawDB, oldRun.ID), "old run steps should cascade")
|
||||
|
||||
_, err = db.GetChatDebugRunByID(ctx, unfinishedOldRun.ID)
|
||||
require.ErrorIs(t, err, sql.ErrNoRows, "old unfinished run should be deleted")
|
||||
require.Zero(t, countDebugSteps(ctx, t, rawDB, unfinishedOldRun.ID), "old unfinished run steps should cascade")
|
||||
|
||||
_, err = db.GetChatDebugRunByID(ctx, recentRun.ID)
|
||||
require.NoError(t, err, "recent run should remain")
|
||||
require.Equal(t, 1, countDebugSteps(ctx, t, rawDB, recentRun.ID), "recent run step should remain")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "RetentionDisabledKeepsOldRuns",
|
||||
run: func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
clk := quartz.NewMock(t)
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure())
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
deps := setupChatDebugDeps(t, db)
|
||||
require.NoError(t, db.UpsertChatDebugRetentionDays(ctx, int32(0)))
|
||||
|
||||
chat := createChat(ctx, t, db, rawDB, deps, false, now)
|
||||
oldRun := createDebugRunWithStep(ctx, t, db, chat.ID, now.Add(-90*24*time.Hour), true)
|
||||
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
_, err := db.GetChatDebugRunByID(ctx, oldRun.ID)
|
||||
require.NoError(t, err, "old run should remain when retention is disabled")
|
||||
require.Equal(t, 1, countDebugSteps(ctx, t, rawDB, oldRun.ID), "old run step should remain")
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ChatCascadeDeletesDebugRows",
|
||||
run: func(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
clk := quartz.NewMock(t)
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
db, _, rawDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure())
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
deps := setupChatDebugDeps(t, db)
|
||||
require.NoError(t, db.UpsertChatRetentionDays(ctx, int32(30)))
|
||||
require.NoError(t, db.UpsertChatDebugRetentionDays(ctx, int32(0)))
|
||||
|
||||
oldArchivedChat := createChat(ctx, t, db, rawDB, deps, true, now.Add(-31*24*time.Hour))
|
||||
run := createDebugRunWithStep(ctx, t, db, oldArchivedChat.ID, now, true)
|
||||
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, prometheus.NewRegistry(), nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
_, err := db.GetChatByID(ctx, oldArchivedChat.ID)
|
||||
require.ErrorIs(t, err, sql.ErrNoRows, "old archived chat should be deleted")
|
||||
_, err = db.GetChatDebugRunByID(ctx, run.ID)
|
||||
require.ErrorIs(t, err, sql.ErrNoRows, "chat deletion should cascade to debug runs")
|
||||
require.Zero(t, countDebugSteps(ctx, t, rawDB, run.ID), "chat deletion should cascade to debug steps")
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) { //nolint:paralleltest // subtests use LockIDDBPurge.
|
||||
tt.run(t)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
//nolint:paralleltest // It uses LockIDDBPurge.
|
||||
func TestDeleteOldChatFiles(t *testing.T) {
|
||||
now := time.Date(2025, 6, 15, 12, 0, 0, 0, time.UTC)
|
||||
@@ -1949,7 +2259,7 @@ func TestDeleteOldChatFiles(t *testing.T) {
|
||||
|
||||
// Old archived chat should be gone.
|
||||
_, err = db.GetChatByID(ctx, oldChat.ID)
|
||||
require.Error(t, err, "old archived chat should be deleted")
|
||||
require.ErrorIs(t, err, sql.ErrNoRows, "old archived chat should be deleted")
|
||||
|
||||
// Its messages should be gone too (CASCADE).
|
||||
msgs, err := db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{
|
||||
|
||||
Generated
+2
@@ -3787,6 +3787,8 @@ CREATE UNIQUE INDEX idx_chat_debug_runs_id_chat ON chat_debug_runs USING btree (
|
||||
|
||||
CREATE INDEX idx_chat_debug_runs_stale ON chat_debug_runs USING btree (updated_at) WHERE (finished_at IS NULL);
|
||||
|
||||
CREATE INDEX idx_chat_debug_runs_updated_at ON chat_debug_runs USING btree (updated_at);
|
||||
|
||||
CREATE INDEX idx_chat_debug_steps_chat_assistant_msg ON chat_debug_steps USING btree (chat_id, assistant_message_id) WHERE (assistant_message_id IS NOT NULL);
|
||||
|
||||
CREATE INDEX idx_chat_debug_steps_chat_tip ON chat_debug_steps USING btree (chat_id, history_tip_message_id);
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DROP INDEX IF EXISTS idx_chat_debug_runs_updated_at;
|
||||
@@ -0,0 +1 @@
|
||||
CREATE INDEX idx_chat_debug_runs_updated_at ON chat_debug_runs (updated_at);
|
||||
@@ -146,6 +146,11 @@ type sqlcQuerier interface {
|
||||
// connection events (connect, disconnect, open, close) which are handled
|
||||
// separately by DeleteOldAuditLogConnectionEvents.
|
||||
DeleteOldAuditLogs(ctx context.Context, arg DeleteOldAuditLogsParams) (int64, error)
|
||||
// updated_at is the retention clock, so the window starts after the run
|
||||
// stops being written to.
|
||||
// Intentionally no finished_at IS NOT NULL guard: abandoned in-flight rows
|
||||
// older than the cutoff are also purged.
|
||||
DeleteOldChatDebugRuns(ctx context.Context, arg DeleteOldChatDebugRunsParams) (int64, error)
|
||||
// TODO(cian): Add indexes on chats(archived, updated_at) and
|
||||
// chat_files(created_at) for purge query performance.
|
||||
// See: https://github.com/coder/internal/issues/1438
|
||||
@@ -300,6 +305,8 @@ type sqlcQuerier interface {
|
||||
// allows users to opt into chat debug logging when the deployment does
|
||||
// not already force debug logging on globally.
|
||||
GetChatDebugLoggingAllowUsers(ctx context.Context) (bool, error)
|
||||
// Chat debug run retention window in days. 0 disables.
|
||||
GetChatDebugRetentionDays(ctx context.Context, defaultDebugRetentionDays int32) (int32, error)
|
||||
GetChatDebugRunByID(ctx context.Context, id uuid.UUID) (ChatDebugRun, error)
|
||||
// Returns the most recent debug runs for a chat, ordered newest-first.
|
||||
// Callers must supply an explicit limit to avoid unbounded result sets.
|
||||
@@ -846,6 +853,8 @@ type sqlcQuerier interface {
|
||||
InsertAllUsersGroup(ctx context.Context, organizationID uuid.UUID) (Group, error)
|
||||
InsertAuditLog(ctx context.Context, arg InsertAuditLogParams) (AuditLog, error)
|
||||
InsertChat(ctx context.Context, arg InsertChatParams) (Chat, error)
|
||||
// updated_at is the retention clock used by DeleteOldChatDebugRuns.
|
||||
// Set it on every write to keep retention semantics correct.
|
||||
InsertChatDebugRun(ctx context.Context, arg InsertChatDebugRunParams) (ChatDebugRun, error)
|
||||
// The CTE atomically locks the parent run via UPDATE, bumps its
|
||||
// updated_at (eliminating a separate TouchChatDebugRunUpdatedAt
|
||||
@@ -1073,6 +1082,7 @@ type sqlcQuerier interface {
|
||||
// write-once-finalize pattern where fields are set at creation
|
||||
// or finalization and never cleared back to NULL. The @now
|
||||
// parameter keeps updated_at under the caller's clock.
|
||||
// updated_at is also the retention clock used by DeleteOldChatDebugRuns.
|
||||
//
|
||||
// finished_at is enforced as write-once at the SQL level: once
|
||||
// populated it cannot be overwritten by a later call. Callers
|
||||
@@ -1229,6 +1239,7 @@ type sqlcQuerier interface {
|
||||
// UpsertChatDebugLoggingAllowUsers updates the runtime admin setting that
|
||||
// allows users to opt into chat debug logging.
|
||||
UpsertChatDebugLoggingAllowUsers(ctx context.Context, allowUsers bool) error
|
||||
UpsertChatDebugRetentionDays(ctx context.Context, debugRetentionDays int32) error
|
||||
UpsertChatDesktopEnabled(ctx context.Context, enableDesktop bool) error
|
||||
UpsertChatDiffStatus(ctx context.Context, arg UpsertChatDiffStatusParams) (ChatDiffStatus, error)
|
||||
UpsertChatDiffStatusReference(ctx context.Context, arg UpsertChatDiffStatusReferenceParams) (ChatDiffStatus, error)
|
||||
|
||||
@@ -2993,6 +2993,37 @@ func (q *sqlQuerier) DeleteChatDebugDataByChatID(ctx context.Context, arg Delete
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
const deleteOldChatDebugRuns = `-- name: DeleteOldChatDebugRuns :execrows
|
||||
WITH deletable AS (
|
||||
SELECT id, chat_id
|
||||
FROM chat_debug_runs
|
||||
WHERE updated_at < $1::timestamptz
|
||||
ORDER BY updated_at ASC
|
||||
LIMIT $2::int
|
||||
)
|
||||
DELETE FROM chat_debug_runs
|
||||
USING deletable
|
||||
WHERE chat_debug_runs.id = deletable.id
|
||||
AND chat_debug_runs.chat_id = deletable.chat_id
|
||||
`
|
||||
|
||||
type DeleteOldChatDebugRunsParams struct {
|
||||
BeforeTime time.Time `db:"before_time" json:"before_time"`
|
||||
LimitCount int32 `db:"limit_count" json:"limit_count"`
|
||||
}
|
||||
|
||||
// updated_at is the retention clock, so the window starts after the run
|
||||
// stops being written to.
|
||||
// Intentionally no finished_at IS NOT NULL guard: abandoned in-flight rows
|
||||
// older than the cutoff are also purged.
|
||||
func (q *sqlQuerier) DeleteOldChatDebugRuns(ctx context.Context, arg DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
result, err := q.db.ExecContext(ctx, deleteOldChatDebugRuns, arg.BeforeTime, arg.LimitCount)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
const finalizeStaleChatDebugRows = `-- name: FinalizeStaleChatDebugRows :one
|
||||
WITH finalized_runs AS (
|
||||
UPDATE chat_debug_runs
|
||||
@@ -3236,6 +3267,8 @@ type InsertChatDebugRunParams struct {
|
||||
FinishedAt sql.NullTime `db:"finished_at" json:"finished_at"`
|
||||
}
|
||||
|
||||
// updated_at is the retention clock used by DeleteOldChatDebugRuns.
|
||||
// Set it on every write to keep retention semantics correct.
|
||||
func (q *sqlQuerier) InsertChatDebugRun(ctx context.Context, arg InsertChatDebugRunParams) (ChatDebugRun, error) {
|
||||
row := q.db.QueryRowContext(ctx, insertChatDebugRun,
|
||||
arg.ChatID,
|
||||
@@ -3503,6 +3536,7 @@ type UpdateChatDebugRunParams struct {
|
||||
// write-once-finalize pattern where fields are set at creation
|
||||
// or finalization and never cleared back to NULL. The @now
|
||||
// parameter keeps updated_at under the caller's clock.
|
||||
// updated_at is also the retention clock used by DeleteOldChatDebugRuns.
|
||||
//
|
||||
// finished_at is enforced as write-once at the SQL level: once
|
||||
// populated it cannot be overwritten by a later call. Callers
|
||||
@@ -20595,6 +20629,22 @@ func (q *sqlQuerier) GetChatDebugLoggingAllowUsers(ctx context.Context) (bool, e
|
||||
return allow_users, err
|
||||
}
|
||||
|
||||
const getChatDebugRetentionDays = `-- name: GetChatDebugRetentionDays :one
|
||||
SELECT COALESCE(
|
||||
(SELECT value::integer FROM site_configs
|
||||
WHERE key = 'agents_chat_debug_retention_days'),
|
||||
$1::integer
|
||||
) :: integer AS debug_retention_days
|
||||
`
|
||||
|
||||
// Chat debug run retention window in days. 0 disables.
|
||||
func (q *sqlQuerier) GetChatDebugRetentionDays(ctx context.Context, defaultDebugRetentionDays int32) (int32, error) {
|
||||
row := q.db.QueryRowContext(ctx, getChatDebugRetentionDays, defaultDebugRetentionDays)
|
||||
var debug_retention_days int32
|
||||
err := row.Scan(&debug_retention_days)
|
||||
return debug_retention_days, err
|
||||
}
|
||||
|
||||
const getChatDesktopEnabled = `-- name: GetChatDesktopEnabled :one
|
||||
SELECT
|
||||
COALESCE((SELECT value = 'true' FROM site_configs WHERE key = 'agents_desktop_enabled'), false) :: boolean AS enable_desktop
|
||||
@@ -21027,6 +21077,18 @@ func (q *sqlQuerier) UpsertChatDebugLoggingAllowUsers(ctx context.Context, allow
|
||||
return err
|
||||
}
|
||||
|
||||
const upsertChatDebugRetentionDays = `-- name: UpsertChatDebugRetentionDays :exec
|
||||
INSERT INTO site_configs (key, value)
|
||||
VALUES ('agents_chat_debug_retention_days', CAST($1 AS integer)::text)
|
||||
ON CONFLICT (key) DO UPDATE SET value = CAST($1 AS integer)::text
|
||||
WHERE site_configs.key = 'agents_chat_debug_retention_days'
|
||||
`
|
||||
|
||||
func (q *sqlQuerier) UpsertChatDebugRetentionDays(ctx context.Context, debugRetentionDays int32) error {
|
||||
_, err := q.db.ExecContext(ctx, upsertChatDebugRetentionDays, debugRetentionDays)
|
||||
return err
|
||||
}
|
||||
|
||||
const upsertChatDesktopEnabled = `-- name: UpsertChatDesktopEnabled :exec
|
||||
INSERT INTO site_configs (key, value)
|
||||
VALUES (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
-- updated_at is the retention clock used by DeleteOldChatDebugRuns.
|
||||
-- Set it on every write to keep retention semantics correct.
|
||||
-- name: InsertChatDebugRun :one
|
||||
INSERT INTO chat_debug_runs (
|
||||
chat_id,
|
||||
@@ -39,6 +41,7 @@ RETURNING *;
|
||||
-- write-once-finalize pattern where fields are set at creation
|
||||
-- or finalization and never cleared back to NULL. The @now
|
||||
-- parameter keeps updated_at under the caller's clock.
|
||||
-- updated_at is also the retention clock used by DeleteOldChatDebugRuns.
|
||||
--
|
||||
-- finished_at is enforced as write-once at the SQL level: once
|
||||
-- populated it cannot be overwritten by a later call. Callers
|
||||
@@ -246,6 +249,23 @@ DELETE FROM chat_debug_runs
|
||||
WHERE chat_id = @chat_id::uuid
|
||||
AND id IN (SELECT id FROM affected_runs);
|
||||
|
||||
-- updated_at is the retention clock, so the window starts after the run
|
||||
-- stops being written to.
|
||||
-- Intentionally no finished_at IS NOT NULL guard: abandoned in-flight rows
|
||||
-- older than the cutoff are also purged.
|
||||
-- name: DeleteOldChatDebugRuns :execrows
|
||||
WITH deletable AS (
|
||||
SELECT id, chat_id
|
||||
FROM chat_debug_runs
|
||||
WHERE updated_at < @before_time::timestamptz
|
||||
ORDER BY updated_at ASC
|
||||
LIMIT @limit_count::int
|
||||
)
|
||||
DELETE FROM chat_debug_runs
|
||||
USING deletable
|
||||
WHERE chat_debug_runs.id = deletable.id
|
||||
AND chat_debug_runs.chat_id = deletable.chat_id;
|
||||
|
||||
-- name: FinalizeStaleChatDebugRows :one
|
||||
-- Marks orphaned in-progress rows as interrupted so they do not stay
|
||||
-- in a non-terminal state forever. The NOT IN list must match the
|
||||
|
||||
@@ -358,6 +358,20 @@ VALUES ('agents_chat_retention_days', CAST(@retention_days AS integer)::text)
|
||||
ON CONFLICT (key) DO UPDATE SET value = CAST(@retention_days AS integer)::text
|
||||
WHERE site_configs.key = 'agents_chat_retention_days';
|
||||
|
||||
-- name: GetChatDebugRetentionDays :one
|
||||
-- Chat debug run retention window in days. 0 disables.
|
||||
SELECT COALESCE(
|
||||
(SELECT value::integer FROM site_configs
|
||||
WHERE key = 'agents_chat_debug_retention_days'),
|
||||
@default_debug_retention_days::integer
|
||||
) :: integer AS debug_retention_days;
|
||||
|
||||
-- name: UpsertChatDebugRetentionDays :exec
|
||||
INSERT INTO site_configs (key, value)
|
||||
VALUES ('agents_chat_debug_retention_days', CAST(@debug_retention_days AS integer)::text)
|
||||
ON CONFLICT (key) DO UPDATE SET value = CAST(@debug_retention_days AS integer)::text
|
||||
WHERE site_configs.key = 'agents_chat_debug_retention_days';
|
||||
|
||||
-- name: GetChatAutoArchiveDays :one
|
||||
-- Auto-archive window in days. 0 disables.
|
||||
SELECT COALESCE(
|
||||
|
||||
@@ -5248,6 +5248,57 @@ func (api *API) putChatRetentionDays(rw http.ResponseWriter, r *http.Request) {
|
||||
rw.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// getChatDebugRetentionDays returns the deployment-wide chat debug run
|
||||
// retention window. Any authenticated user can read it; writes require admin.
|
||||
//
|
||||
//nolint:revive // get-return: revive assumes get* must be a getter, but this is an HTTP handler.
|
||||
func (api *API) getChatDebugRetentionDays(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
retentionDays, err := api.Database.GetChatDebugRetentionDays(ctx, codersdk.DefaultChatDebugRetentionDays)
|
||||
if err != nil {
|
||||
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
|
||||
Message: "Failed to get chat debug retention days.",
|
||||
Detail: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
httpapi.Write(ctx, rw, http.StatusOK, codersdk.ChatDebugRetentionDaysResponse{
|
||||
DebugRetentionDays: retentionDays,
|
||||
})
|
||||
}
|
||||
|
||||
// Keep in sync with the validation schema in
|
||||
// site/src/pages/AgentsPage/components/DebugRetentionSettings.tsx.
|
||||
const chatDebugRetentionDaysMaximum = 3650 // ~10 years
|
||||
|
||||
// putChatDebugRetentionDays updates the deployment-wide chat debug run
|
||||
// retention window. Admin-only.
|
||||
func (api *API) putChatDebugRetentionDays(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
if !api.Authorize(r, policy.ActionUpdate, rbac.ResourceDeploymentConfig) {
|
||||
httpapi.Forbidden(rw)
|
||||
return
|
||||
}
|
||||
var req codersdk.UpdateChatDebugRetentionDaysRequest
|
||||
if !httpapi.Read(ctx, rw, r, &req) {
|
||||
return
|
||||
}
|
||||
if req.DebugRetentionDays < 0 || req.DebugRetentionDays > chatDebugRetentionDaysMaximum {
|
||||
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
|
||||
Message: fmt.Sprintf("Chat debug retention days must be between 0 and %d.", chatDebugRetentionDaysMaximum),
|
||||
})
|
||||
return
|
||||
}
|
||||
if err := api.Database.UpsertChatDebugRetentionDays(ctx, req.DebugRetentionDays); err != nil {
|
||||
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
|
||||
Message: "Failed to update chat debug retention days.",
|
||||
Detail: err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
rw.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// getChatAutoArchiveDays returns the deployment-wide auto-archive
|
||||
// window. Any authenticated user can read it (same as retention
|
||||
// days); writes require admin.
|
||||
|
||||
@@ -12103,6 +12103,63 @@ func TestChatRetentionDays(t *testing.T) {
|
||||
requireSDKError(t, err, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
func TestChatDebugRetentionDays(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
adminClient := newChatClient(t)
|
||||
firstUser := coderdtest.CreateFirstUser(t, adminClient.Client)
|
||||
memberClientRaw, _ := coderdtest.CreateAnotherUser(t, adminClient.Client, firstUser.OrganizationID)
|
||||
memberClient := codersdk.NewExperimentalClient(memberClientRaw)
|
||||
|
||||
// Default value is DefaultChatDebugRetentionDays when nothing has
|
||||
// been configured.
|
||||
resp, err := adminClient.GetChatDebugRetentionDays(ctx)
|
||||
require.NoError(t, err, "get default")
|
||||
require.Equal(t, codersdk.DefaultChatDebugRetentionDays, resp.DebugRetentionDays, "default should match DefaultChatDebugRetentionDays")
|
||||
|
||||
// Admin can set debug retention days to 14.
|
||||
err = adminClient.UpdateChatDebugRetentionDays(ctx, codersdk.UpdateChatDebugRetentionDaysRequest{
|
||||
DebugRetentionDays: 14,
|
||||
})
|
||||
require.NoError(t, err, "admin set 14")
|
||||
|
||||
resp, err = adminClient.GetChatDebugRetentionDays(ctx)
|
||||
require.NoError(t, err, "get after set")
|
||||
require.Equal(t, int32(14), resp.DebugRetentionDays, "should return 14")
|
||||
|
||||
// Non-admin member can read the value.
|
||||
memberResp, err := memberClient.GetChatDebugRetentionDays(ctx)
|
||||
require.NoError(t, err, "member read")
|
||||
require.Equal(t, int32(14), memberResp.DebugRetentionDays, "member sees same value")
|
||||
|
||||
// Non-admin member cannot write.
|
||||
err = memberClient.UpdateChatDebugRetentionDays(ctx, codersdk.UpdateChatDebugRetentionDaysRequest{DebugRetentionDays: 7})
|
||||
requireSDKError(t, err, http.StatusForbidden)
|
||||
|
||||
// Admin can disable chat debug retention purge by setting 0.
|
||||
err = adminClient.UpdateChatDebugRetentionDays(ctx, codersdk.UpdateChatDebugRetentionDaysRequest{
|
||||
DebugRetentionDays: 0,
|
||||
})
|
||||
require.NoError(t, err, "admin set 0")
|
||||
|
||||
resp, err = adminClient.GetChatDebugRetentionDays(ctx)
|
||||
require.NoError(t, err, "get after zero")
|
||||
require.Equal(t, int32(0), resp.DebugRetentionDays, "should be 0 after disable")
|
||||
|
||||
// Validation: negative value is rejected.
|
||||
err = adminClient.UpdateChatDebugRetentionDays(ctx, codersdk.UpdateChatDebugRetentionDaysRequest{
|
||||
DebugRetentionDays: -1,
|
||||
})
|
||||
requireSDKError(t, err, http.StatusBadRequest)
|
||||
|
||||
// Validation: exceeding the 3650-day maximum is rejected.
|
||||
err = adminClient.UpdateChatDebugRetentionDays(ctx, codersdk.UpdateChatDebugRetentionDaysRequest{
|
||||
DebugRetentionDays: 3651, // chatDebugRetentionDaysMaximum + 1; keep in sync with coderd/exp_chats.go.
|
||||
})
|
||||
requireSDKError(t, err, http.StatusBadRequest)
|
||||
}
|
||||
|
||||
func TestChatAutoArchiveDays(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
|
||||
Reference in New Issue
Block a user