mirror of
https://github.com/coder/coder.git
synced 2026-09-21 12:44:32 +08:00
feat: purge boundary logs past retention (#24815)
Add a periodic purge job for `boundary_logs` rows past their retention threshold, following the same pattern as the existing audit log and connection log purge jobs in `dbpurge`. Expose a `--boundary-log-retention` deployment flag (env `CODER_BOUNDARY_LOG_RETENTION`, YAML `retention.boundary_logs`). Default is `0` (keep indefinitely). When set to a positive duration, `purgeTick` deletes rows where `captured_at` is older than the threshold in batches of 10,000, matching other log purge operations. The `boundary_logs` label is added to the `records_purged_total` Prometheus counter. Also removes the random-UUID fallback for `OwnerID` in `dbgen.BoundarySession`. The previous fallback generated a UUID that could never satisfy the `boundary_sessions_owner_id_fkey` FK constraint, masking test setup bugs. Callers must now provide a valid user ID or accept NULL (the legitimate "user deleted" state).
This commit is contained in:
+6
@@ -878,6 +878,12 @@ that data type.
|
||||
indefinitely). We advise keeping audit logs for at least a year, and
|
||||
in accordance with your compliance requirements.
|
||||
|
||||
--boundary-log-retention duration, $CODER_BOUNDARY_LOG_RETENTION (default: 0)
|
||||
How long boundary audit log entries are retained. Boundary logs record
|
||||
HTTP requests processed by a Boundary confinement proxy. Set to 0 to
|
||||
disable automatic deletion (keep indefinitely). Adjust to match your
|
||||
organization's regulatory requirements.
|
||||
|
||||
--connection-logs-retention duration, $CODER_CONNECTION_LOGS_RETENTION (default: 0)
|
||||
How long connection log entries are retained. Set to 0 to disable
|
||||
(keep indefinitely).
|
||||
|
||||
+6
@@ -1122,6 +1122,12 @@ retention:
|
||||
# build are always retained. Set to 0 to disable automatic deletion.
|
||||
# (default: 7d, type: duration)
|
||||
workspace_agent_logs: 168h0m0s
|
||||
# How long boundary audit log entries are retained. Boundary logs record HTTP
|
||||
# requests processed by a Boundary confinement proxy. Set to 0 to disable
|
||||
# automatic deletion (keep indefinitely). Adjust to match your organization's
|
||||
# regulatory requirements.
|
||||
# (default: 0, type: duration)
|
||||
boundary_logs: 0s
|
||||
templateBuilder:
|
||||
# Disable the template builder feature for guided template creation. When
|
||||
# disabled, all /api/v2/templatebuilder/* endpoints return 404.
|
||||
|
||||
Generated
+4
@@ -22930,6 +22930,10 @@ const docTemplate = `{
|
||||
"description": "AuditLogs controls how long audit log entries are retained.\nSet to 0 to disable (keep indefinitely).",
|
||||
"type": "integer"
|
||||
},
|
||||
"boundary_logs": {
|
||||
"description": "BoundaryLogs controls how long boundary audit log entries are\nretained. Boundary logs record every HTTP request processed by\na Boundary confinement proxy. Set to 0 to disable automatic\ndeletion (keep indefinitely). Adjust to match your\norganization's regulatory requirements.",
|
||||
"type": "integer"
|
||||
},
|
||||
"connection_logs": {
|
||||
"description": "ConnectionLogs controls how long connection log entries are retained.\nSet to 0 to disable (keep indefinitely).",
|
||||
"type": "integer"
|
||||
|
||||
Generated
+4
@@ -21013,6 +21013,10 @@
|
||||
"description": "AuditLogs controls how long audit log entries are retained.\nSet to 0 to disable (keep indefinitely).",
|
||||
"type": "integer"
|
||||
},
|
||||
"boundary_logs": {
|
||||
"description": "BoundaryLogs controls how long boundary audit log entries are\nretained. Boundary logs record every HTTP request processed by\na Boundary confinement proxy. Set to 0 to disable automatic\ndeletion (keep indefinitely). Adjust to match your\norganization's regulatory requirements.",
|
||||
"type": "integer"
|
||||
},
|
||||
"connection_logs": {
|
||||
"description": "ConnectionLogs controls how long connection log entries are retained.\nSet to 0 to disable (keep indefinitely).",
|
||||
"type": "integer"
|
||||
|
||||
@@ -2299,6 +2299,13 @@ func (q *querier) DeleteOldBoundaryLogs(ctx context.Context, arg database.Delete
|
||||
return q.db.DeleteOldBoundaryLogs(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) DeleteOldBoundarySessions(ctx context.Context, arg database.DeleteOldBoundarySessionsParams) (int64, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceBoundaryLog); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return q.db.DeleteOldBoundarySessions(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) DeleteOldChatDebugRuns(ctx context.Context, arg database.DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil {
|
||||
return 0, err
|
||||
|
||||
@@ -482,6 +482,10 @@ func (s *MethodTestSuite) TestBoundaryLogs() {
|
||||
dbm.EXPECT().DeleteOldBoundaryLogs(gomock.Any(), database.DeleteOldBoundaryLogsParams{}).Return(int64(0), nil).AnyTimes()
|
||||
check.Args(database.DeleteOldBoundaryLogsParams{}).Asserts(rbac.ResourceBoundaryLog, policy.ActionDelete)
|
||||
}))
|
||||
s.Run("DeleteOldBoundarySessions", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
dbm.EXPECT().DeleteOldBoundarySessions(gomock.Any(), database.DeleteOldBoundarySessionsParams{}).Return(int64(0), nil).AnyTimes()
|
||||
check.Args(database.DeleteOldBoundarySessionsParams{}).Asserts(rbac.ResourceBoundaryLog, policy.ActionDelete)
|
||||
}))
|
||||
}
|
||||
|
||||
func (s *MethodTestSuite) TestConnectionLogs() {
|
||||
|
||||
@@ -468,7 +468,7 @@ func BoundarySession(t testing.TB, db database.Store, seed database.BoundarySess
|
||||
session, err := db.InsertBoundarySession(genCtx, database.InsertBoundarySessionParams{
|
||||
ID: takeFirst(seed.ID, uuid.New()),
|
||||
WorkspaceAgentID: takeFirst(seed.WorkspaceAgentID, uuid.New()),
|
||||
OwnerID: takeFirst(seed.OwnerID, uuid.NullUUID{UUID: uuid.New(), Valid: true}),
|
||||
OwnerID: seed.OwnerID,
|
||||
ConfinedProcessName: takeFirst(seed.ConfinedProcessName, "claude-code"),
|
||||
StartedAt: takeFirst(seed.StartedAt, dbtime.Now()),
|
||||
UpdatedAt: takeFirst(seed.UpdatedAt, dbtime.Now()),
|
||||
|
||||
+8
@@ -722,6 +722,14 @@ func (m queryMetricsStore) DeleteOldBoundaryLogs(ctx context.Context, arg databa
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) DeleteOldBoundarySessions(ctx context.Context, arg database.DeleteOldBoundarySessionsParams) (int64, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.DeleteOldBoundarySessions(ctx, arg)
|
||||
m.queryLatencies.WithLabelValues("DeleteOldBoundarySessions").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldBoundarySessions").Inc()
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) DeleteOldChatDebugRuns(ctx context.Context, arg database.DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.DeleteOldChatDebugRuns(ctx, arg)
|
||||
|
||||
Generated
+15
@@ -1206,6 +1206,21 @@ func (mr *MockStoreMockRecorder) DeleteOldBoundaryLogs(ctx, arg any) *gomock.Cal
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldBoundaryLogs", reflect.TypeOf((*MockStore)(nil).DeleteOldBoundaryLogs), ctx, arg)
|
||||
}
|
||||
|
||||
// DeleteOldBoundarySessions mocks base method.
|
||||
func (m *MockStore) DeleteOldBoundarySessions(ctx context.Context, arg database.DeleteOldBoundarySessionsParams) (int64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "DeleteOldBoundarySessions", ctx, arg)
|
||||
ret0, _ := ret[0].(int64)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// DeleteOldBoundarySessions indicates an expected call of DeleteOldBoundarySessions.
|
||||
func (mr *MockStoreMockRecorder) DeleteOldBoundarySessions(ctx, arg any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldBoundarySessions", reflect.TypeOf((*MockStore)(nil).DeleteOldBoundarySessions), ctx, arg)
|
||||
}
|
||||
|
||||
// DeleteOldChatDebugRuns mocks base method.
|
||||
func (m *MockStore) DeleteOldChatDebugRuns(ctx context.Context, arg database.DeleteOldChatDebugRunsParams) (int64, error) {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
@@ -29,6 +29,10 @@ const (
|
||||
connectionLogsBatchSize = 10000
|
||||
// Batch size for audit log deletion.
|
||||
auditLogsBatchSize = 10000
|
||||
// Batch size for boundary log deletion.
|
||||
boundaryLogsBatchSize = 10000
|
||||
// Batch size for boundary session deletion.
|
||||
boundarySessionsBatchSize = 10000
|
||||
// Telemetry heartbeats are used to deduplicate events across replicas. We
|
||||
// don't need to persist heartbeat rows for longer than 24 hours, as they
|
||||
// are only used for deduplication across replicas. The time needs to be
|
||||
@@ -251,6 +255,26 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
}
|
||||
}
|
||||
|
||||
var purgedBoundaryLogs, purgedBoundarySessions int64
|
||||
boundaryLogsRetention := i.vals.Retention.BoundaryLogs.Value()
|
||||
if boundaryLogsRetention > 0 {
|
||||
deleteBoundaryLogsBefore := start.Add(-boundaryLogsRetention)
|
||||
purgedBoundaryLogs, err = tx.DeleteOldBoundaryLogs(ctx, database.DeleteOldBoundaryLogsParams{
|
||||
BeforeTime: deleteBoundaryLogsBefore,
|
||||
LimitCount: boundaryLogsBatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to delete old boundary logs: %w", err)
|
||||
}
|
||||
purgedBoundarySessions, err = tx.DeleteOldBoundarySessions(ctx, database.DeleteOldBoundarySessionsParams{
|
||||
BeforeTime: deleteBoundaryLogsBefore,
|
||||
LimitCount: boundarySessionsBatchSize,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to delete old boundary sessions: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var purgedChats, purgedChatFiles, purgedChatDebugRuns int64
|
||||
if purgeChats {
|
||||
purgedChats, purgedChatFiles, err = i.purgeChatsInTx(ctx, tx, start, chatRetentionDays)
|
||||
@@ -278,6 +302,8 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
slog.F("aibridge_records", purgedAIBridgeRecords),
|
||||
slog.F("connection_logs", purgedConnectionLogs),
|
||||
slog.F("audit_logs", purgedAuditLogs),
|
||||
slog.F("boundary_logs", purgedBoundaryLogs),
|
||||
slog.F("boundary_sessions", purgedBoundarySessions),
|
||||
slog.F("chats", purgedChats),
|
||||
slog.F("chat_files", purgedChatFiles),
|
||||
slog.F("chat_debug_runs", purgedChatDebugRuns),
|
||||
@@ -290,6 +316,8 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
|
||||
i.recordsPurged.WithLabelValues("aibridge_records").Add(float64(purgedAIBridgeRecords))
|
||||
i.recordsPurged.WithLabelValues("connection_logs").Add(float64(purgedConnectionLogs))
|
||||
i.recordsPurged.WithLabelValues("audit_logs").Add(float64(purgedAuditLogs))
|
||||
i.recordsPurged.WithLabelValues("boundary_logs").Add(float64(purgedBoundaryLogs))
|
||||
i.recordsPurged.WithLabelValues("boundary_sessions").Add(float64(purgedBoundarySessions))
|
||||
i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats))
|
||||
i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns))
|
||||
i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles))
|
||||
|
||||
@@ -1671,6 +1671,265 @@ func TestDeleteOldAuditLogs(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestDeleteOldBoundaryLogs(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2025, 1, 15, 7, 30, 0, 0, time.UTC)
|
||||
retentionPeriod := 90 * 24 * time.Hour
|
||||
beforeThreshold := now.Add(-retentionPeriod).Add(-24 * time.Hour) // 91 days ago (older than threshold, before the cutoff)
|
||||
afterThreshold := now.Add(-15 * 24 * time.Hour) // 15 days ago (newer than threshold, after the cutoff)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
retentionConfig codersdk.RetentionConfig
|
||||
oldLogTime time.Time
|
||||
recentLogTime *time.Time // nil means no recent log created
|
||||
expectOldDeleted bool
|
||||
expectedLogsRemaining int
|
||||
}{
|
||||
{
|
||||
name: "RetentionEnabled",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(retentionPeriod),
|
||||
},
|
||||
oldLogTime: beforeThreshold,
|
||||
recentLogTime: &afterThreshold,
|
||||
expectOldDeleted: true,
|
||||
expectedLogsRemaining: 1, // only recent log remains
|
||||
},
|
||||
{
|
||||
name: "RetentionDisabled",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(0),
|
||||
},
|
||||
oldLogTime: now.Add(-365 * 24 * time.Hour), // 1 year ago
|
||||
recentLogTime: nil,
|
||||
expectOldDeleted: false,
|
||||
expectedLogsRemaining: 1, // old log is kept
|
||||
},
|
||||
{
|
||||
name: "RetentionNegative",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(-retentionPeriod),
|
||||
},
|
||||
oldLogTime: now.Add(-365 * 24 * time.Hour), // 1 year ago
|
||||
recentLogTime: nil,
|
||||
expectOldDeleted: false,
|
||||
expectedLogsRemaining: 1, // old log is kept
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
clk := quartz.NewMock(t)
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
|
||||
// Create the prerequisite rows (user, org, template, workspace,
|
||||
// build, agent) needed to satisfy boundary_sessions foreign keys.
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
org := dbgen.Organization(t, db, database.Organization{})
|
||||
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
|
||||
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: org.ID, CreatedBy: user.ID})
|
||||
tmpl := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, ActiveVersionID: tv.ID, CreatedBy: user.ID})
|
||||
ws := dbgen.Workspace(t, db, database.WorkspaceTable{
|
||||
OwnerID: user.ID,
|
||||
OrganizationID: org.ID,
|
||||
TemplateID: tmpl.ID,
|
||||
})
|
||||
wb := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, now, 1)
|
||||
agent := mustCreateAgent(t, db, wb)
|
||||
|
||||
session := dbgen.BoundarySession(t, db, database.BoundarySession{
|
||||
WorkspaceAgentID: agent.ID,
|
||||
OwnerID: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
|
||||
// Create old boundary log.
|
||||
oldLogs := dbgen.BoundaryLogs(t, db, []database.BoundaryLog{{
|
||||
SessionID: session.ID,
|
||||
SequenceNumber: 0,
|
||||
CapturedAt: tc.oldLogTime,
|
||||
CreatedAt: tc.oldLogTime,
|
||||
}})
|
||||
oldLog := oldLogs[0]
|
||||
|
||||
// Create recent boundary log if specified.
|
||||
var recentLog database.BoundaryLog
|
||||
if tc.recentLogTime != nil {
|
||||
recentLogs := dbgen.BoundaryLogs(t, db, []database.BoundaryLog{{
|
||||
SessionID: session.ID,
|
||||
SequenceNumber: 1,
|
||||
CapturedAt: *tc.recentLogTime,
|
||||
CreatedAt: *tc.recentLogTime,
|
||||
}})
|
||||
recentLog = recentLogs[0]
|
||||
}
|
||||
|
||||
// Run the purge.
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
|
||||
Retention: tc.retentionConfig,
|
||||
}, prometheus.NewRegistry(), nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
// Verify results.
|
||||
logs, err := db.ListBoundaryLogsBySessionID(ctx, database.ListBoundaryLogsBySessionIDParams{
|
||||
SessionID: session.ID,
|
||||
LimitOpt: 100,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, logs, tc.expectedLogsRemaining, "unexpected number of boundary logs remaining")
|
||||
|
||||
logIDs := make([]uuid.UUID, len(logs))
|
||||
for i, l := range logs {
|
||||
logIDs[i] = l.ID
|
||||
}
|
||||
|
||||
if tc.expectOldDeleted {
|
||||
require.NotContains(t, logIDs, oldLog.ID, "old boundary log should be deleted")
|
||||
} else {
|
||||
require.Contains(t, logIDs, oldLog.ID, "old boundary log should NOT be deleted")
|
||||
}
|
||||
|
||||
if tc.recentLogTime != nil {
|
||||
require.Contains(t, logIDs, recentLog.ID, "recent boundary log should be kept")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteOldBoundarySessions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
now := time.Date(2025, 1, 15, 7, 30, 0, 0, time.UTC)
|
||||
retentionPeriod := 90 * 24 * time.Hour
|
||||
// oldTime is 91 days ago (past threshold).
|
||||
oldTime := now.Add(-retentionPeriod).Add(-24 * time.Hour)
|
||||
// recentTime is 15 days ago (within threshold).
|
||||
recentTime := now.Add(-15 * 24 * time.Hour)
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
retentionConfig codersdk.RetentionConfig
|
||||
sessionUpdatedAt time.Time
|
||||
// logTime is the captured_at for the single log inserted with the session.
|
||||
// Set to nil to create a session with no logs.
|
||||
logTime *time.Time
|
||||
expectSessionDeleted bool
|
||||
}{
|
||||
{
|
||||
name: "SessionDeletedWhenAllLogsExpired",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(retentionPeriod),
|
||||
},
|
||||
sessionUpdatedAt: oldTime,
|
||||
logTime: &oldTime, // log is old; will be purged first, leaving session empty
|
||||
expectSessionDeleted: true,
|
||||
},
|
||||
{
|
||||
name: "SessionKeptWhenRecentLogExists",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(retentionPeriod),
|
||||
},
|
||||
sessionUpdatedAt: oldTime,
|
||||
logTime: &recentTime, // recent log survives log purge, so session kept
|
||||
expectSessionDeleted: false,
|
||||
},
|
||||
{
|
||||
name: "SessionKeptWhenRetentionDisabled",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(0),
|
||||
},
|
||||
sessionUpdatedAt: oldTime,
|
||||
logTime: &oldTime,
|
||||
expectSessionDeleted: false,
|
||||
},
|
||||
{
|
||||
name: "SessionKeptWhenRetentionNegative",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(-retentionPeriod),
|
||||
},
|
||||
sessionUpdatedAt: oldTime,
|
||||
logTime: &oldTime,
|
||||
expectSessionDeleted: false,
|
||||
},
|
||||
{
|
||||
name: "SessionKeptWhenUpdatedAtRecent",
|
||||
retentionConfig: codersdk.RetentionConfig{
|
||||
BoundaryLogs: serpent.Duration(retentionPeriod),
|
||||
},
|
||||
sessionUpdatedAt: recentTime, // session itself is recent. NOT eligible for session purge
|
||||
logTime: nil, // no logs; but updated_at guard keeps it
|
||||
expectSessionDeleted: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
clk := quartz.NewMock(t)
|
||||
clk.Set(now).MustWait(ctx)
|
||||
|
||||
db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
||||
|
||||
// Create the prerequisite rows needed to satisfy boundary_sessions FKs.
|
||||
user := dbgen.User(t, db, database.User{})
|
||||
org := dbgen.Organization(t, db, database.Organization{})
|
||||
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
|
||||
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: org.ID, CreatedBy: user.ID})
|
||||
tmpl := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, ActiveVersionID: tv.ID, CreatedBy: user.ID})
|
||||
ws := dbgen.Workspace(t, db, database.WorkspaceTable{
|
||||
OwnerID: user.ID,
|
||||
OrganizationID: org.ID,
|
||||
TemplateID: tmpl.ID,
|
||||
})
|
||||
wb := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, now, 1)
|
||||
agent := mustCreateAgent(t, db, wb)
|
||||
|
||||
session := dbgen.BoundarySession(t, db, database.BoundarySession{
|
||||
WorkspaceAgentID: agent.ID,
|
||||
OwnerID: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedAt: tc.sessionUpdatedAt,
|
||||
})
|
||||
|
||||
if tc.logTime != nil {
|
||||
dbgen.BoundaryLogs(t, db, []database.BoundaryLog{{
|
||||
SessionID: session.ID,
|
||||
SequenceNumber: 0,
|
||||
CapturedAt: *tc.logTime,
|
||||
CreatedAt: *tc.logTime,
|
||||
}})
|
||||
}
|
||||
|
||||
// Run the purge.
|
||||
done := awaitDoTick(ctx, t, clk)
|
||||
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
|
||||
Retention: tc.retentionConfig,
|
||||
}, prometheus.NewRegistry(), nopAuditorPtr(t), dbpurge.WithClock(clk))
|
||||
defer closer.Close()
|
||||
testutil.TryReceive(ctx, t, done)
|
||||
|
||||
// Verify session presence/absence.
|
||||
_, err := db.GetBoundarySessionByID(ctx, session.ID)
|
||||
if tc.expectSessionDeleted {
|
||||
require.ErrorIs(t, err, sql.ErrNoRows, "session should have been deleted")
|
||||
} else {
|
||||
require.NoError(t, err, "session should still exist")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteExpiredAPIKeys(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Generated
+3
@@ -175,6 +175,9 @@ type sqlcQuerier interface {
|
||||
// Deletes boundary logs older than the given time, bounded by a row limit
|
||||
// to avoid long-running transactions.
|
||||
DeleteOldBoundaryLogs(ctx context.Context, arg DeleteOldBoundaryLogsParams) (int64, error)
|
||||
// Deletes boundary sessions that have aged past retention and no longer
|
||||
// have any associated logs.
|
||||
DeleteOldBoundarySessions(ctx context.Context, arg DeleteOldBoundarySessionsParams) (int64, error)
|
||||
// updated_at is the retention clock, so the window starts after the run
|
||||
// stops being written to.
|
||||
// Intentionally no finished_at IS NOT NULL guard: abandoned in-flight rows
|
||||
|
||||
Generated
+31
@@ -3622,6 +3622,37 @@ func (q *sqlQuerier) DeleteOldBoundaryLogs(ctx context.Context, arg DeleteOldBou
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
const deleteOldBoundarySessions = `-- name: DeleteOldBoundarySessions :execrows
|
||||
WITH old_sessions AS (
|
||||
SELECT bs.id
|
||||
FROM boundary_sessions bs
|
||||
WHERE bs.updated_at < $1::timestamptz
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM boundary_logs bl WHERE bl.session_id = bs.id
|
||||
)
|
||||
ORDER BY bs.updated_at ASC
|
||||
LIMIT $2
|
||||
)
|
||||
DELETE FROM boundary_sessions
|
||||
USING old_sessions
|
||||
WHERE boundary_sessions.id = old_sessions.id
|
||||
`
|
||||
|
||||
type DeleteOldBoundarySessionsParams struct {
|
||||
BeforeTime time.Time `db:"before_time" json:"before_time"`
|
||||
LimitCount int32 `db:"limit_count" json:"limit_count"`
|
||||
}
|
||||
|
||||
// Deletes boundary sessions that have aged past retention and no longer
|
||||
// have any associated logs.
|
||||
func (q *sqlQuerier) DeleteOldBoundarySessions(ctx context.Context, arg DeleteOldBoundarySessionsParams) (int64, error) {
|
||||
result, err := q.db.ExecContext(ctx, deleteOldBoundarySessions, arg.BeforeTime, arg.LimitCount)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
const getBoundaryLogByID = `-- name: GetBoundaryLogByID :one
|
||||
SELECT id, session_id, sequence_number, captured_at, created_at, proto, method, detail, matched_rule FROM boundary_logs WHERE id = $1
|
||||
`
|
||||
|
||||
@@ -77,3 +77,20 @@ WITH old_logs AS (
|
||||
DELETE FROM boundary_logs
|
||||
USING old_logs
|
||||
WHERE boundary_logs.id = old_logs.id;
|
||||
|
||||
-- name: DeleteOldBoundarySessions :execrows
|
||||
-- Deletes boundary sessions that have aged past retention and no longer
|
||||
-- have any associated logs.
|
||||
WITH old_sessions AS (
|
||||
SELECT bs.id
|
||||
FROM boundary_sessions bs
|
||||
WHERE bs.updated_at < @before_time::timestamptz
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM boundary_logs bl WHERE bl.session_id = bs.id
|
||||
)
|
||||
ORDER BY bs.updated_at ASC
|
||||
LIMIT @limit_count
|
||||
)
|
||||
DELETE FROM boundary_sessions
|
||||
USING old_sessions
|
||||
WHERE boundary_sessions.id = old_sessions.id;
|
||||
|
||||
@@ -1194,6 +1194,12 @@ type RetentionConfig struct {
|
||||
// Logs from the latest build are always retained regardless of age.
|
||||
// Defaults to 7 days to preserve existing behavior.
|
||||
WorkspaceAgentLogs serpent.Duration `json:"workspace_agent_logs" typescript:",notnull"`
|
||||
// BoundaryLogs controls how long boundary audit log entries are
|
||||
// retained. Boundary logs record every HTTP request processed by
|
||||
// a Boundary confinement proxy. Set to 0 to disable automatic
|
||||
// deletion (keep indefinitely). Adjust to match your
|
||||
// organization's regulatory requirements.
|
||||
BoundaryLogs serpent.Duration `json:"boundary_logs" typescript:",notnull"`
|
||||
}
|
||||
|
||||
type NotificationsConfig struct {
|
||||
@@ -4703,6 +4709,17 @@ Write out the current server config as YAML to stdout.`,
|
||||
YAML: "workspace_agent_logs",
|
||||
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
|
||||
},
|
||||
{
|
||||
Name: "Boundary Log Retention",
|
||||
Description: "How long boundary audit log entries are retained. Boundary logs record HTTP requests processed by a Boundary confinement proxy. Set to 0 to disable automatic deletion (keep indefinitely). Adjust to match your organization's regulatory requirements.",
|
||||
Flag: "boundary-log-retention",
|
||||
Env: "CODER_BOUNDARY_LOG_RETENTION",
|
||||
Value: &c.Retention.BoundaryLogs,
|
||||
Default: "0",
|
||||
Group: &deploymentGroupRetention,
|
||||
YAML: "boundary_logs",
|
||||
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
|
||||
},
|
||||
{
|
||||
Name: "Enable Authorization Recordings",
|
||||
Description: "All api requests will have a header including all authorization calls made during the request. " +
|
||||
|
||||
Generated
+1
@@ -534,6 +534,7 @@ curl -X GET http://coder-server:8080/api/v2/deployment/config \
|
||||
"retention": {
|
||||
"api_keys": 0,
|
||||
"audit_logs": 0,
|
||||
"boundary_logs": 0,
|
||||
"connection_logs": 0,
|
||||
"workspace_agent_logs": 0
|
||||
},
|
||||
|
||||
Generated
+10
-6
@@ -5803,6 +5803,7 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
|
||||
"retention": {
|
||||
"api_keys": 0,
|
||||
"audit_logs": 0,
|
||||
"boundary_logs": 0,
|
||||
"connection_logs": 0,
|
||||
"workspace_agent_logs": 0
|
||||
},
|
||||
@@ -6403,6 +6404,7 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o
|
||||
"retention": {
|
||||
"api_keys": 0,
|
||||
"audit_logs": 0,
|
||||
"boundary_logs": 0,
|
||||
"connection_logs": 0,
|
||||
"workspace_agent_logs": 0
|
||||
},
|
||||
@@ -11095,6 +11097,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit|
|
||||
{
|
||||
"api_keys": 0,
|
||||
"audit_logs": 0,
|
||||
"boundary_logs": 0,
|
||||
"connection_logs": 0,
|
||||
"workspace_agent_logs": 0
|
||||
}
|
||||
@@ -11102,12 +11105,13 @@ Only certain features set these fields: - FeatureManagedAgentLimit|
|
||||
|
||||
### Properties
|
||||
|
||||
| Name | Type | Required | Restrictions | Description |
|
||||
|------------------------|---------|----------|--------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `api_keys` | integer | false | | Api keys controls how long expired API keys are retained before being deleted. Keys are only deleted if they have been expired for at least this duration. Defaults to 7 days to preserve existing behavior. |
|
||||
| `audit_logs` | integer | false | | Audit logs controls how long audit log entries are retained. Set to 0 to disable (keep indefinitely). |
|
||||
| `connection_logs` | integer | false | | Connection logs controls how long connection log entries are retained. Set to 0 to disable (keep indefinitely). |
|
||||
| `workspace_agent_logs` | integer | false | | Workspace agent logs controls how long workspace agent logs are retained. Logs are deleted if the agent hasn't connected within this period. Logs from the latest build are always retained regardless of age. Defaults to 7 days to preserve existing behavior. |
|
||||
| Name | Type | Required | Restrictions | Description |
|
||||
|------------------------|---------|----------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| `api_keys` | integer | false | | Api keys controls how long expired API keys are retained before being deleted. Keys are only deleted if they have been expired for at least this duration. Defaults to 7 days to preserve existing behavior. |
|
||||
| `audit_logs` | integer | false | | Audit logs controls how long audit log entries are retained. Set to 0 to disable (keep indefinitely). |
|
||||
| `boundary_logs` | integer | false | | Boundary logs controls how long boundary audit log entries are retained. Boundary logs record every HTTP request processed by a Boundary confinement proxy. Set to 0 to disable automatic deletion (keep indefinitely). Adjust to match your organization's regulatory requirements. |
|
||||
| `connection_logs` | integer | false | | Connection logs controls how long connection log entries are retained. Set to 0 to disable (keep indefinitely). |
|
||||
| `workspace_agent_logs` | integer | false | | Workspace agent logs controls how long workspace agent logs are retained. Logs are deleted if the agent hasn't connected within this period. Logs from the latest build are always retained regardless of age. Defaults to 7 days to preserve existing behavior. |
|
||||
|
||||
## codersdk.Role
|
||||
|
||||
|
||||
Generated
+11
@@ -2089,6 +2089,17 @@ How long expired API keys are retained before being deleted. Keeping expired key
|
||||
|
||||
How long workspace agent logs are retained. Logs from non-latest builds are deleted if the agent hasn't connected within this period. Logs from the latest build are always retained. Set to 0 to disable automatic deletion.
|
||||
|
||||
### --boundary-log-retention
|
||||
|
||||
| | |
|
||||
|-------------|--------------------------------------------|
|
||||
| Type | <code>duration</code> |
|
||||
| Environment | <code>$CODER_BOUNDARY_LOG_RETENTION</code> |
|
||||
| YAML | <code>retention.boundary_logs</code> |
|
||||
| Default | <code>0</code> |
|
||||
|
||||
How long boundary audit log entries are retained. Boundary logs record HTTP requests processed by a Boundary confinement proxy. Set to 0 to disable automatic deletion (keep indefinitely). Adjust to match your organization's regulatory requirements.
|
||||
|
||||
### --disable-template-builder
|
||||
|
||||
| | |
|
||||
|
||||
@@ -879,6 +879,12 @@ that data type.
|
||||
indefinitely). We advise keeping audit logs for at least a year, and
|
||||
in accordance with your compliance requirements.
|
||||
|
||||
--boundary-log-retention duration, $CODER_BOUNDARY_LOG_RETENTION (default: 0)
|
||||
How long boundary audit log entries are retained. Boundary logs record
|
||||
HTTP requests processed by a Boundary confinement proxy. Set to 0 to
|
||||
disable automatic deletion (keep indefinitely). Adjust to match your
|
||||
organization's regulatory requirements.
|
||||
|
||||
--connection-logs-retention duration, $CODER_CONNECTION_LOGS_RETENTION (default: 0)
|
||||
How long connection log entries are retained. Set to 0 to disable
|
||||
(keep indefinitely).
|
||||
|
||||
Generated
+8
@@ -7244,6 +7244,14 @@ export interface RetentionConfig {
|
||||
* Defaults to 7 days to preserve existing behavior.
|
||||
*/
|
||||
readonly workspace_agent_logs: number;
|
||||
/**
|
||||
* BoundaryLogs controls how long boundary audit log entries are
|
||||
* retained. Boundary logs record every HTTP request processed by
|
||||
* a Boundary confinement proxy. Set to 0 to disable automatic
|
||||
* deletion (keep indefinitely). Adjust to match your
|
||||
* organization's regulatory requirements.
|
||||
*/
|
||||
readonly boundary_logs: number;
|
||||
}
|
||||
|
||||
// From codersdk/roles.go
|
||||
|
||||
Reference in New Issue
Block a user