opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement observability of notifications subsystem (#13799)

Browse Source
This commit is contained in:
Danny Kopping
2024-07-11 10:57:49 +02:00
committed by GitHub
parent a6d66cc7ec
commit b2dab3308d
22 changed files with 769 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/dbauthz/dbauthz.go
+2 -2
View File
@@ -1143,9 +1143,9 @@ func (q *querier) DeleteWorkspaceAgentPortSharesByTemplate(ctx context.Context,
return q.db.DeleteWorkspaceAgentPortSharesByTemplate(ctx, templateID)
}
func (q *querier) EnqueueNotificationMessage(ctx context.Context, arg database.EnqueueNotificationMessageParams) (database.NotificationMessage, error) {
func (q *querier) EnqueueNotificationMessage(ctx context.Context, arg database.EnqueueNotificationMessageParams) error {
if err := q.authorizeContext(ctx, policy.ActionCreate, rbac.ResourceSystem); err != nil {
return database.NotificationMessage{}, err
return err
}
return q.db.EnqueueNotificationMessage(ctx, arg)
}
coderd/database/dbmem/dbmem.go
+15 -10
View File
@@ -935,12 +935,17 @@ func (q *FakeQuerier) AcquireNotificationMessages(_ context.Context, arg databas
q.mutex.Lock()
defer q.mutex.Unlock()
var out []database.AcquireNotificationMessagesRow
for _, nm := range q.notificationMessages {
if len(out) >= int(arg.Count) {
break
}
// Shift the first "Count" notifications off the slice (FIFO).
sz := len(q.notificationMessages)
if sz > int(arg.Count) {
sz = int(arg.Count)
}
list := q.notificationMessages[:sz]
q.notificationMessages = q.notificationMessages[sz:]
var out []database.AcquireNotificationMessagesRow
for _, nm := range list {
acquirableStatuses := []database.NotificationMessageStatus{database.NotificationMessageStatusPending, database.NotificationMessageStatusTemporaryFailure}
if !slices.Contains(acquirableStatuses, nm.Status) {
continue
@@ -956,9 +961,9 @@ func (q *FakeQuerier) AcquireNotificationMessages(_ context.Context, arg databas
ID: nm.ID,
Payload: nm.Payload,
Method: nm.Method,
CreatedBy: nm.CreatedBy,
TitleTemplate: "This is a title with {{.Labels.variable}}",
BodyTemplate: "This is a body with {{.Labels.variable}}",
TemplateID: nm.NotificationTemplateID,
})
}
@@ -1815,10 +1820,10 @@ func (q *FakeQuerier) DeleteWorkspaceAgentPortSharesByTemplate(_ context.Context
return nil
}
func (q *FakeQuerier) EnqueueNotificationMessage(_ context.Context, arg database.EnqueueNotificationMessageParams) (database.NotificationMessage, error) {
func (q *FakeQuerier) EnqueueNotificationMessage(_ context.Context, arg database.EnqueueNotificationMessageParams) error {
err := validateDatabaseType(arg)
if err != nil {
return database.NotificationMessage{}, err
return err
}
q.mutex.Lock()
@@ -1827,7 +1832,7 @@ func (q *FakeQuerier) EnqueueNotificationMessage(_ context.Context, arg database
var payload types.MessagePayload
err = json.Unmarshal(arg.Payload, &payload)
if err != nil {
return database.NotificationMessage{}, err
return err
}
nm := database.NotificationMessage{
@@ -1845,7 +1850,7 @@ func (q *FakeQuerier) EnqueueNotificationMessage(_ context.Context, arg database
q.notificationMessages = append(q.notificationMessages, nm)
return nm, err
return err
}
func (q *FakeQuerier) FavoriteWorkspace(_ context.Context, arg uuid.UUID) error {
coderd/database/dbmetrics/dbmetrics.go
+3 -3
View File
@@ -382,11 +382,11 @@ func (m metricsStore) DeleteWorkspaceAgentPortSharesByTemplate(ctx context.Conte
return r0
}
func (m metricsStore) EnqueueNotificationMessage(ctx context.Context, arg database.EnqueueNotificationMessageParams) (database.NotificationMessage, error) {
func (m metricsStore) EnqueueNotificationMessage(ctx context.Context, arg database.EnqueueNotificationMessageParams) error {
start := time.Now()
r0, r1 := m.s.EnqueueNotificationMessage(ctx, arg)
r0 := m.s.EnqueueNotificationMessage(ctx, arg)
m.queryLatencies.WithLabelValues("EnqueueNotificationMessage").Observe(time.Since(start).Seconds())
return r0, r1
return r0
}
func (m metricsStore) FavoriteWorkspace(ctx context.Context, arg uuid.UUID) error {
coderd/database/dbmock/dbmock.go
+3 -4
View File
@@ -659,12 +659,11 @@ func (mr *MockStoreMockRecorder) DeleteWorkspaceAgentPortSharesByTemplate(arg0,
}
// EnqueueNotificationMessage mocks base method.
func (m *MockStore) EnqueueNotificationMessage(arg0 context.Context, arg1 database.EnqueueNotificationMessageParams) (database.NotificationMessage, error) {
func (m *MockStore) EnqueueNotificationMessage(arg0 context.Context, arg1 database.EnqueueNotificationMessageParams) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "EnqueueNotificationMessage", arg0, arg1)
ret0, _ := ret[0].(database.NotificationMessage)
ret1, _ := ret[1].(error)
return ret0, ret1
ret0, _ := ret[0].(error)
return ret0
}
// EnqueueNotificationMessage indicates an expected call of EnqueueNotificationMessage.
coderd/database/dump.sql
Generated
+2 -1
View File
@@ -563,7 +563,8 @@ CREATE TABLE notification_messages (
created_at timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
updated_at timestamp with time zone,
leased_until timestamp with time zone,
next_retry_after timestamp with time zone
next_retry_after timestamp with time zone,
queued_seconds double precision
);
CREATE TABLE notification_templates (
coderd/database/migrations/000225_notifications_metrics.down.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE notification_messages
DROP COLUMN IF EXISTS queued_seconds;
coderd/database/migrations/000225_notifications_metrics.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE notification_messages
ADD COLUMN queued_seconds FLOAT NULL;
coderd/database/models.go
+1
View File
@@ -2031,6 +2031,7 @@ type NotificationMessage struct {
UpdatedAt sql.NullTime `db:"updated_at" json:"updated_at"`
LeasedUntil sql.NullTime `db:"leased_until" json:"leased_until"`
NextRetryAfter sql.NullTime `db:"next_retry_after" json:"next_retry_after"`
QueuedSeconds sql.NullFloat64 `db:"queued_seconds" json:"queued_seconds"`
}
// Templates from which to create notification messages.
coderd/database/querier.go
+1 -1
View File
@@ -100,7 +100,7 @@ type sqlcQuerier interface {
DeleteTailnetTunnel(ctx context.Context, arg DeleteTailnetTunnelParams) (DeleteTailnetTunnelRow, error)
DeleteWorkspaceAgentPortShare(ctx context.Context, arg DeleteWorkspaceAgentPortShareParams) error
DeleteWorkspaceAgentPortSharesByTemplate(ctx context.Context, templateID uuid.UUID) error
EnqueueNotificationMessage(ctx context.Context, arg EnqueueNotificationMessageParams) (NotificationMessage, error)
EnqueueNotificationMessage(ctx context.Context, arg EnqueueNotificationMessageParams) error
FavoriteWorkspace(ctx context.Context, id uuid.UUID) error
// This is used to build up the notification_message's JSON payload.
FetchNewMessageMetadata(ctx context.Context, arg FetchNewMessageMetadataParams) (FetchNewMessageMetadataRow, error)
coderd/database/queries.sql.go
+23 -31
View File
@@ -3292,7 +3292,8 @@ const acquireNotificationMessages = `-- name: AcquireNotificationMessages :many
WITH acquired AS (
UPDATE
notification_messages
SET updated_at = NOW(),
SET queued_seconds = GREATEST(0, EXTRACT(EPOCH FROM (NOW() - updated_at)))::FLOAT,
updated_at = NOW(),
status = 'leased'::notification_message_status,
status_reason = 'Leased by notifier ' || $1::uuid,
leased_until = NOW() + CONCAT($2::int, ' seconds')::interval
@@ -3328,14 +3329,16 @@ WITH acquired AS (
FOR UPDATE OF nm
SKIP LOCKED
LIMIT $4)
RETURNING id, notification_template_id, user_id, method, status, status_reason, created_by, payload, attempt_count, targets, created_at, updated_at, leased_until, next_retry_after)
RETURNING id, notification_template_id, user_id, method, status, status_reason, created_by, payload, attempt_count, targets, created_at, updated_at, leased_until, next_retry_after, queued_seconds)
SELECT
-- message
nm.id,
nm.payload,
nm.method,
nm.created_by,
nm.attempt_count::int AS attempt_count,
nm.queued_seconds::float AS queued_seconds,
-- template
nt.id AS template_id,
nt.title_template,
nt.body_template
FROM acquired nm
@@ -3353,7 +3356,9 @@ type AcquireNotificationMessagesRow struct {
ID uuid.UUID `db:"id" json:"id"`
Payload json.RawMessage `db:"payload" json:"payload"`
Method NotificationMethod `db:"method" json:"method"`
CreatedBy string `db:"created_by" json:"created_by"`
AttemptCount int32 `db:"attempt_count" json:"attempt_count"`
QueuedSeconds float64 `db:"queued_seconds" json:"queued_seconds"`
TemplateID uuid.UUID `db:"template_id" json:"template_id"`
TitleTemplate string `db:"title_template" json:"title_template"`
BodyTemplate string `db:"body_template" json:"body_template"`
}
@@ -3386,7 +3391,9 @@ func (q *sqlQuerier) AcquireNotificationMessages(ctx context.Context, arg Acquir
&i.ID,
&i.Payload,
&i.Method,
&i.CreatedBy,
&i.AttemptCount,
&i.QueuedSeconds,
&i.TemplateID,
&i.TitleTemplate,
&i.BodyTemplate,
); err != nil {
@@ -3405,7 +3412,8 @@ func (q *sqlQuerier) AcquireNotificationMessages(ctx context.Context, arg Acquir
const bulkMarkNotificationMessagesFailed = `-- name: BulkMarkNotificationMessagesFailed :execrows
UPDATE notification_messages
SET updated_at = subquery.failed_at,
SET queued_seconds = 0,
updated_at = subquery.failed_at,
attempt_count = attempt_count + 1,
status = CASE
WHEN attempt_count + 1 < $1::int THEN subquery.status
@@ -3448,13 +3456,14 @@ func (q *sqlQuerier) BulkMarkNotificationMessagesFailed(ctx context.Context, arg
const bulkMarkNotificationMessagesSent = `-- name: BulkMarkNotificationMessagesSent :execrows
UPDATE notification_messages
SET updated_at = new_values.sent_at,
SET queued_seconds = 0,
updated_at = new_values.sent_at,
attempt_count = attempt_count + 1,
status = 'sent'::notification_message_status,
status_reason = NULL,
leased_until = NULL,
next_retry_after = NULL
FROM (SELECT UNNEST($1::uuid[]) AS id,
FROM (SELECT UNNEST($1::uuid[]) AS id,
UNNEST($2::timestamptz[]) AS sent_at)
AS new_values
WHERE notification_messages.id = new_values.id
@@ -3488,7 +3497,7 @@ func (q *sqlQuerier) DeleteOldNotificationMessages(ctx context.Context) error {
return err
}
const enqueueNotificationMessage = `-- name: EnqueueNotificationMessage :one
const enqueueNotificationMessage = `-- name: EnqueueNotificationMessage :exec
INSERT INTO notification_messages (id, notification_template_id, user_id, method, payload, targets, created_by)
VALUES ($1,
$2,
@@ -3497,7 +3506,6 @@ VALUES ($1,
$5::jsonb,
$6,
$7)
RETURNING id, notification_template_id, user_id, method, status, status_reason, created_by, payload, attempt_count, targets, created_at, updated_at, leased_until, next_retry_after
`
type EnqueueNotificationMessageParams struct {
@@ -3510,8 +3518,8 @@ type EnqueueNotificationMessageParams struct {
CreatedBy string `db:"created_by" json:"created_by"`
}
func (q *sqlQuerier) EnqueueNotificationMessage(ctx context.Context, arg EnqueueNotificationMessageParams) (NotificationMessage, error) {
row := q.db.QueryRowContext(ctx, enqueueNotificationMessage,
func (q *sqlQuerier) EnqueueNotificationMessage(ctx context.Context, arg EnqueueNotificationMessageParams) error {
_, err := q.db.ExecContext(ctx, enqueueNotificationMessage,
arg.ID,
arg.NotificationTemplateID,
arg.UserID,
@@ -3520,24 +3528,7 @@ func (q *sqlQuerier) EnqueueNotificationMessage(ctx context.Context, arg Enqueue
pq.Array(arg.Targets),
arg.CreatedBy,
)
var i NotificationMessage
err := row.Scan(
&i.ID,
&i.NotificationTemplateID,
&i.UserID,
&i.Method,
&i.Status,
&i.StatusReason,
&i.CreatedBy,
&i.Payload,
&i.AttemptCount,
pq.Array(&i.Targets),
&i.CreatedAt,
&i.UpdatedAt,
&i.LeasedUntil,
&i.NextRetryAfter,
)
return i, err
return err
}
const fetchNewMessageMetadata = `-- name: FetchNewMessageMetadata :one
@@ -3580,7 +3571,7 @@ func (q *sqlQuerier) FetchNewMessageMetadata(ctx context.Context, arg FetchNewMe
}
const getNotificationMessagesByStatus = `-- name: GetNotificationMessagesByStatus :many
SELECT id, notification_template_id, user_id, method, status, status_reason, created_by, payload, attempt_count, targets, created_at, updated_at, leased_until, next_retry_after FROM notification_messages WHERE status = $1 LIMIT $2::int
SELECT id, notification_template_id, user_id, method, status, status_reason, created_by, payload, attempt_count, targets, created_at, updated_at, leased_until, next_retry_after, queued_seconds FROM notification_messages WHERE status = $1 LIMIT $2::int
`
type GetNotificationMessagesByStatusParams struct {
@@ -3612,6 +3603,7 @@ func (q *sqlQuerier) GetNotificationMessagesByStatus(ctx context.Context, arg Ge
&i.UpdatedAt,
&i.LeasedUntil,
&i.NextRetryAfter,
&i.QueuedSeconds,
); err != nil {
return nil, err
}
coderd/database/queries/notifications.sql
+12 -8
View File
@@ -10,7 +10,7 @@ FROM notification_templates nt,
WHERE nt.id = @notification_template_id
AND u.id = @user_id;
-- name: EnqueueNotificationMessage :one
-- name: EnqueueNotificationMessage :exec
INSERT INTO notification_messages (id, notification_template_id, user_id, method, payload, targets, created_by)
VALUES (@id,
@notification_template_id,
@@ -18,8 +18,7 @@ VALUES (@id,
@method::notification_method,
@payload::jsonb,
@targets,
@created_by)
RETURNING *;
@created_by);
-- Acquires the lease for a given count of notification messages, to enable concurrent dequeuing and subsequent sending.
-- Only rows that aren't already leased (or ones which are leased but have exceeded their lease period) are returned.
@@ -36,7 +35,8 @@ RETURNING *;
WITH acquired AS (
UPDATE
notification_messages
SET updated_at = NOW(),
SET queued_seconds = GREATEST(0, EXTRACT(EPOCH FROM (NOW() - updated_at)))::FLOAT,
updated_at = NOW(),
status = 'leased'::notification_message_status,
status_reason = 'Leased by notifier ' || sqlc.arg('notifier_id')::uuid,
leased_until = NOW() + CONCAT(sqlc.arg('lease_seconds')::int, ' seconds')::interval
@@ -78,8 +78,10 @@ SELECT
nm.id,
nm.payload,
nm.method,
nm.created_by,
nm.attempt_count::int AS attempt_count,
nm.queued_seconds::float AS queued_seconds,
-- template
nt.id AS template_id,
nt.title_template,
nt.body_template
FROM acquired nm
@@ -87,7 +89,8 @@ FROM acquired nm
-- name: BulkMarkNotificationMessagesFailed :execrows
UPDATE notification_messages
SET updated_at = subquery.failed_at,
SET queued_seconds = 0,
updated_at = subquery.failed_at,
attempt_count = attempt_count + 1,
status = CASE
WHEN attempt_count + 1 < @max_attempts::int THEN subquery.status
@@ -105,13 +108,14 @@ WHERE notification_messages.id = subquery.id;
-- name: BulkMarkNotificationMessagesSent :execrows
UPDATE notification_messages
SET updated_at = new_values.sent_at,
SET queued_seconds = 0,
updated_at = new_values.sent_at,
attempt_count = attempt_count + 1,
status = 'sent'::notification_message_status,
status_reason = NULL,
leased_until = NULL,
next_retry_after = NULL
FROM (SELECT UNNEST(@ids::uuid[]) AS id,
FROM (SELECT UNNEST(@ids::uuid[]) AS id,
UNNEST(@sent_ats::timestamptz[]) AS sent_at)
AS new_values
WHERE notification_messages.id = new_values.id;