mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
perf: optimize prebuilds membership reconciliation to check orgs not presets (#20493)
## Description The membership reconciliation ensures the prebuilds system user is a member of all organizations with prebuilds configured. To support prebuilds quota management, each organization must have a prebuilds group that the system user belongs to. ## Problem Previously, membership reconciliation iterated over all presets to check and update membership status. This meant database queries `GetGroupByOrgAndName` and `InsertGroupMember` were executed for each preset. Since presets are unique combinations of `(organization, template, template version, preset)`, this resulted in several redundant checks for the same organization. In dogfood, `InsertGroupMember` was called thousands of times per day, even though memberships were already configured ([internal Grafana dashboard link](https://grafana.dev.coder.com/goto/46MZ1UgDg?orgId=1)) <img width="5382" height="1788" alt="Screenshot 2025-10-28 at 16 01 36" src="https://github.com/user-attachments/assets/757b7253-106f-4f72-8586-8e2ede9f18db" /> ## Solution This PR introduces `GetOrganizationsWithPrebuildStatus`, a single query that returns: * All unique organizations with prebuilds configured * Whether the prebuilds user is a member of each organization * Whether the prebuilds group exists in each organization * Whether the prebuilds user is in the prebuilds group The membership reconciliation logic now: * Fetches status for all organizations in one query * Only performs inserts for organizations missing required memberships or groups * Safely handles concurrent operations via unique constraint violations * This reduces database load from `O(presets)` to `O(organizations)` per reconciliation loop, with a single read query when everything is configured. ## Changes * Add `GetOrganizationsWithPrebuildStatus` SQL query * Update `membership.ReconcileAll` to use organization-based reconciliation instead of preset-based * Update tests to reflect new behavior Related to internal thread: https://codercom.slack.com/archives/C07GRNNRW03/p1760535570381369
This commit is contained in:
@@ -8285,6 +8285,93 @@ func (q *sqlQuerier) FindMatchingPresetID(ctx context.Context, arg FindMatchingP
|
||||
return template_version_preset_id, err
|
||||
}
|
||||
|
||||
const getOrganizationsWithPrebuildStatus = `-- name: GetOrganizationsWithPrebuildStatus :many
|
||||
WITH orgs_with_prebuilds AS (
|
||||
-- Get unique organizations that have presets with prebuilds configured
|
||||
SELECT DISTINCT o.id, o.name
|
||||
FROM organizations o
|
||||
INNER JOIN templates t ON t.organization_id = o.id
|
||||
INNER JOIN template_versions tv ON tv.template_id = t.id
|
||||
INNER JOIN template_version_presets tvp ON tvp.template_version_id = tv.id
|
||||
WHERE tvp.desired_instances IS NOT NULL
|
||||
),
|
||||
prebuild_user_membership AS (
|
||||
-- Check if the user is a member of the organizations
|
||||
SELECT om.organization_id
|
||||
FROM organization_members om
|
||||
INNER JOIN orgs_with_prebuilds owp ON owp.id = om.organization_id
|
||||
WHERE om.user_id = $1::uuid
|
||||
),
|
||||
prebuild_groups AS (
|
||||
-- Check if the organizations have the prebuilds group
|
||||
SELECT g.organization_id, g.id as group_id
|
||||
FROM groups g
|
||||
INNER JOIN orgs_with_prebuilds owp ON owp.id = g.organization_id
|
||||
WHERE g.name = $2::text
|
||||
),
|
||||
prebuild_group_membership AS (
|
||||
-- Check if the user is in the prebuilds group
|
||||
SELECT pg.organization_id
|
||||
FROM prebuild_groups pg
|
||||
INNER JOIN group_members gm ON gm.group_id = pg.group_id
|
||||
WHERE gm.user_id = $1::uuid
|
||||
)
|
||||
SELECT
|
||||
owp.id AS organization_id,
|
||||
owp.name AS organization_name,
|
||||
(pum.organization_id IS NOT NULL)::boolean AS has_prebuild_user,
|
||||
pg.group_id AS prebuilds_group_id,
|
||||
(pgm.organization_id IS NOT NULL)::boolean AS has_prebuild_user_in_group
|
||||
FROM orgs_with_prebuilds owp
|
||||
LEFT JOIN prebuild_groups pg ON pg.organization_id = owp.id
|
||||
LEFT JOIN prebuild_user_membership pum ON pum.organization_id = owp.id
|
||||
LEFT JOIN prebuild_group_membership pgm ON pgm.organization_id = owp.id
|
||||
`
|
||||
|
||||
type GetOrganizationsWithPrebuildStatusParams struct {
|
||||
UserID uuid.UUID `db:"user_id" json:"user_id"`
|
||||
GroupName string `db:"group_name" json:"group_name"`
|
||||
}
|
||||
|
||||
type GetOrganizationsWithPrebuildStatusRow struct {
|
||||
OrganizationID uuid.UUID `db:"organization_id" json:"organization_id"`
|
||||
OrganizationName string `db:"organization_name" json:"organization_name"`
|
||||
HasPrebuildUser bool `db:"has_prebuild_user" json:"has_prebuild_user"`
|
||||
PrebuildsGroupID uuid.NullUUID `db:"prebuilds_group_id" json:"prebuilds_group_id"`
|
||||
HasPrebuildUserInGroup bool `db:"has_prebuild_user_in_group" json:"has_prebuild_user_in_group"`
|
||||
}
|
||||
|
||||
// GetOrganizationsWithPrebuildStatus returns organizations with prebuilds configured and their
|
||||
// membership status for the prebuilds system user (org membership, group existence, group membership).
|
||||
func (q *sqlQuerier) GetOrganizationsWithPrebuildStatus(ctx context.Context, arg GetOrganizationsWithPrebuildStatusParams) ([]GetOrganizationsWithPrebuildStatusRow, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getOrganizationsWithPrebuildStatus, arg.UserID, arg.GroupName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []GetOrganizationsWithPrebuildStatusRow
|
||||
for rows.Next() {
|
||||
var i GetOrganizationsWithPrebuildStatusRow
|
||||
if err := rows.Scan(
|
||||
&i.OrganizationID,
|
||||
&i.OrganizationName,
|
||||
&i.HasPrebuildUser,
|
||||
&i.PrebuildsGroupID,
|
||||
&i.HasPrebuildUserInGroup,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getPrebuildMetrics = `-- name: GetPrebuildMetrics :many
|
||||
SELECT
|
||||
t.name as template_name,
|
||||
|
||||
Reference in New Issue
Block a user