opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: drop N+1 db query on template ACL available (#25465)

Browse Source 
Fixes
[PLAT-149](https://linear.app/codercom/issue/PLAT-149/template-permissions-search-is-extremely-slow-with-many-groups).

`/acl/available` ran a db query per group. A deployment with >5,000
groups made this route extremely slow.
This commit is contained in:
Steven Masley
2026-05-20 22:40:50 +00:00
committed by GitHub
parent 889add734e
commit 9b6eadab77
12 changed files with 315 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/dbauthz/dbauthz.go
+9
View File
@@ -3476,6 +3476,15 @@ func (q *querier) GetGroupMembersCountByGroupID(ctx context.Context, arg databas
return memberCount, nil
}
func (q *querier) GetGroupMembersCountByGroupIDs(ctx context.Context, arg database.GetGroupMembersCountByGroupIDsParams) ([]database.GetGroupMembersCountByGroupIDsRow, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceGroup); err != nil {
// Ideally we would check read access on each group ID, but that would be N queries.
// So this function is really only usable by admins.
return nil, err
}
return q.db.GetGroupMembersCountByGroupIDs(ctx, arg)
}
func (q *querier) GetGroups(ctx context.Context, arg database.GetGroupsParams) ([]database.GetGroupsRow, error) {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err == nil {
// Optimize this query for system users as it is used in telemetry.
coderd/database/dbauthz/dbauthz_test.go
+12
View File
@@ -1820,6 +1820,18 @@ func (s *MethodTestSuite) TestGroup() {
check.Args(arg).Asserts(g, policy.ActionRead)
}))
s.Run("GetGroupMembersCountByGroupIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
g1 := testutil.Fake(s.T(), faker, database.Group{})
g2 := testutil.Fake(s.T(), faker, database.Group{})
arg := database.GetGroupMembersCountByGroupIDsParams{GroupIds: []uuid.UUID{g1.ID, g2.ID}, IncludeSystem: false}
rows := []database.GetGroupMembersCountByGroupIDsRow{
{GroupID: g1.ID, MemberCount: 1},
{GroupID: g2.ID, MemberCount: 2},
}
dbm.EXPECT().GetGroupMembersCountByGroupIDs(gomock.Any(), arg).Return(rows, nil).AnyTimes()
check.Args(arg).Asserts(rbac.ResourceGroup, policy.ActionRead).Returns(rows)
}))
s.Run("GetGroupMembers", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
dbm.EXPECT().GetGroupMembers(gomock.Any(), false).Return([]database.GroupMember{}, nil).AnyTimes()
check.Args(false).Asserts(rbac.ResourceSystem, policy.ActionRead)
coderd/database/dbmetrics/querymetrics.go
+8
View File
@@ -1945,6 +1945,14 @@ func (m queryMetricsStore) GetGroupMembersCountByGroupID(ctx context.Context, ar
return r0, r1
}
func (m queryMetricsStore) GetGroupMembersCountByGroupIDs(ctx context.Context, arg database.GetGroupMembersCountByGroupIDsParams) ([]database.GetGroupMembersCountByGroupIDsRow, error) {
start := time.Now()
r0, r1 := m.s.GetGroupMembersCountByGroupIDs(ctx, arg)
m.queryLatencies.WithLabelValues("GetGroupMembersCountByGroupIDs").Observe(time.Since(start).Seconds())
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetGroupMembersCountByGroupIDs").Inc()
return r0, r1
}
func (m queryMetricsStore) GetGroups(ctx context.Context, arg database.GetGroupsParams) ([]database.GetGroupsRow, error) {
start := time.Now()
r0, r1 := m.s.GetGroups(ctx, arg)
coderd/database/dbmock/dbmock.go
+15
View File
@@ -3616,6 +3616,21 @@ func (mr *MockStoreMockRecorder) GetGroupMembersCountByGroupID(ctx, arg any) *go
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGroupMembersCountByGroupID", reflect.TypeOf((*MockStore)(nil).GetGroupMembersCountByGroupID), ctx, arg)
}
// GetGroupMembersCountByGroupIDs mocks base method.
func (m *MockStore) GetGroupMembersCountByGroupIDs(ctx context.Context, arg database.GetGroupMembersCountByGroupIDsParams) ([]database.GetGroupMembersCountByGroupIDsRow, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetGroupMembersCountByGroupIDs", ctx, arg)
ret0, _ := ret[0].([]database.GetGroupMembersCountByGroupIDsRow)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetGroupMembersCountByGroupIDs indicates an expected call of GetGroupMembersCountByGroupIDs.
func (mr *MockStoreMockRecorder) GetGroupMembersCountByGroupIDs(ctx, arg any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGroupMembersCountByGroupIDs", reflect.TypeOf((*MockStore)(nil).GetGroupMembersCountByGroupIDs), ctx, arg)
}
// GetGroups mocks base method.
func (m *MockStore) GetGroups(ctx context.Context, arg database.GetGroupsParams) ([]database.GetGroupsRow, error) {
m.ctrl.T.Helper()
coderd/database/querier.go
+6
View File
@@ -473,6 +473,12 @@ type sqlcQuerier interface {
// count even if the caller does not have read access to ResourceGroupMember.
// They only need ResourceGroup read access.
GetGroupMembersCountByGroupID(ctx context.Context, arg GetGroupMembersCountByGroupIDParams) (int64, error)
// Returns the total member count for each of the given group IDs in a
// single query. Used to avoid N+1 lookups when listing many groups. Like
// GetGroupMembersCountByGroupID, the count is returned even when the
// caller does not have read access to individual group members.
GetGroupMembersCountByGroupIDs(ctx context.Context, arg GetGroupMembersCountByGroupIDsParams) ([]GetGroupMembersCountByGroupIDsRow, error)
// A limit of 0 means "no limit".
GetGroups(ctx context.Context, arg GetGroupsParams) ([]GetGroupsRow, error)
GetHealthSettings(ctx context.Context) (string, error)
GetInboxNotificationByID(ctx context.Context, id uuid.UUID) (InboxNotification, error)
coderd/database/queries.sql.go
+63
View File
@@ -12689,6 +12689,56 @@ func (q *sqlQuerier) GetGroupMembersCountByGroupID(ctx context.Context, arg GetG
return count, err
}
const getGroupMembersCountByGroupIDs = `-- name: GetGroupMembersCountByGroupIDs :many
SELECT
group_id,
COUNT(*) AS member_count
FROM group_members_expanded
WHERE group_id = ANY($1 :: uuid[])
AND CASE
WHEN $2::bool THEN TRUE
ELSE user_is_system = false
END
GROUP BY group_id
`
type GetGroupMembersCountByGroupIDsParams struct {
GroupIds []uuid.UUID `db:"group_ids" json:"group_ids"`
IncludeSystem bool `db:"include_system" json:"include_system"`
}
type GetGroupMembersCountByGroupIDsRow struct {
GroupID uuid.UUID `db:"group_id" json:"group_id"`
MemberCount int64 `db:"member_count" json:"member_count"`
}
// Returns the total member count for each of the given group IDs in a
// single query. Used to avoid N+1 lookups when listing many groups. Like
// GetGroupMembersCountByGroupID, the count is returned even when the
// caller does not have read access to individual group members.
func (q *sqlQuerier) GetGroupMembersCountByGroupIDs(ctx context.Context, arg GetGroupMembersCountByGroupIDsParams) ([]GetGroupMembersCountByGroupIDsRow, error) {
rows, err := q.db.QueryContext(ctx, getGroupMembersCountByGroupIDs, pq.Array(arg.GroupIds), arg.IncludeSystem)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetGroupMembersCountByGroupIDsRow
for rows.Next() {
var i GetGroupMembersCountByGroupIDsRow
if err := rows.Scan(&i.GroupID, &i.MemberCount); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const insertGroupMember = `-- name: InsertGroupMember :exec
INSERT INTO
group_members (user_id, group_id)
@@ -12906,6 +12956,14 @@ WHERE
groups.id = ANY($4)
ELSE true
END
-- Filter by group name or display name (substring, case-insensitive).
AND CASE WHEN $5 :: text != '' THEN (
groups.name ILIKE concat('%', $5, '%')
OR groups.display_name ILIKE concat('%', $5, '%')
)
ELSE true
END
LIMIT NULLIF($6 :: int, 0)
`
type GetGroupsParams struct {
@@ -12913,6 +12971,8 @@ type GetGroupsParams struct {
HasMemberID uuid.UUID `db:"has_member_id" json:"has_member_id"`
GroupNames []string `db:"group_names" json:"group_names"`
GroupIds []uuid.UUID `db:"group_ids" json:"group_ids"`
Search string `db:"search" json:"search"`
LimitOpt int32 `db:"limit_opt" json:"limit_opt"`
}
type GetGroupsRow struct {
@@ -12921,12 +12981,15 @@ type GetGroupsRow struct {
OrganizationDisplayName string `db:"organization_display_name" json:"organization_display_name"`
}
// A limit of 0 means "no limit".
func (q *sqlQuerier) GetGroups(ctx context.Context, arg GetGroupsParams) ([]GetGroupsRow, error) {
rows, err := q.db.QueryContext(ctx, getGroups,
arg.OrganizationID,
arg.HasMemberID,
pq.Array(arg.GroupNames),
pq.Array(arg.GroupIds),
arg.Search,
arg.LimitOpt,
)
if err != nil {
return nil, err
coderd/database/queries/groupmembers.sql
+16
View File
@@ -142,6 +142,22 @@ WHERE group_id = @group_id
user_is_system = false
END;
-- name: GetGroupMembersCountByGroupIDs :many
-- Returns the total member count for each of the given group IDs in a
-- single query. Used to avoid N+1 lookups when listing many groups. Like
-- GetGroupMembersCountByGroupID, the count is returned even when the
-- caller does not have read access to individual group members.
SELECT
group_id,
COUNT(*) AS member_count
FROM group_members_expanded
WHERE group_id = ANY(@group_ids :: uuid[])
AND CASE
WHEN @include_system::bool THEN TRUE
ELSE user_is_system = false
END
GROUP BY group_id;
-- InsertUserGroupsByID adds a user to all provided groups, if they exist.
-- name: InsertUserGroupsByID :many
WITH groups AS (
coderd/database/queries/groups.sql
+9
View File
@@ -78,6 +78,15 @@ WHERE
groups.id = ANY(@group_ids)
ELSE true
END
-- Filter by group name or display name (substring, case-insensitive).
AND CASE WHEN @search :: text != '' THEN (
groups.name ILIKE concat('%', @search, '%')
OR groups.display_name ILIKE concat('%', @search, '%')
)
ELSE true
END
-- A limit of 0 means "no limit".
LIMIT NULLIF(@limit_opt :: int, 0)
;
-- name: InsertGroup :one