fix(coderd/database): consider tag sets when calculating queue position (#16685)

Relates to https://github.com/coder/coder/issues/15843

## PR Contents

- Reimplementation of the `GetProvisionerJobsByIDsWithQueuePosition` SQL
query to **take into account** provisioner job tags and provisioner
daemon tags.
- Unit tests covering different **tag sets**, **job statuses**, and
**job ordering** scenarios.

## Notes

- The original row order is preserved by introducing the `ordinality`
field.
- Unnecessary rows are filtered as early as possible to ensure that
expensive joins operate on a smaller dataset.
- A "fake" join with `provisioner_jobs` is added at the end to ensure
`sqlc.embed` compiles successfully.
- **Backward compatibility is preserved**—only the SQL query has been
updated, while the Go code remains unchanged.

coderd/database/querier_test.go

@@ -1257,6 +1257,15 @@ func TestQueuePosition(t *testing.T) {
 		time.Sleep(time.Millisecond)
 	}
 
+	// Create default provisioner daemon:
+	dbgen.ProvisionerDaemon(t, db, database.ProvisionerDaemon{
+		Name:         "default_provisioner",
+		Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho},
+		// Ensure the `tags` field is NOT NULL for the default provisioner;
+		// otherwise, it won't be able to pick up any jobs.
+		Tags: database.StringMap{},
+	})
+
 	queued, err := db.GetProvisionerJobsByIDsWithQueuePosition(ctx, jobIDs)
 	require.NoError(t, err)
 	require.Len(t, queued, jobCount)
@@ -2159,6 +2168,307 @@ func TestExpectOne(t *testing.T) {
 
 func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 	t.Parallel()
+
+	now := dbtime.Now()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	testCases := []struct {
+		name           string
+		jobTags        []database.StringMap
+		daemonTags     []database.StringMap
+		queueSizes     []int64
+		queuePositions []int64
+		// GetProvisionerJobsByIDsWithQueuePosition takes jobIDs as a parameter.
+		// If skipJobIDs is empty, all jobs are passed to the function; otherwise, the specified jobs are skipped.
+		// NOTE: Skipping job IDs means they will be excluded from the result,
+		// but this should not affect the queue position or queue size of other jobs.
+		skipJobIDs map[int]struct{}
+	}{
+		// Baseline test case
+		{
+			name: "test-case-1",
+			jobTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+			},
+			queueSizes:     []int64{2, 2, 0},
+			queuePositions: []int64{1, 1, 0},
+		},
+		// Includes an additional provisioner
+		{
+			name: "test-case-2",
+			jobTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{3, 3, 3},
+			queuePositions: []int64{1, 1, 3},
+		},
+		// Skips job at index 0
+		{
+			name: "test-case-3",
+			jobTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{3, 3},
+			queuePositions: []int64{1, 3},
+			skipJobIDs: map[int]struct{}{
+				0: {},
+			},
+		},
+		// Skips job at index 1
+		{
+			name: "test-case-4",
+			jobTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{3, 3},
+			queuePositions: []int64{1, 3},
+			skipJobIDs: map[int]struct{}{
+				1: {},
+			},
+		},
+		// Skips job at index 2
+		{
+			name: "test-case-5",
+			jobTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{3, 3},
+			queuePositions: []int64{1, 1},
+			skipJobIDs: map[int]struct{}{
+				2: {},
+			},
+		},
+		// Skips jobs at indexes 0 and 2
+		{
+			name: "test-case-6",
+			jobTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{3},
+			queuePositions: []int64{1},
+			skipJobIDs: map[int]struct{}{
+				0: {},
+				2: {},
+			},
+		},
+		// Includes two additional jobs that any provisioner can execute.
+		{
+			name: "test-case-7",
+			jobTags: []database.StringMap{
+				{},
+				{},
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{5, 5, 5, 5, 5},
+			queuePositions: []int64{1, 2, 3, 3, 5},
+		},
+		// Includes two additional jobs that any provisioner can execute, but they are intentionally skipped.
+		{
+			name: "test-case-8",
+			jobTags: []database.StringMap{
+				{},
+				{},
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "c": "3"},
+			},
+			daemonTags: []database.StringMap{
+				{"a": "1", "b": "2"},
+				{"a": "1"},
+				{"a": "1", "b": "2", "c": "3"},
+			},
+			queueSizes:     []int64{5, 5, 5},
+			queuePositions: []int64{3, 3, 5},
+			skipJobIDs: map[int]struct{}{
+				0: {},
+				1: {},
+			},
+		},
+		// N jobs (1 job with 0 tags) & 0 provisioners exist
+		{
+			name: "test-case-9",
+			jobTags: []database.StringMap{
+				{},
+				{"a": "1"},
+				{"b": "2"},
+			},
+			daemonTags:     []database.StringMap{},
+			queueSizes:     []int64{0, 0, 0},
+			queuePositions: []int64{0, 0, 0},
+		},
+		// N jobs (1 job with 0 tags) & N provisioners
+		{
+			name: "test-case-10",
+			jobTags: []database.StringMap{
+				{},
+				{"a": "1"},
+				{"b": "2"},
+			},
+			daemonTags: []database.StringMap{
+				{},
+				{"a": "1"},
+				{"b": "2"},
+			},
+			queueSizes:     []int64{2, 2, 2},
+			queuePositions: []int64{1, 2, 2},
+		},
+		// (N + 1) jobs (1 job with 0 tags) & N provisioners
+		// 1 job not matching any provisioner (first in the list)
+		{
+			name: "test-case-11",
+			jobTags: []database.StringMap{
+				{"c": "3"},
+				{},
+				{"a": "1"},
+				{"b": "2"},
+			},
+			daemonTags: []database.StringMap{
+				{},
+				{"a": "1"},
+				{"b": "2"},
+			},
+			queueSizes:     []int64{0, 2, 2, 2},
+			queuePositions: []int64{0, 1, 2, 2},
+		},
+		// 0 jobs & 0 provisioners
+		{
+			name:           "test-case-12",
+			jobTags:        []database.StringMap{},
+			daemonTags:     []database.StringMap{},
+			queueSizes:     nil, // TODO(yevhenii): should it be empty array instead?
+			queuePositions: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		tc := tc // Capture loop variable to avoid data races
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			db, _ := dbtestutil.NewDB(t)
+
+			// Create provisioner jobs based on provided tags:
+			allJobs := make([]database.ProvisionerJob, len(tc.jobTags))
+			for idx, tags := range tc.jobTags {
+				// Make sure jobs are stored in correct order, first job should have the earliest createdAt timestamp.
+				// Example for 3 jobs:
+				// job_1 createdAt: now - 3 minutes
+				// job_2 createdAt: now - 2 minutes
+				// job_3 createdAt: now - 1 minute
+				timeOffsetInMinutes := len(tc.jobTags) - idx
+				timeOffset := time.Duration(timeOffsetInMinutes) * time.Minute
+				createdAt := now.Add(-timeOffset)
+
+				allJobs[idx] = dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+					CreatedAt: createdAt,
+					Tags:      tags,
+				})
+			}
+
+			// Create provisioner daemons based on provided tags:
+			for idx, tags := range tc.daemonTags {
+				dbgen.ProvisionerDaemon(t, db, database.ProvisionerDaemon{
+					Name:         fmt.Sprintf("prov_%v", idx),
+					Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho},
+					Tags:         tags,
+				})
+			}
+
+			// Assert invariant: the jobs are in pending status
+			for idx, job := range allJobs {
+				require.Equal(t, database.ProvisionerJobStatusPending, job.JobStatus, "expected job %d to have status %s", idx, database.ProvisionerJobStatusPending)
+			}
+
+			filteredJobs := make([]database.ProvisionerJob, 0)
+			filteredJobIDs := make([]uuid.UUID, 0)
+			for idx, job := range allJobs {
+				if _, skip := tc.skipJobIDs[idx]; skip {
+					continue
+				}
+
+				filteredJobs = append(filteredJobs, job)
+				filteredJobIDs = append(filteredJobIDs, job.ID)
+			}
+
+			// When: we fetch the jobs by their IDs
+			actualJobs, err := db.GetProvisionerJobsByIDsWithQueuePosition(ctx, filteredJobIDs)
+			require.NoError(t, err)
+			require.Len(t, actualJobs, len(filteredJobs), "should return all unskipped jobs")
+
+			// Then: the jobs should be returned in the correct order (sorted by createdAt)
+			sort.Slice(filteredJobs, func(i, j int) bool {
+				return filteredJobs[i].CreatedAt.Before(filteredJobs[j].CreatedAt)
+			})
+			for idx, job := range actualJobs {
+				assert.EqualValues(t, filteredJobs[idx], job.ProvisionerJob)
+			}
+
+			// Then: the queue size should be set correctly
+			var queueSizes []int64
+			for _, job := range actualJobs {
+				queueSizes = append(queueSizes, job.QueueSize)
+			}
+			assert.EqualValues(t, tc.queueSizes, queueSizes, "expected queue positions to be set correctly")
+
+			// Then: the queue position should be set correctly:
+			var queuePositions []int64
+			for _, job := range actualJobs {
+				queuePositions = append(queuePositions, job.QueuePosition)
+			}
+			assert.EqualValues(t, tc.queuePositions, queuePositions, "expected queue positions to be set correctly")
+		})
+	}
+}
+
+func TestGetProvisionerJobsByIDsWithQueuePosition_MixedStatuses(t *testing.T) {
+	t.Parallel()
 	if !dbtestutil.WillUsePostgres() {
 		t.SkipNow()
 	}
@@ -2167,7 +2477,7 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 	now := dbtime.Now()
 	ctx := testutil.Context(t, testutil.WaitShort)
 
-	// Given the following provisioner jobs:
+	// Create the following provisioner jobs:
 	allJobs := []database.ProvisionerJob{
 		// Pending. This will be the last in the queue because
 		// it was created most recently.
@@ -2177,6 +2487,9 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{},
 			CompletedAt: sql.NullTime{},
 			Error:       sql.NullString{},
+			// Ensure the `tags` field is NOT NULL for both provisioner jobs and provisioner daemons;
+			// otherwise, provisioner daemons won't be able to pick up any jobs.
+			Tags: database.StringMap{},
 		}),
 
 		// Another pending. This will come first in the queue
@@ -2187,6 +2500,7 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{},
 			CompletedAt: sql.NullTime{},
 			Error:       sql.NullString{},
+			Tags:        database.StringMap{},
 		}),
 
 		// Running
@@ -2196,6 +2510,7 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{},
 			CompletedAt: sql.NullTime{},
 			Error:       sql.NullString{},
+			Tags:        database.StringMap{},
 		}),
 
 		// Succeeded
@@ -2205,6 +2520,7 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{},
 			CompletedAt: sql.NullTime{Valid: true, Time: now},
 			Error:       sql.NullString{},
+			Tags:        database.StringMap{},
 		}),
 
 		// Canceling
@@ -2214,6 +2530,7 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{Valid: true, Time: now},
 			CompletedAt: sql.NullTime{},
 			Error:       sql.NullString{},
+			Tags:        database.StringMap{},
 		}),
 
 		// Canceled
@@ -2223,6 +2540,7 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{Valid: true, Time: now},
 			CompletedAt: sql.NullTime{Valid: true, Time: now},
 			Error:       sql.NullString{},
+			Tags:        database.StringMap{},
 		}),
 
 		// Failed
@@ -2232,9 +2550,17 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 			CanceledAt:  sql.NullTime{},
 			CompletedAt: sql.NullTime{},
 			Error:       sql.NullString{String: "failed", Valid: true},
+			Tags:        database.StringMap{},
 		}),
 	}
 
+	// Create default provisioner daemon:
+	dbgen.ProvisionerDaemon(t, db, database.ProvisionerDaemon{
+		Name:         "default_provisioner",
+		Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho},
+		Tags:         database.StringMap{},
+	})
+
 	// Assert invariant: the jobs are in the expected order
 	require.Len(t, allJobs, 7, "expected 7 jobs")
 	for idx, status := range []database.ProvisionerJobStatus{
@@ -2259,22 +2585,123 @@ func TestGetProvisionerJobsByIDsWithQueuePosition(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, actualJobs, len(allJobs), "should return all jobs")
 
-	// Then: the jobs should be returned in the correct order (by IDs in the input slice)
+	// Then: the jobs should be returned in the correct order (sorted by createdAt)
+	sort.Slice(allJobs, func(i, j int) bool {
+		return allJobs[i].CreatedAt.Before(allJobs[j].CreatedAt)
+	})
 	for idx, job := range actualJobs {
 		assert.EqualValues(t, allJobs[idx], job.ProvisionerJob)
 	}
 
 	// Then: the queue size should be set correctly
+	var queueSizes []int64
 	for _, job := range actualJobs {
-		assert.EqualValues(t, job.QueueSize, 2, "should have queue size 2")
+		queueSizes = append(queueSizes, job.QueueSize)
 	}
+	assert.EqualValues(t, []int64{0, 0, 0, 0, 0, 2, 2}, queueSizes, "expected queue positions to be set correctly")
 
 	// Then: the queue position should be set correctly:
 	var queuePositions []int64
 	for _, job := range actualJobs {
 		queuePositions = append(queuePositions, job.QueuePosition)
 	}
-	assert.EqualValues(t, []int64{2, 1, 0, 0, 0, 0, 0}, queuePositions, "expected queue positions to be set correctly")
+	assert.EqualValues(t, []int64{0, 0, 0, 0, 0, 1, 2}, queuePositions, "expected queue positions to be set correctly")
+}
+
+func TestGetProvisionerJobsByIDsWithQueuePosition_OrderValidation(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	now := dbtime.Now()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	// Create the following provisioner jobs:
+	allJobs := []database.ProvisionerJob{
+		dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+			CreatedAt: now.Add(-4 * time.Minute),
+			// Ensure the `tags` field is NOT NULL for both provisioner jobs and provisioner daemons;
+			// otherwise, provisioner daemons won't be able to pick up any jobs.
+			Tags: database.StringMap{},
+		}),
+
+		dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+			CreatedAt: now.Add(-5 * time.Minute),
+			Tags:      database.StringMap{},
+		}),
+
+		dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+			CreatedAt: now.Add(-6 * time.Minute),
+			Tags:      database.StringMap{},
+		}),
+
+		dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+			CreatedAt: now.Add(-3 * time.Minute),
+			Tags:      database.StringMap{},
+		}),
+
+		dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+			CreatedAt: now.Add(-2 * time.Minute),
+			Tags:      database.StringMap{},
+		}),
+
+		dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+			CreatedAt: now.Add(-1 * time.Minute),
+			Tags:      database.StringMap{},
+		}),
+	}
+
+	// Create default provisioner daemon:
+	dbgen.ProvisionerDaemon(t, db, database.ProvisionerDaemon{
+		Name:         "default_provisioner",
+		Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho},
+		Tags:         database.StringMap{},
+	})
+
+	// Assert invariant: the jobs are in the expected order
+	require.Len(t, allJobs, 6, "expected 7 jobs")
+	for idx, status := range []database.ProvisionerJobStatus{
+		database.ProvisionerJobStatusPending,
+		database.ProvisionerJobStatusPending,
+		database.ProvisionerJobStatusPending,
+		database.ProvisionerJobStatusPending,
+		database.ProvisionerJobStatusPending,
+		database.ProvisionerJobStatusPending,
+	} {
+		require.Equal(t, status, allJobs[idx].JobStatus, "expected job %d to have status %s", idx, status)
+	}
+
+	var jobIDs []uuid.UUID
+	for _, job := range allJobs {
+		jobIDs = append(jobIDs, job.ID)
+	}
+
+	// When: we fetch the jobs by their IDs
+	actualJobs, err := db.GetProvisionerJobsByIDsWithQueuePosition(ctx, jobIDs)
+	require.NoError(t, err)
+	require.Len(t, actualJobs, len(allJobs), "should return all jobs")
+
+	// Then: the jobs should be returned in the correct order (sorted by createdAt)
+	sort.Slice(allJobs, func(i, j int) bool {
+		return allJobs[i].CreatedAt.Before(allJobs[j].CreatedAt)
+	})
+	for idx, job := range actualJobs {
+		assert.EqualValues(t, allJobs[idx], job.ProvisionerJob)
+		assert.EqualValues(t, allJobs[idx].CreatedAt, job.ProvisionerJob.CreatedAt)
+	}
+
+	// Then: the queue size should be set correctly
+	var queueSizes []int64
+	for _, job := range actualJobs {
+		queueSizes = append(queueSizes, job.QueueSize)
+	}
+	assert.EqualValues(t, []int64{6, 6, 6, 6, 6, 6}, queueSizes, "expected queue positions to be set correctly")
+
+	// Then: the queue position should be set correctly:
+	var queuePositions []int64
+	for _, job := range actualJobs {
+		queuePositions = append(queuePositions, job.QueuePosition)
+	}
+	assert.EqualValues(t, []int64{1, 2, 3, 4, 5, 6}, queuePositions, "expected queue positions to be set correctly")
 }
 
 func TestGroupRemovalTrigger(t *testing.T) {