fix: deflake TestWorkspaceTagsTerraform with context-aware build waits (#26315)

`TestWorkspaceTagsTerraform` runs a real terraform provisioner but
waited on builds with `coderdtest` helpers whose deadlines are sized for
the echo provisioner used by most tests, which replays canned responses
and completes in well under a second. On Windows runners, where
terraform providers are not cached and every `terraform init` downloads
from the registry, template imports exceeded the 25s budget in
`AwaitTemplateVersionJobCompleted` and workspace builds exceeded the 10s
context in `AwaitWorkspaceBuildJobCompleted`, even though the test
intends a 120s budget.

Add `AwaitTemplateVersionJobCompletedWithTimeout` and
`AwaitWorkspaceBuildJobCompletedWithTimeout`, which take a
caller-provided wait bound, and use them in the test with
`2*testutil.WaitSuperLong` (120s). Also fix
`AwaitWorkspaceBuildJobCompleted` creating a `WaitShort` (10s) context
while polling for `WaitMedium` (15s), which guaranteed `context deadline
exceeded` errors for the final five seconds of polling.

`TestWorkspaceTemplateParamsChange` has the same shape (real terraform
provisioner, 120s test context, plain await helpers) and the same latent
bug, so it gets the same fix.

Closes https://github.com/coder/internal/issues/1470 (Linear: PLAT-176)

<details>
<summary>Root cause analysis</summary>

Two CI failures, same mechanism:

- 2026-04-16 (run 24493089585, windows-2022):
`overrides_with_dynamic_option_from_var/dynamic` failed at
`coderdtest.AwaitTemplateVersionJobCompleted` with `Condition never
satisfied ... make sure you set IncludeProvisionerDaemon!`. The template
import job (real terraform init/plan, with network provider download)
did not complete within `WaitLong` (25s).
- 2026-05-27 (run 26492817796, windows-2022): `tag_param/dynamic` failed
at `coderdtest.AwaitWorkspaceBuildJobCompleted` with `failed to get
workspace build ...: context deadline exceeded`. The helper's internal
context was `WaitShort` (10s) while its polling window was `WaitMedium`
(15s), so after 10s every poll could only fail. The logged `terraform
apply: exit status 1` and the `TempDir RemoveAll ... Access is denied`
cleanup error are consequences of test teardown canceling the in-flight
job while the provider exe was still file-locked.

The test declares a 120s budget (`2*testutil.WaitSuperLong`, commented
"This can take a while"), but the await helpers ignored it and applied
their own 10-25s budgets. `testutil.CacheTFProviders` is a no-op on
Windows, so real builds are much slower there.

This change raises the ceiling for the tests rather than making
terraform faster; both observed failure signatures are eliminated. The
default helper budgets are unchanged for the ~880 existing call sites.
One small behavior change: `AwaitTemplateVersionJobCompleted` previously
marked the test failed on any transient poll error via `assert.NoError`;
it now logs and keeps polling, matching the workspace build helper, and
still fails on timeout.

`TestWorkspaceTemplateParamsChange` is the sibling real-terraform test
in the same file (also covered by the original provider-caching work in
#20603). It runs three sequential real builds with the plain await
helpers under a 120s context, so it is exposed to the same Windows
slowness even though it has not produced its own issue yet. Its context
is raised to `6*testutil.WaitSuperLong` to outlast three sequential
await budgets.

API note: a context-taking variant was considered first, but a
`time.Duration` parameter avoids an implicit "context must have a
deadline" contract and matches how the existing helpers manage their own
wait budgets.

</details>

---

🤖 This PR was generated by Coder Agents on behalf of @jscottmiller.
This commit is contained in:
J. Scott Miller
2026-06-29 09:56:42 -05:00
committed by GitHub
parent 5f42bbcdbf
commit 1dea00dd04
2 changed files with 56 additions and 20 deletions
+38 -11
View File
@@ -1223,35 +1223,59 @@ func AwaitTemplateVersionJobRunning(t testing.TB, client *codersdk.Client, versi
}
// AwaitTemplateVersionJobCompleted waits for the build to be completed. This may result
// from cancelation, an error, or from completing successfully.
// from cancelation, an error, or from completing successfully. The wait is bounded by
// testutil.WaitLong; use AwaitTemplateVersionJobCompletedWithTimeout to wait longer.
func AwaitTemplateVersionJobCompleted(t testing.TB, client *codersdk.Client, version uuid.UUID) codersdk.TemplateVersion {
t.Helper()
return AwaitTemplateVersionJobCompletedWithTimeout(t, client, version, testutil.WaitLong)
}
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
// AwaitTemplateVersionJobCompletedWithTimeout waits up to timeout for the template
// version build job to complete, polling at testutil.IntervalFast. Transient API errors
// are logged and retried. Fails the test if the job does not complete in time.
func AwaitTemplateVersionJobCompletedWithTimeout(t testing.TB, client *codersdk.Client, version uuid.UUID, timeout time.Duration) codersdk.TemplateVersion {
t.Helper()
ctx := testutil.Context(t, timeout)
t.Logf("waiting for template version %s build job to complete", version)
var templateVersion codersdk.TemplateVersion
require.Eventually(t, func() bool {
completed := testutil.Eventually(ctx, t, func(ctx context.Context) bool {
var err error
templateVersion, err = client.TemplateVersion(ctx, version)
if err != nil {
t.Logf("failed to get template version %s: %v", version, err)
return false
}
t.Logf("template version job status: %s", templateVersion.Job.Status)
return assert.NoError(t, err) && templateVersion.Job.CompletedAt != nil
}, testutil.WaitLong, testutil.IntervalFast, "make sure you set `IncludeProvisionerDaemon`!")
return templateVersion.Job.CompletedAt != nil
}, testutil.IntervalFast, "make sure you set `IncludeProvisionerDaemon`!")
if !completed {
t.FailNow()
}
t.Logf("template version %s job has completed", version)
return templateVersion
}
// AwaitWorkspaceBuildJobCompleted waits for a workspace provision job to reach completed status.
// AwaitWorkspaceBuildJobCompleted waits for a workspace provision job to reach completed
// status. The wait is bounded by testutil.WaitMedium; use
// AwaitWorkspaceBuildJobCompletedWithTimeout to wait longer.
func AwaitWorkspaceBuildJobCompleted(t testing.TB, client *codersdk.Client, build uuid.UUID) codersdk.WorkspaceBuild {
t.Helper()
return AwaitWorkspaceBuildJobCompletedWithTimeout(t, client, build, testutil.WaitMedium)
}
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancel()
// AwaitWorkspaceBuildJobCompletedWithTimeout waits up to timeout for a workspace
// provision job to reach completed status, polling at testutil.IntervalFast. Transient
// API errors are logged and retried. Fails the test if the job does not complete in time.
func AwaitWorkspaceBuildJobCompletedWithTimeout(t testing.TB, client *codersdk.Client, build uuid.UUID, timeout time.Duration) codersdk.WorkspaceBuild {
t.Helper()
ctx := testutil.Context(t, timeout)
t.Logf("waiting for workspace build job %s", build)
var workspaceBuild codersdk.WorkspaceBuild
require.Eventually(t, func() bool {
completed := testutil.Eventually(ctx, t, func(ctx context.Context) bool {
var err error
workspaceBuild, err = client.WorkspaceBuild(ctx, build)
if err != nil {
@@ -1263,7 +1287,10 @@ func AwaitWorkspaceBuildJobCompleted(t testing.TB, client *codersdk.Client, buil
return false
}
return true
}, testutil.WaitMedium, testutil.IntervalFast)
}, testutil.IntervalFast, "workspace build %s did not complete", build)
if !completed {
t.FailNow()
}
t.Logf("got workspace build job %s (status: %s)", build, workspaceBuild.Job.Status)
return workspaceBuild
}