From bd693ad4aeb678e9fd499493e88ce98e3aa19476 Mon Sep 17 00:00:00 2001 From: Bobby Ho Date: Mon, 10 Aug 2026 12:29:05 -0700 Subject: [PATCH] fix: raise test-go-pg timeout budget, lower macOS parallelism (#27997) `test-go-pg (macos-latest)` has recurred three times on [coder/internal#1365](https://github.com/coder/internal/issues/1365), most recently as the `cli` package hitting its own `go test -timeout 20m` after several minutes of runner-side scheduling delay, which marks hundreds of in-flight subtests as failed instead of surfacing the actual cause. Raise `GOTEST_FLAGS -timeout` from `20m` to `25m` and the matching `timeout-minutes` on every job that shares it (`test-go-pg`, `test-go-pg-17`, `test-go-race-pg` in `ci.yaml`; `flake_go` in `flake-go.yaml`; `test-go-pg` in `nightly-gauntlet.yaml`) from `25` to `30`, preserving the 5-minute margin that lets `go test`'s own timeout fire before GitHub cancels the job and produce a goroutine dump for a genuine hang. Also lower macOS `test-parallelism-tests` from `16` to `8`, matching Linux, to test whether the 2x core oversubscription is contributing scheduler contention. Windows keeps `16`; nothing in the current evidence implicates it. This is a mitigation, not a confirmed root cause; the runner scheduling delay itself is still unexplained. Refs coder/internal#1365 ## AI disclosure Per [AI Contribution guidelines](https://coder.com/docs/about/contributing/AI_CONTRIBUTING): this PR was primarily authored by Claude Code, directed by @BobbyHo, based on triage of the linked ticket's CI-flake-bot comments and a review of the current `Makefile`/`ci.yaml`/`flake-go.yaml`/`nightly-gauntlet.yaml` timeout configuration. **Verification:** - `make lint/actions/actionlint` and `make lint/actions/zizmor` both pass clean on the three edited workflow files. - `make pre-commit` passes (includes `make gen`, `make fmt`, `make lint`, and a full local build). - All three edited workflow YAMLs parse (`ruby -ryaml`) and `make -n test TEST_PACKAGES="./cli/..."` confirms `GOTEST_FLAGS` resolves to `-timeout 25m` as intended. - This is a config-only change (no application code touched); the intended effect can only be confirmed by watching real `test-go-pg` (macOS) runs on this PR and on `main` after merge, since the underlying runner-stall failure is intermittent. --- .github/workflows/ci.yaml | 19 ++++++++++++------- .github/workflows/flake-go.yaml | 4 ++-- .github/workflows/nightly-gauntlet.yaml | 9 ++++++--- Makefile | 4 ++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6681a7b1d0..f69937ffae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -527,7 +527,7 @@ jobs: # `make test` to ensure we receive a trace of running goroutines. # Setting this to the timeout +5m should work quite well even if # some of the preceding steps are slow. - timeout-minutes: 25 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -662,11 +662,12 @@ jobs: uses: ./.github/actions/test-go-pg with: postgres-version: "13" - # Our macOS runners have 8 cores. - # Even though this parallelism seems high, we've observed relatively low flakiness in the past. - # See https://github.com/coder/coder/pull/21091#discussion_r2609891540. + # Our macOS runners have 8 cores. test-parallelism-tests matches + # Linux's rather than the previous 16, to rule out scheduler + # contention as a contributor to the macOS test-go-pg timeouts + # tracked in coder/internal#1365. test-parallelism-packages: "8" - test-parallelism-tests: "16" + test-parallelism-tests: "8" # By default, run tests with cache for improved speed (possibly at the expense of correctness). # On main, run tests without cache for the inverse. test-count: ${{ github.ref == 'refs/heads/main' && '1' || '' }} @@ -737,7 +738,7 @@ jobs: # `make test` to ensure we receive a trace of running goroutines. # Setting this to the timeout +5m should work quite well even if # some of the preceding steps are slow. - timeout-minutes: 25 + timeout-minutes: 30 steps: - name: Harden Runner uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 @@ -808,7 +809,11 @@ jobs: runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} needs: changes if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main' - timeout-minutes: 25 + # This timeout must be greater than the timeout set by `go test` in + # `make test-race` to ensure we receive a trace of running goroutines. + # Setting this to the timeout +5m should work quite well even if + # some of the preceding steps are slow. + timeout-minutes: 30 steps: - name: Harden Runner uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 diff --git a/.github/workflows/flake-go.yaml b/.github/workflows/flake-go.yaml index e17f84885a..41b2cefa58 100644 --- a/.github/workflows/flake-go.yaml +++ b/.github/workflows/flake-go.yaml @@ -25,9 +25,9 @@ jobs: name: Flake Check runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-4' || 'ubuntu-latest' }} # This timeout must be greater than the Go test timeout set in `make test` - # (-timeout 20m) so we receive a goroutine trace before the runner kills + # (-timeout 25m) so we receive a goroutine trace before the runner kills # the job. Mirrors the test-go-pg job in ci.yaml. - timeout-minutes: 25 + timeout-minutes: 30 steps: - name: Harden Runner uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 diff --git a/.github/workflows/nightly-gauntlet.yaml b/.github/workflows/nightly-gauntlet.yaml index 7c623a7e62..55a8774581 100644 --- a/.github/workflows/nightly-gauntlet.yaml +++ b/.github/workflows/nightly-gauntlet.yaml @@ -19,7 +19,7 @@ jobs: # `make test` to ensure we receive a trace of running goroutines. # Setting this to the timeout +5m should work quite well even if # some of the preceding steps are slow. - timeout-minutes: 25 + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -98,9 +98,12 @@ jobs: uses: ./.github/actions/test-go-pg with: postgres-version: "13" - # Our macOS runners have 8 cores. + # Our macOS runners have 8 cores. test-parallelism-tests matches + # Linux's rather than the previous 16, to rule out scheduler + # contention as a contributor to the macOS test-go-pg timeouts + # tracked in coder/internal#1365. test-parallelism-packages: "8" - test-parallelism-tests: "16" + test-parallelism-tests: "8" test-count: "1" embedded-pg-path: "/tmp/tmpfs/embedded-pg" embedded-pg-cache: ${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }} diff --git a/Makefile b/Makefile index e9683190d8..4070f89241 100644 --- a/Makefile +++ b/Makefile @@ -1519,10 +1519,10 @@ RACE_PARALLEL_TESTS := $(or $(TEST_NUM_PARALLEL_TESTS),4) # Use testsmallbatch tag to reduce wireguard memory allocation in tests # (from ~18GB to negligible). Recursively expanded so target-specific # overrides of TEST_PARALLEL_* take effect (e.g. test-race lowers -# parallelism). CI job timeout is 25m (see test-go-pg in ci.yaml), +# parallelism). CI job timeout is 30m (see test-go-pg in ci.yaml), # keep the Go timeout 5m shorter so tests produce goroutine dumps # instead of the CI runner killing the process with no output. -GOTEST_FLAGS = -tags=testsmallbatch -v -timeout 20m -p $(TEST_PARALLEL_PACKAGES) -parallel=$(TEST_PARALLEL_TESTS) +GOTEST_FLAGS = -tags=testsmallbatch -v -timeout 25m -p $(TEST_PARALLEL_PACKAGES) -parallel=$(TEST_PARALLEL_TESTS) # The most common use is to set TEST_COUNT=1 to avoid Go's test cache. ifdef TEST_COUNT