opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: run test-go-pg on macOS and Windows in regular CI (#17853)

Browse Source 
This PR starts running test-go-pg on macOS and Windows in regular CI.
Previously this suite was only run in the nightly gauntlet for 2
reasons:

- it was flaky
- it was slow (took 17 minutes)

We've since stabilized the flakiness by switching to depot runners,
using ram disks, optimizing the number of tests run in parallel, and
automatically re-running failing tests. We've also [brought
down](https://github.com/coder/coder/pull/17756) the time to run the
suite to 9 minutes. Additionally, this PR allows test-go-pg to use cache
from previous runs, which speeds it up further. The cache is only used
on PRs, `main` will still run tests without it.

This PR also:

- removes the nightly gauntlet since all tests now run in regular CI
- removes the `test-cli` job for the same reason
- removes the `setup-imdisk` action which is now fully replaced by
[coder/setup-ramdisk-action](https://github.com/coder/setup-ramdisk-action)
- makes 2 minor changes which could be separate PRs, but I rolled them
into this because they were helpful when iterating on it:
- replace the `if: always()` condition on the `gen` job with a `if: ${{
!cancelled() }}` to allow the job to be cancelled. Previously the job
would run to completion even if the entire workflow was cancelled. See
[the GitHub
docs](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#always)
for more details.
- disable the recently added `TestReinitializeAgent` since it does not
pass on Windows with Postgres. There's an open issue to fix it:
https://github.com/coder/internal/issues/642

This PR will:

- unblock https://github.com/coder/coder/issues/15109
- alleviate https://github.com/coder/internal/issues/647

I tested caching by temporarily enabling cache upload on this PR: here's
[a
run](https://github.com/coder/coder/actions/runs/15119046903/job/42496939341?pr=17853#step:13:1296)
showing cache being used.
This commit is contained in:
Hugo Dutka
2025-05-22 15:53:37 +02:00
committed by GitHub
parent f825477a5c
commit a0e229afec
6 changed files with 247 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yaml
+127 -69
View File
@@ -224,7 +224,7 @@ jobs:
gen:
timeout-minutes: 8
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
if: always()
if: ${{ !cancelled() }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
@@ -336,13 +336,16 @@ jobs:
# a separate repository to allow its use before actions/checkout.
- name: Setup RAM Disks
if: runner.os == 'Windows'
uses: coder/setup-ramdisk-action@81c5c441bda00c6c3d6bcee2e5a33ed4aadbbcc1
uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Setup Go Paths
uses: ./.github/actions/setup-go-paths
- name: Setup Go
uses: ./.github/actions/setup-go
with:
@@ -350,7 +353,6 @@ jobs:
# download the toolchain configured in go.mod, so we don't
# need to reinstall it. It's faster on Windows runners.
use-preinstalled-go: ${{ runner.os == 'Windows' }}
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
- name: Setup Terraform
uses: ./.github/actions/setup-tf
@@ -398,63 +400,10 @@ jobs:
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
# We don't run the full test-suite for Windows & MacOS, so we just run the CLI tests on every PR.
# We run the test suite in test-go-pg, including CLI.
test-cli:
runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'windows-latest-16-cores' || matrix.os }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
strategy:
matrix:
os:
- macos-latest
- windows-2022
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Setup Go
uses: ./.github/actions/setup-go
- name: Setup Terraform
uses: ./.github/actions/setup-tf
# Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
- name: Setup ImDisk
if: runner.os == 'Windows'
uses: ./.github/actions/setup-imdisk
- name: Test CLI
env:
TS_DEBUG_DISCO: "true"
LC_CTYPE: "en_US.UTF-8"
LC_ALL: "en_US.UTF-8"
TEST_RETRIES: 2
shell: bash
run: |
# By default Go will use the number of logical CPUs, which
# is a fine default.
PARALLEL_FLAG=""
make test-cli
- name: Upload test stats to Datadog
timeout-minutes: 1
continue-on-error: true
uses: ./.github/actions/upload-datadog
if: success() || failure()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
test-go-pg:
runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os }}
# make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
# when changing runner sizes
runs-on: ${{ matrix.os == 'ubuntu-latest' && github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || matrix.os && matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
needs: changes
if: needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' || github.ref == 'refs/heads/main'
# This timeout must be greater than the timeout set by `go test` in
@@ -466,48 +415,157 @@ jobs:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-2022
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
# macOS indexes all new files in the background. Our Postgres tests
# create and destroy thousands of databases on disk, and Spotlight
# tries to index all of them, seriously slowing down the tests.
- name: Disable Spotlight Indexing
if: runner.os == 'macOS'
run: |
sudo mdutil -a -i off
sudo mdutil -X /
sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
# Set up RAM disks to speed up the rest of the job. This action is in
# a separate repository to allow its use before actions/checkout.
- name: Setup RAM Disks
if: runner.os == 'Windows'
uses: coder/setup-ramdisk-action@a4b59caa8be2e88c348abeef042d7c1a33d8743e
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Setup Go Paths
id: go-paths
uses: ./.github/actions/setup-go-paths
- name: Download Go Build Cache
id: download-go-build-cache
uses: ./.github/actions/test-cache/download
with:
key-prefix: test-go-build-${{ runner.os }}-${{ runner.arch }}
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
- name: Setup Go
uses: ./.github/actions/setup-go
with:
# Runners have Go baked-in and Go will automatically
# download the toolchain configured in go.mod, so we don't
# need to reinstall it. It's faster on Windows runners.
use-preinstalled-go: ${{ runner.os == 'Windows' }}
# Cache is already downloaded above
use-cache: false
- name: Setup Terraform
uses: ./.github/actions/setup-tf
# Sets up the ImDisk toolkit for Windows and creates a RAM disk on drive R:.
- name: Setup ImDisk
if: runner.os == 'Windows'
uses: ./.github/actions/setup-imdisk
- name: Download Test Cache
id: download-cache
uses: ./.github/actions/test-cache/download
with:
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
- name: Normalize File and Directory Timestamps
shell: bash
# Normalize file modification timestamps so that go test can use the
# cache from the previous CI run. See https://github.com/golang/go/issues/58571
# for more details.
run: |
find . -type f ! -path ./.git/\*\* | mtimehash
find . -type d ! -path ./.git/\*\* -exec touch -t 200601010000 {} +
- name: Test with PostgreSQL Database
env:
POSTGRES_VERSION: "13"
TS_DEBUG_DISCO: "true"
LC_CTYPE: "en_US.UTF-8"
LC_ALL: "en_US.UTF-8"
TEST_RETRIES: 2
shell: bash
run: |
# By default Go will use the number of logical CPUs, which
# is a fine default.
PARALLEL_FLAG=""
set -o errexit
set -o pipefail
make test-postgres
if [ "${{ runner.os }}" == "Windows" ]; then
# Create a temp dir on the R: ramdisk drive for Windows. The default
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
mkdir -p "R:/temp/embedded-pg"
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
elif [ "${{ runner.os }}" == "macOS" ]; then
# Postgres runs faster on a ramdisk on macOS too
mkdir -p /tmp/tmpfs
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
elif [ "${{ runner.os }}" == "Linux" ]; then
make test-postgres-docker
fi
# if macOS, install google-chrome for scaletests
# As another concern, should we really have this kind of external dependency
# requirement on standard CI?
if [ "${{ matrix.os }}" == "macos-latest" ]; then
brew install google-chrome
fi
# macOS will output "The default interactive shell is now zsh"
# intermittently in CI...
if [ "${{ matrix.os }}" == "macos-latest" ]; then
touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
fi
if [ "${{ runner.os }}" == "Windows" ]; then
# Our Windows runners have 16 cores.
# On Windows Postgres chokes up when we have 16x16=256 tests
# running in parallel, and dbtestutil.NewDB starts to take more than
# 10s to complete sometimes causing test timeouts. With 16x8=128 tests
# Postgres tends not to choke.
NUM_PARALLEL_PACKAGES=8
NUM_PARALLEL_TESTS=16
elif [ "${{ runner.os }}" == "macOS" ]; then
# Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
# because the tests complete faster and Postgres doesn't choke. It seems
# that macOS's tmpfs is faster than the one on Windows.
NUM_PARALLEL_PACKAGES=8
NUM_PARALLEL_TESTS=16
elif [ "${{ runner.os }}" == "Linux" ]; then
# Our Linux runners have 8 cores.
NUM_PARALLEL_PACKAGES=8
NUM_PARALLEL_TESTS=8
fi
# by default, run tests with cache
TESTCOUNT=""
if [ "${{ github.ref }}" == "refs/heads/main" ]; then
# on main, run tests without cache
TESTCOUNT="-count=1"
fi
mkdir -p "$RUNNER_TEMP/sym"
source scripts/normalize_path.sh
# terraform gets installed in a random directory, so we need to normalize
# the path to the terraform binary or a bunch of cached tests will be
# invalidated. See scripts/normalize_path.sh for more details.
normalize_path_with_symlinks "$RUNNER_TEMP/sym" "$(dirname $(which terraform))"
# We rerun failing tests to counteract flakiness coming from Postgres
# choking on macOS and Windows sometimes.
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=50 \
--format standard-quiet --packages "./..." \
-- -timeout=20m -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS $TESTCOUNT
- name: Upload Go Build Cache
uses: ./.github/actions/test-cache/upload
with:
cache-key: ${{ steps.download-go-build-cache.outputs.cache-key }}
cache-path: ${{ steps.go-paths.outputs.cached-dirs }}
- name: Upload Test Cache
uses: ./.github/actions/test-cache/upload
.github/workflows/nightly-gauntlet.yaml
-183
View File
@@ -1,183 +0,0 @@
# The nightly-gauntlet runs tests that are either too flaky or too slow to block
# every PR.
name: nightly-gauntlet
on:
schedule:
# Every day at 4AM
- cron: "0 4 * * 1-5"
workflow_dispatch:
permissions:
contents: read
jobs:
test-go-pg:
# make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
# when changing runner sizes
runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
# This timeout must be greater than the timeout set by `go test` in
# `make test-postgres` to ensure we receive a trace of running
# goroutines. Setting this to the timeout +5m should work quite well
# even if some of the preceding steps are slow.
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
os:
- macos-latest
- windows-2022
steps:
- name: Harden Runner
uses: step-security/harden-runner@0634a2670c59f64b4a01f0f96f84700a4088b9f0 # v2.12.0
with:
egress-policy: audit
# macOS indexes all new files in the background. Our Postgres tests
# create and destroy thousands of databases on disk, and Spotlight
# tries to index all of them, seriously slowing down the tests.
- name: Disable Spotlight Indexing
if: runner.os == 'macOS'
run: |
sudo mdutil -a -i off
sudo mdutil -X /
sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
# Set up RAM disks to speed up the rest of the job. This action is in
# a separate repository to allow its use before actions/checkout.
- name: Setup RAM Disks
if: runner.os == 'Windows'
uses: coder/setup-ramdisk-action@79dacfe70c47ad6d6c0dd7f45412368802641439
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 1
- name: Setup Go
uses: ./.github/actions/setup-go
with:
# Runners have Go baked-in and Go will automatically
# download the toolchain configured in go.mod, so we don't
# need to reinstall it. It's faster on Windows runners.
use-preinstalled-go: ${{ runner.os == 'Windows' }}
use-temp-cache-dirs: ${{ runner.os == 'Windows' }}
- name: Setup Terraform
uses: ./.github/actions/setup-tf
- name: Test with PostgreSQL Database
env:
POSTGRES_VERSION: "13"
TS_DEBUG_DISCO: "true"
LC_CTYPE: "en_US.UTF-8"
LC_ALL: "en_US.UTF-8"
shell: bash
run: |
if [ "${{ runner.os }}" == "Windows" ]; then
# Create a temp dir on the R: ramdisk drive for Windows. The default
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
mkdir -p "R:/temp/embedded-pg"
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
fi
if [ "${{ runner.os }}" == "macOS" ]; then
# Postgres runs faster on a ramdisk on macOS too
mkdir -p /tmp/tmpfs
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
fi
# if macOS, install google-chrome for scaletests
# As another concern, should we really have this kind of external dependency
# requirement on standard CI?
if [ "${{ matrix.os }}" == "macos-latest" ]; then
brew install google-chrome
fi
# By default Go will use the number of logical CPUs, which
# is a fine default.
PARALLEL_FLAG=""
# macOS will output "The default interactive shell is now zsh"
# intermittently in CI...
if [ "${{ matrix.os }}" == "macos-latest" ]; then
touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
fi
# Golang's default for these 2 variables is the number of logical CPUs.
# Our Windows and Linux runners have 16 cores, so they match up there.
NUM_PARALLEL_PACKAGES=16
NUM_PARALLEL_TESTS=16
if [ "${{ runner.os }}" == "Windows" ]; then
# On Windows Postgres chokes up when we have 16x16=256 tests
# running in parallel, and dbtestutil.NewDB starts to take more than
# 10s to complete sometimes causing test timeouts. With 16x8=128 tests
# Postgres tends not to choke.
NUM_PARALLEL_PACKAGES=8
fi
if [ "${{ runner.os }}" == "macOS" ]; then
# Our macOS runners have 8 cores. We leave NUM_PARALLEL_TESTS at 16
# because the tests complete faster and Postgres doesn't choke. It seems
# that macOS's tmpfs is faster than the one on Windows.
NUM_PARALLEL_PACKAGES=8
fi
# We rerun failing tests to counteract flakiness coming from Postgres
# choking on macOS and Windows sometimes.
DB=ci gotestsum --rerun-fails=2 --rerun-fails-max-failures=1000 \
--format standard-quiet --packages "./..." \
-- -v -p $NUM_PARALLEL_PACKAGES -parallel=$NUM_PARALLEL_TESTS -count=1
- name: Upload test stats to Datadog
timeout-minutes: 1
continue-on-error: true
uses: ./.github/actions/upload-datadog
if: success() || failure()
with:
api-key: ${{ secrets.DATADOG_API_KEY }}
notify-slack-on-failure:
needs:
- test-go-pg
runs-on: ubuntu-latest
if: failure() && github.ref == 'refs/heads/main'
steps:
- name: Send Slack notification
run: |
curl -X POST -H 'Content-type: application/json' \
--data '{
"blocks": [
{
"type": "header",
"text": {
"type": "plain_text",
"text": "❌ Nightly gauntlet failed",
"emoji": true
}
},
{
"type": "section",
"fields": [
{
"type": "mrkdwn",
"text": "*Workflow:*\n${{ github.workflow }}"
},
{
"type": "mrkdwn",
"text": "*Committer:*\n${{ github.actor }}"
},
{
"type": "mrkdwn",
"text": "*Commit:*\n${{ github.sha }}"
}
]
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*View failure:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Click here>"
}
}
]
}' ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}