chore: cache terraform providers between CI test runs (#17373)

Addresses https://github.com/coder/internal/issues/322.

This PR starts caching Terraform providers used by `TestProvision` in
`provisioner/terraform/provision_test.go`. The goal is to improve the
reliability of this test by cutting down on the number of network calls
to external services. It leverages GitHub Actions cache, which [on depot
runners is persisted for 14 days by
default](https://depot.dev/docs/github-actions/overview#cache-retention-policy).

Other than the aforementioned `TestProvision`, I couldn't find any other
tests which depend on external terraform providers.

.github/actions/test-cache/download/action.yml

@@ -0,0 +1,50 @@
+name: "Download Test Cache"
+description: |
+  Downloads the test cache and outputs today's cache key.
+  A PR job can use a cache if it was created by its base branch, its current
+  branch, or the default branch.
+  https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
+outputs:
+  cache-key:
+    description: "Today's cache key"
+    value: ${{ steps.vars.outputs.cache-key }}
+inputs:
+  key-prefix:
+    description: "Prefix for the cache key"
+    required: true
+  cache-path:
+    description: "Path to the cache directory"
+    required: true
+    # This path is defined in testutil/cache.go
+    default: "~/.cache/coderv2-test"
+runs:
+  using: "composite"
+  steps:
+    - name: Get date values and cache key
+      id: vars
+      shell: bash
+      run: |
+        export YEAR_MONTH=$(date +'%Y-%m')
+        export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
+        export DAY=$(date +'%d')
+        echo "year-month=$YEAR_MONTH" >> $GITHUB_OUTPUT
+        echo "prev-year-month=$PREV_YEAR_MONTH" >> $GITHUB_OUTPUT
+        echo "cache-key=${{ inputs.key-prefix }}-${YEAR_MONTH}-${DAY}" >> $GITHUB_OUTPUT
+
+    # TODO: As a cost optimization, we could remove caches that are older than
+    # a day or two. By default, depot keeps caches for 14 days, which isn't
+    # necessary for the test cache.
+    # https://depot.dev/docs/github-actions/overview#cache-retention-policy
+    - name: Download test cache
+      uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: ${{ inputs.cache-path }}
+        key: ${{ steps.vars.outputs.cache-key }}
+        # > If there are multiple partial matches for a restore key, the action returns the most recently created cache.
+        # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
+        # The second restore key allows non-main branches to use the cache from the previous month.
+        # This prevents PRs from rebuilding the cache on the first day of the month.
+        # It also makes sure that once a month, the cache is fully reset.
+        restore-keys: |
+          ${{ inputs.key-prefix }}-${{ steps.vars.outputs.year-month }}-
+          ${{ github.ref != 'refs/heads/main' && format('{0}-{1}-', inputs.key-prefix, steps.vars.outputs.prev-year-month) || '' }}

.github/actions/test-cache/upload/action.yml

@@ -0,0 +1,20 @@
+name: "Upload Test Cache"
+description: Uploads the test cache. Only works on the main branch.
+inputs:
+  cache-key:
+    description: "Cache key"
+    required: true
+  cache-path:
+    description: "Path to the cache directory"
+    required: true
+    # This path is defined in testutil/cache.go
+    default: "~/.cache/coderv2-test"
+runs:
+  using: "composite"
+  steps:
+    - name: Upload test cache
+      if: ${{ github.ref == 'refs/heads/main' }}
+      uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      with:
+        path: ${{ inputs.cache-path }}
+        key: ${{ inputs.cache-key }}

.github/workflows/ci.yaml

@@ -341,6 +341,12 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-${{ runner.os }}-${{ runner.arch }}
+
       - name: Test with Mock Database
         id: test
         shell: bash
@@ -365,6 +371,11 @@ jobs:
           gotestsum --junitfile="gotests.xml" --jsonfile="gotests.json" \
             --packages="./..." -- $PARALLEL_FLAG -short -failfast
 
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
+
       - name: Upload test stats to Datadog
         timeout-minutes: 1
         continue-on-error: true
@@ -462,6 +473,12 @@ jobs:
         if: runner.os == 'Windows'
         uses: ./.github/actions/setup-imdisk
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
+
       - name: Test with PostgreSQL Database
         env:
           POSTGRES_VERSION: "13"
@@ -476,6 +493,11 @@ jobs:
 
           make test-postgres
 
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
+
       - name: Upload test stats to Datadog
         timeout-minutes: 1
         continue-on-error: true
@@ -514,6 +536,12 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-pg-16-${{ runner.os }}-${{ runner.arch }}
+
       - name: Test with PostgreSQL Database
         env:
           POSTGRES_VERSION: "16"
@@ -521,6 +549,11 @@ jobs:
         run: |
           make test-postgres
 
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
+
       - name: Upload test stats to Datadog
         timeout-minutes: 1
         continue-on-error: true
@@ -551,6 +584,12 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-race-${{ runner.os }}-${{ runner.arch }}
+
       # We run race tests with reduced parallelism because they use more CPU and we were finding
       # instances where tests appear to hang for multiple seconds, resulting in flaky tests when
       # short timeouts are used.
@@ -559,6 +598,11 @@ jobs:
         run: |
           gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
 
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
+
       - name: Upload test stats to Datadog
         timeout-minutes: 1
         continue-on-error: true
@@ -589,6 +633,12 @@ jobs:
       - name: Setup Terraform
         uses: ./.github/actions/setup-tf
 
+      - name: Download Test Cache
+        id: download-cache
+        uses: ./.github/actions/test-cache/download
+        with:
+          key-prefix: test-go-race-pg-${{ runner.os }}-${{ runner.arch }}
+
       # We run race tests with reduced parallelism because they use more CPU and we were finding
       # instances where tests appear to hang for multiple seconds, resulting in flaky tests when
       # short timeouts are used.
@@ -600,6 +650,11 @@ jobs:
           make test-postgres-docker
           DB=ci gotestsum --junitfile="gotests.xml" -- -race -parallel 4 -p 4 ./...
 
+      - name: Upload Test Cache
+        uses: ./.github/actions/test-cache/upload
+        with:
+          cache-key: ${{ steps.download-cache.outputs.cache-key }}
+
       - name: Upload test stats to Datadog
         timeout-minutes: 1
         continue-on-error: true