coder/.github/workflows/nightly-gauntlet.yaml

# The nightly-gauntlet runs tests that are either too flaky or too slow to block
# every PR.
name: nightly-gauntlet
on:
  schedule:
    # Every day at 4AM
    - cron: "0 4 * * 1-5"
  workflow_dispatch:

permissions:
  contents: read

jobs:
  test-go-pg:
    # make sure to adjust NUM_PARALLEL_PACKAGES and NUM_PARALLEL_TESTS below
    # when changing runner sizes
    runs-on: ${{ matrix.os == 'macos-latest' && github.repository_owner == 'coder' && 'depot-macos-latest' || matrix.os == 'windows-2022' && github.repository_owner == 'coder' && 'depot-windows-2022-16' || matrix.os }}
    # This timeout must be greater than the timeout set by `go test` in
    # `make test-postgres` to ensure we receive a trace of running
    # goroutines. Setting this to the timeout +5m should work quite well
    # even if some of the preceding steps are slow.
    timeout-minutes: 25
    strategy:
      matrix:
        os:
          - macos-latest
          - windows-2022
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2
        with:
          egress-policy: audit

      # macOS indexes all new files in the background. Our Postgres tests
      # create and destroy thousands of databases on disk, and Spotlight
      # tries to index all of them, seriously slowing down the tests.
      - name: Disable Spotlight Indexing
        if: runner.os == 'macOS'
        run: |
          enabled=$(sudo mdutil -a -s | { grep -Fc "Indexing enabled" || true; })
          if [ "$enabled" -eq 0 ]; then
            echo "Spotlight indexing is already disabled"
            exit 0
          fi
          sudo mdutil -a -i off
          sudo mdutil -X /
          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist

      # Set up RAM disks to speed up the rest of the job. This action is in
      # a separate repository to allow its use before actions/checkout.
      - name: Setup RAM Disks
        if: runner.os == 'Windows'
        uses: coder/setup-ramdisk-action@e1100847ab2d7bcd9d14bcda8f2d1b0f07b36f1b # v0.1.0

      - name: Checkout
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          fetch-depth: 1
          persist-credentials: false

      - name: Setup Go
        uses: ./.github/actions/setup-go
        with:
          # Runners have Go baked-in and Go will automatically
          # download the toolchain configured in go.mod, so we don't
          # need to reinstall it. It's faster on Windows runners.
          use-preinstalled-go: ${{ runner.os == 'Windows' }}

      - name: Setup Terraform
        uses: ./.github/actions/setup-tf

      - name: Setup Embedded Postgres Cache Paths
        id: embedded-pg-cache
        uses: ./.github/actions/setup-embedded-pg-cache-paths

      - name: Download Embedded Postgres Cache
        id: download-embedded-pg-cache
        uses: ./.github/actions/embedded-pg-cache/download
        with:
          key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }}
          cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }}

      - name: Test with PostgreSQL Database
        env:
          POSTGRES_VERSION: "13"
          TS_DEBUG_DISCO: "true"
          LC_CTYPE: "en_US.UTF-8"
          LC_ALL: "en_US.UTF-8"
        shell: bash
        run: |
          set -o errexit
          set -o pipefail

          if [ "${{ runner.os }}" == "Windows" ]; then
            # Create a temp dir on the R: ramdisk drive for Windows. The default
            # C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
            mkdir -p "R:/temp/embedded-pg"
            go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}"
          elif [ "${{ runner.os }}" == "macOS" ]; then
            # Postgres runs faster on a ramdisk on macOS too
            mkdir -p /tmp/tmpfs
            sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
            go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}"
          elif [ "${{ runner.os }}" == "Linux" ]; then
            make test-postgres-docker
          fi

          # if macOS, install google-chrome for scaletests
          # As another concern, should we really have this kind of external dependency
          # requirement on standard CI?
          if [ "${{ matrix.os }}" == "macos-latest" ]; then
            brew install google-chrome
          fi

          # macOS will output "The default interactive shell is now zsh"
          # intermittently in CI...
          if [ "${{ matrix.os }}" == "macos-latest" ]; then
            touch ~/.bash_profile && echo "export BASH_SILENCE_DEPRECATION_WARNING=1" >> ~/.bash_profile
          fi

          if [ "${{ runner.os }}" == "Windows" ]; then
            # Our Windows runners have 16 cores.
            # On Windows Postgres chokes up when we have 16x16=256 tests
            # running in parallel, and dbtestutil.NewDB starts to take more than
            # 10s to complete sometimes causing test timeouts. With 16x8=128 tests
            # Postgres tends not to choke.
            NUM_PARALLEL_PACKAGES=8
            NUM_PARALLEL_TESTS=16
          elif [ "${{ runner.os }}" == "macOS" ]; then
            # Our macOS runners have 8 cores. We set NUM_PARALLEL_TESTS to 16
            # because the tests complete faster and Postgres doesn't choke. It seems
            # that macOS's tmpfs is faster than the one on Windows.
            NUM_PARALLEL_PACKAGES=8
            NUM_PARALLEL_TESTS=16
          elif [ "${{ runner.os }}" == "Linux" ]; then
            # Our Linux runners have 8 cores.
            NUM_PARALLEL_PACKAGES=8
            NUM_PARALLEL_TESTS=8
          fi

          # run tests without cache
          TESTCOUNT="-count=1"

          DB=ci gotestsum \
            --format standard-quiet --packages "./..." \
            -- -timeout=20m -v -p "$NUM_PARALLEL_PACKAGES" -parallel="$NUM_PARALLEL_TESTS" "$TESTCOUNT"

      - name: Upload Embedded Postgres Cache
        uses: ./.github/actions/embedded-pg-cache/upload
        # We only use the embedded Postgres cache on macOS and Windows runners.
        if: runner.OS == 'macOS' || runner.OS == 'Windows'
        with:
          cache-key: ${{ steps.download-embedded-pg-cache.outputs.cache-key }}
          cache-path: "${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}"

      - name: Upload test stats to Datadog
        timeout-minutes: 1
        continue-on-error: true
        uses: ./.github/actions/upload-datadog
        if: success() || failure()
        with:
          api-key: ${{ secrets.DATADOG_API_KEY }}

  notify-slack-on-failure:
    needs:
      - test-go-pg
    runs-on: ubuntu-latest
    if: failure() && github.ref == 'refs/heads/main'

    steps:
      - name: Send Slack notification
        run: |
          ESCAPED_PROMPT=$(printf "%s" "<@U09LQ75AHKR> $BLINK_CI_FAILURE_PROMPT" | jq -Rsa .)
          curl -X POST -H 'Content-type: application/json' \
          --data '{
            "blocks": [
              {
                "type": "header",
                "text": {
                  "type": "plain_text",
                  "text": "❌ Nightly gauntlet failed",
                  "emoji": true
                }
              },
              {
                "type": "section",
                "text": {
                  "type": "mrkdwn",
                  "text": "*View failure:* <'"${RUN_URL}"'|Click here>"
                }
              },
              {
                "type": "section",
                "text": {
                  "type": "mrkdwn",
                  "text": '"$ESCAPED_PROMPT"'
                }
              }
            ]
          }' "${SLACK_WEBHOOK}"
        env:
          SLACK_WEBHOOK: ${{ secrets.CI_FAILURE_SLACK_WEBHOOK }}
          RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          BLINK_CI_FAILURE_PROMPT: ${{ vars.BLINK_CI_FAILURE_PROMPT }}