ci: add skip directives for long tests (#3151)

This PR introduces many CI optimizations:

1. The `[ci-skip]` PR body directive to skip the Postgres and end to end tests
2. Improved caching that cuts the Go test matrix in half
3. Increasing Go test parallelism for ~20% gains
4. Enable caching in webpack (4x frontend build)

.github/workflows/coder.yaml

@@ -60,13 +60,31 @@ jobs:
           echo "${{ toJSON(steps.filter )}}"
 
   # Debug step
-  show-changes:
-    needs: changes
+  debug-inputs:
+    needs:
+      - changes
+      - pr-context
     runs-on: ubuntu-latest
     steps:
       - id: log
         run: |
-          echo "${{ toJSON(needs.changes) }}"
+          echo "${{ toJSON(needs) }}"
+          echo "${{ contains(needs.pr-context.outputs.skips, 'postgres') }} "
+
+  pr-context:
+    runs-on: ubuntu-latest
+    outputs:
+      skips: ${{ steps.pr-context.outputs.skips }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: "~1.18"
+      - id: pr-context
+        env:
+          GITHUB_CONTEXT: ${{ toJSON(github) }}
+        run: |
+          go run github.com/coder/coder/.github/workflows/prcontext
 
   style-lint-golangci:
     name: style/lint/golangci
@@ -157,6 +175,25 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: "~1.18"
+
+      - name: Echo Go Cache Paths
+        id: go-cache-paths
+        run: |
+          echo "::set-output name=go-build::$(go env GOCACHE)"
+          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
+
+      - name: Go Build Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-build }}
+          key: ${{ github.job }}-go-build-${{ hashFiles('**/go.sum', '**/**.go') }}
+
+      - name: Go Mod Cache
+        uses: actions/cache@v3
+        with:
+          path: ${{ steps.go-cache-paths.outputs.go-mod }}
+          key: ${{ github.job }}-go-mod-${{ hashFiles('**/go.sum') }}
+
       - run: |
           curl -sSL https://github.com/kyleconroy/sqlc/releases/download/v1.13.0/sqlc_1.13.0_linux_amd64.tar.gz | sudo tar -C /usr/bin -xz sqlc
 
@@ -229,7 +266,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{ steps.go-cache-paths.outputs.go-build }}
-          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.**', '**.go') }}
 
       - name: Go Mod Cache
         uses: actions/cache@v3
@@ -251,14 +288,23 @@ jobs:
           terraform_wrapper: false
 
       - name: Test with Mock Database
+        id: test
         shell: bash
-        run: gotestsum --junitfile="gotests.xml" --packages="./..." --
-          -covermode=atomic -coverprofile="gotests.coverage"
-          -coverpkg=./...,github.com/coder/coder/codersdk
-          -timeout=5m -short -failfast
+        run: |
+          # Code coverage is more computationally expensive and also
+          # prevents test caching, so we disable it on alternate operating
+          # systems.
+          if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
+            echo ::set-output name=cover::true
+            export COVERAGE_FLAGS='-covermode=atomic -coverprofile="gotests.coverage" -coverpkg=./...,github.com/coder/coder/codersdk'
+          else
+            echo ::set-output name=cover::false
+          fi
+          set -x
+          gotestsum --junitfile="gotests.xml" --packages="./..." -- -parallel=8 -timeout=5m -short -failfast $COVERAGE_FLAGS
 
       - name: Upload DataDog Trace
-        if: always() && github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
+        if: github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
         env:
           DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
           DD_DATABASE: fake
@@ -272,7 +318,7 @@ jobs:
         # that is no guarantee, see:
         # https://github.com/codecov/codecov-action/issues/788
         continue-on-error: true
-        if: github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
+        if: steps.test.outputs.cover && github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./gotests.coverage
@@ -280,8 +326,12 @@ jobs:
 
   test-go-postgres:
     name: "test/go/postgres"
-    needs: changes
-    if: needs.changes.outputs.docs-only == 'false'
+    needs:
+      - changes
+      - pr-context
+    if: >
+      needs.changes.outputs.docs-only == 'false' &&
+      contains(needs.pr-context.outputs.skips, github.job) == 'false'
     runs-on: ubuntu-latest
     # This timeout must be greater than the timeout set by `go test` in
     # `make test-postgres` to ensure we receive a trace of running
@@ -305,7 +355,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ${{ steps.go-cache-paths.outputs.go-build }}
-          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum') }}
+          key: ${{ runner.os }}-go-build-${{ hashFiles('**/go.sum', '**/**.go') }}
 
       - name: Go Mod Cache
         uses: actions/cache@v3
@@ -517,8 +567,10 @@ jobs:
 
   test-e2e:
     name: "test/e2e/${{ matrix.os }}"
-    needs: changes
-    if: needs.changes.outputs.docs-only == 'false'
+    needs:
+      - changes
+      - pr-context
+    if: needs.changes.outputs.docs-only == 'false' && !contains(needs.pr-context.outputs.skips, 'test/e2e')
     runs-on: ${{ matrix.os }}
     timeout-minutes: 20
     strategy:
@@ -535,9 +587,7 @@ jobs:
           path: |
             **/node_modules
             .eslintcache
-          key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            js-${{ runner.os }}-
+          key: js-${{ runner.os }}-e2e-${{ hashFiles('**/yarn.lock') }}
 
       # Go is required for uploading the test results to datadog
       - uses: actions/setup-go@v3
@@ -573,6 +623,7 @@ jobs:
 
       - name: Build
         run: |
+          sudo npm install -g prettier
           make -B site/out/index.html
 
       - run: yarn playwright:install

.github/workflows/prcontext/README.md

@@ -0,0 +1,7 @@
+# prcontext
+
+`prcontext` is a simple Go program that extracts CI directives from PRs for a
+more efficient merge cycle.
+
+Right now it only supports the `[ci-skip [job ...]]` directive. Since skips are
+only possible within PRs, the full suite will still run on merge.

.github/workflows/prcontext/main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/coder/flog"
+)
+
+// githubContext is structured as documented here:
+// https://docs.github.com/en/actions/learn-github-actions/contexts#github-context.
+type githubContext struct {
+	EventName string `json:"event_name"`
+	Event     struct {
+		PullRequest struct {
+			Body string `json:"body"`
+		} `json:"pull_request"`
+	} `json:"event"`
+}
+
+func main() {
+	var c githubContext
+	err := json.Unmarshal([]byte(os.Getenv("GITHUB_CONTEXT")), &c)
+	if err != nil {
+		flog.Fatal("decode stdin: %+v", err)
+	}
+	flog.Info("detected event %q", c.EventName)
+	if c.EventName != "pull_request" {
+		flog.Info("aborting since not Pull Request")
+		return
+	}
+
+	_, _ = fmt.Printf("::group::{PR Body}\n%s\n::endgroup::\n", c.Event.PullRequest.Body)
+
+	skips := parseBody(c.Event.PullRequest.Body)
+	_, _ = fmt.Printf("::echo::on\n::set-output name=skips::[%s]\n", strings.Join(skips, " "))
+}

.github/workflows/prcontext/parse.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"regexp"
+	"strings"
+)
+
+const ciSkipPrefix = "ci-skip"
+
+var skipDirective = regexp.MustCompile(`\[` + ciSkipPrefix + ` ([\w-\/ ]+)]`)
+
+func parseBody(body string) (skips []string) {
+	matches := skipDirective.FindAllStringSubmatch(body, -1)
+	// flog.Info("matches: %+v", matches)
+
+	var skipMatches []string
+	for i := range matches {
+		for j := range matches[i] {
+			v := matches[i][j]
+			// flog.Info("%q", v)
+			if !strings.Contains(v, ciSkipPrefix) {
+				skipMatches = append(skipMatches, strings.Split(v, " ")...)
+			}
+		}
+	}
+
+	return skipMatches
+}

.github/workflows/prcontext/parse_test.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+)
+
+func Test_parseBody_basic(t *testing.T) {
+	parseBody(`
+This is a test PR.	
+
+[ci-skip postgres windows]
+	`)
+}
+
+func Test_parseBody(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name      string
+		args      args
+		wantSkips []string
+	}{
+		{"no directive", args{"test pr 123\n\n"}, nil},
+		{"single dir single skip", args{"test pr [ci-skip dog] 123\n\n"}, []string{"dog"}},
+		{"double dir double skip", args{"test pr [ci-skip dog] [ci-skip cat] 123\n\n"}, []string{"dog", "cat"}},
+		{"single dir double skip", args{"test pr [ci-skip test/go/postgres cat] 123\n\n"}, []string{"test/go/postgres", "cat"}},
+		{"confuse", args{"ci ci [ci-skip] dog [ci-skip test/go/postgres test/e2e/ubuntu-latest] 123\n\n"}, []string{"test/go/postgres", "test/e2e/ubuntu-latest"}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotSkips := parseBody(tt.args.body); !reflect.DeepEqual(gotSkips, tt.wantSkips) {
+				t.Errorf("parseBody() = %v, want %v", gotSkips, tt.wantSkips)
+			}
+		})
+	}
+}

go.mod

@@ -135,7 +135,10 @@ require (
 	tailscale.com v1.26.2
 )
 
-require github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
+require (
+	github.com/coder/flog v1.0.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
+)
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect

go.sum

@@ -344,6 +344,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
+github.com/coder/flog v1.0.0 h1:gqr4jYDQWYmsvFD0RV6Vs+SAj1Kbn0HGlV7UghfxP+8=
+github.com/coder/flog v1.0.0/go.mod h1:UQlQvrkJBvnRGo69Le8E24Tcl5SJleAAR7gYEHzAmdQ=
 github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1 h1:UqBrPWSYvRI2s5RtOul20JukUEpu4ip9u7biBL+ntgk=
 github.com/coder/glog v1.0.1-0.20220322161911-7365fe7f2cd1/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/coder/retry v1.3.0 h1:5lAAwt/2Cm6lVmnfBY7sOMXcBOwcwJhmV5QGSELIVWY=

site/webpack.common.ts

@@ -93,6 +93,10 @@ export const createCommonWebpackConfig = (options?: { skipTypecheck: boolean }):
     ],
   },
 
+  cache: {
+    type: "filesystem",
+  },
+
   // resolve extend/modify how modules are resolved.
   //
   // REMARK: Do not add aliases here, unless they cannot be defined in a