diff --git a/.gitignore b/.gitignore index 92f71bf2ad..7e0823020c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ site/.swc # Make target for updating generated/golden files (any dir). .gen +/_gen/ .gen-golden # Build diff --git a/Makefile b/Makefile index 3095ad07d0..1b29fc663f 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,25 @@ SHELL := bash coderd/database/unique_constraint.go \ coderd/database/dbmetrics/querymetrics.go \ coderd/database/dbauthz/dbauthz.go \ + coderd/database/dbmock/dbmock.go \ + coderd/database/pubsub/psmock/psmock.go \ + agent/agentcontainers/acmock/acmock.go \ + coderd/httpmw/loggermw/loggermock/loggermock.go \ + codersdk/workspacesdk/agentconnmock/agentconnmock.go \ + tailnet/tailnettest/coordinatormock.go \ + tailnet/tailnettest/coordinateemock.go \ + tailnet/tailnettest/workspaceupdatesprovidermock.go \ + tailnet/tailnettest/subscriptionmock.go \ + enterprise/aibridged/aibridgedmock/clientmock.go \ + enterprise/aibridged/aibridgedmock/poolmock.go \ + tailnet/proto/tailnet.pb.go \ + agent/proto/agent.pb.go \ + agent/agentsocket/proto/agentsocket.pb.go \ + agent/boundarylogproxy/codec/boundary.pb.go \ + provisionersdk/proto/provisioner.pb.go \ + provisionerd/proto/provisionerd.pb.go \ + vpn/vpn.pb.go \ + enterprise/aibridged/proto/aibridged.pb.go \ site/src/api/typesGenerated.ts \ site/e2e/provisionerGenerated.ts \ site/src/api/chatModelOptionsGenerated.json \ @@ -50,6 +69,23 @@ SHELL := bash codersdk/rbacresources_gen.go \ codersdk/apikey_scopes_gen.go +# atomic_write runs a command, captures stdout into a temp file, and +# atomically replaces $@. An optional second argument is a formatting +# command that receives the temp file path as its argument. +# Usage: $(call atomic_write,GENERATE_CMD[,FORMAT_CMD]) +define atomic_write + tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && \ + $(1) > "$$tmpfile" && \ + $(if $(2),$(2) "$$tmpfile" &&) \ + mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" +endef + +# Shared temp directory for atomic writes. Lives at the project root +# so all targets share the same filesystem, and is gitignored. +# Order-only prerequisite: recipes that need it depend on | _gen +_gen: + mkdir -p _gen + # Don't print the commands in the file unless you specify VERBOSE. This is # essentially the same as putting "@" at the start of each line. ifndef VERBOSE @@ -822,7 +858,7 @@ $(TAILNETTEST_MOCKS): tailnet/coordinator.go tailnet/service.go touch "$@" tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ --go-drpc_out=. \ @@ -830,7 +866,7 @@ tailnet/proto/tailnet.pb.go: tailnet/proto/tailnet.proto ./tailnet/proto/tailnet.proto agent/proto/agent.pb.go: agent/proto/agent.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ --go-drpc_out=. \ @@ -838,7 +874,7 @@ agent/proto/agent.pb.go: agent/proto/agent.proto ./agent/proto/agent.proto agent/agentsocket/proto/agentsocket.pb.go: agent/agentsocket/proto/agentsocket.proto agent/proto/agent.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ --go-drpc_out=. \ @@ -846,7 +882,7 @@ agent/agentsocket/proto/agentsocket.pb.go: agent/agentsocket/proto/agentsocket.p ./agent/agentsocket/proto/agentsocket.proto provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ --go-drpc_out=. \ @@ -854,7 +890,7 @@ provisionersdk/proto/provisioner.pb.go: provisionersdk/proto/provisioner.proto ./provisionersdk/proto/provisioner.proto provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ --go-drpc_out=. \ @@ -862,132 +898,110 @@ provisionerd/proto/provisionerd.pb.go: provisionerd/proto/provisionerd.proto ./provisionerd/proto/provisionerd.proto vpn/vpn.pb.go: vpn/vpn.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ ./vpn/vpn.proto agent/boundarylogproxy/codec/boundary.pb.go: agent/boundarylogproxy/codec/boundary.proto agent/proto/agent.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ ./agent/boundarylogproxy/codec/boundary.proto enterprise/aibridged/proto/aibridged.pb.go: enterprise/aibridged/proto/aibridged.proto - protoc \ + ./scripts/atomic_protoc.sh \ --go_out=. \ --go_opt=paths=source_relative \ --go-drpc_out=. \ --go-drpc_opt=paths=source_relative \ ./enterprise/aibridged/proto/aibridged.proto -site/src/api/typesGenerated.ts: site/node_modules/.installed $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go') - # Generate to a temp file, format it, then atomically move to - # the target so that an interrupt never leaves a partial or - # unformatted file in the working tree. - tmpfile=$$(mktemp -d)/$(notdir $@) && \ - go run -C ./scripts/apitypings main.go > "$$tmpfile" && \ - ./scripts/biome_format.sh "$$tmpfile" && \ - mv "$$tmpfile" "$@" +site/src/api/typesGenerated.ts: site/node_modules/.installed $(wildcard scripts/apitypings/*) $(shell find ./codersdk $(FIND_EXCLUSIONS) -type f -name '*.go') | _gen + $(call atomic_write,go run -C ./scripts/apitypings main.go,./scripts/biome_format.sh) site/e2e/provisionerGenerated.ts: site/node_modules/.installed provisionerd/proto/provisionerd.pb.go provisionersdk/proto/provisioner.pb.go (cd site/ && pnpm run gen:provisioner) touch "$@" -site/src/theme/icons.json: site/node_modules/.installed $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*) - tmpfile=$$(mktemp -d)/$(notdir $@) && \ +site/src/theme/icons.json: site/node_modules/.installed $(wildcard scripts/gensite/*) $(wildcard site/static/icon/*) | _gen + tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && \ go run ./scripts/gensite/ -icons "$$tmpfile" && \ ./scripts/biome_format.sh "$$tmpfile" && \ - mv "$$tmpfile" "$@" + mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" -examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates) - go run ./scripts/examplegen/main.go > "$@.tmp" && mv "$@.tmp" "$@" +examples/examples.gen.json: scripts/examplegen/main.go examples/examples.go $(shell find ./examples/templates) | _gen + $(call atomic_write,go run ./scripts/examplegen/main.go) -coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go - tempdir=$(shell mktemp -d /tmp/typegen_rbac_object.XXXXXX) - go run ./scripts/typegen/main.go rbac object > "$$tempdir/object_gen.go" - mv -v "$$tempdir/object_gen.go" coderd/rbac/object_gen.go - rmdir -v "$$tempdir" +coderd/rbac/object_gen.go: scripts/typegen/rbacobject.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go | _gen + $(call atomic_write,go run ./scripts/typegen/main.go rbac object) touch "$@" # NOTE: depends on object_gen.go because `go run` compiles # coderd/rbac which includes it. coderd/rbac/scopes_constants_gen.go: scripts/typegen/scopenames.gotmpl scripts/typegen/main.go coderd/rbac/policy/policy.go \ - coderd/rbac/object_gen.go - # Generate typed low-level ScopeName constants from RBACPermissions - # Write to a temp file first to avoid truncating the package during build - # since the generator imports the rbac package. - tempfile=$(shell mktemp /tmp/scopes_constants_gen.XXXXXX) - go run ./scripts/typegen/main.go rbac scopenames > "$$tempfile" - mv -v "$$tempfile" coderd/rbac/scopes_constants_gen.go + coderd/rbac/object_gen.go | _gen + # Write to a temp file first to avoid truncating the package + # during build since the generator imports the rbac package. + $(call atomic_write,go run ./scripts/typegen/main.go rbac scopenames) touch "$@" # NOTE: depends on object_gen.go and scopes_constants_gen.go because # `go run` compiles coderd/rbac which includes both. codersdk/rbacresources_gen.go: scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go \ - coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go - # Do no overwrite codersdk/rbacresources_gen.go directly, as it would make the file empty, breaking - # the `codersdk` package and any parallel build targets. - go run scripts/typegen/main.go rbac codersdk > /tmp/rbacresources_gen.go - mv /tmp/rbacresources_gen.go codersdk/rbacresources_gen.go + coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go | _gen + # Write to a temp file to avoid truncating the target, which + # would break the codersdk package and any parallel build targets. + $(call atomic_write,go run scripts/typegen/main.go rbac codersdk) touch "$@" # NOTE: depends on object_gen.go and scopes_constants_gen.go because # `go run` compiles coderd/rbac which includes both. codersdk/apikey_scopes_gen.go: scripts/apikeyscopesgen/main.go coderd/rbac/scopes_catalog.go coderd/rbac/scopes.go \ - coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go + coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go | _gen # Generate SDK constants for external API key scopes. - go run ./scripts/apikeyscopesgen > /tmp/apikey_scopes_gen.go - mv /tmp/apikey_scopes_gen.go codersdk/apikey_scopes_gen.go + $(call atomic_write,go run ./scripts/apikeyscopesgen) touch "$@" # NOTE: depends on object_gen.go and scopes_constants_gen.go because # `go run` compiles coderd/rbac which includes both. site/src/api/rbacresourcesGenerated.ts: site/node_modules/.installed scripts/typegen/codersdk.gotmpl scripts/typegen/main.go coderd/rbac/object.go coderd/rbac/policy/policy.go \ - coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go - tmpfile=$$(mktemp -d)/$(notdir $@) && \ - go run scripts/typegen/main.go rbac typescript > "$$tmpfile" && \ - ./scripts/biome_format.sh "$$tmpfile" && \ - mv "$$tmpfile" "$@" + coderd/rbac/object_gen.go coderd/rbac/scopes_constants_gen.go | _gen + $(call atomic_write,go run scripts/typegen/main.go rbac typescript,./scripts/biome_format.sh) -site/src/api/countriesGenerated.ts: site/node_modules/.installed scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go - tmpfile=$$(mktemp -d)/$(notdir $@) && \ - go run scripts/typegen/main.go countries > "$$tmpfile" && \ - ./scripts/biome_format.sh "$$tmpfile" && \ - mv "$$tmpfile" "$@" +site/src/api/countriesGenerated.ts: site/node_modules/.installed scripts/typegen/countries.tstmpl scripts/typegen/main.go codersdk/countries.go | _gen + $(call atomic_write,go run scripts/typegen/main.go countries,./scripts/biome_format.sh) -site/src/api/chatModelOptionsGenerated.json: scripts/modeloptionsgen/main.go codersdk/chats.go - tmpfile=$$(mktemp -d)/$(notdir $@) && \ - go run ./scripts/modeloptionsgen/main.go | tail -n +2 > "$$tmpfile" && \ - ./scripts/biome_format.sh "$$tmpfile" && \ - mv "$$tmpfile" "$@" +site/src/api/chatModelOptionsGenerated.json: scripts/modeloptionsgen/main.go codersdk/chats.go | _gen + $(call atomic_write,go run ./scripts/modeloptionsgen/main.go | tail -n +2,./scripts/biome_format.sh) -scripts/metricsdocgen/generated_metrics: $(GO_SRC_FILES) - go run ./scripts/metricsdocgen/scanner > $@.tmp && mv $@.tmp $@ +scripts/metricsdocgen/generated_metrics: $(GO_SRC_FILES) | _gen + $(call atomic_write,go run ./scripts/metricsdocgen/scanner) -docs/admin/integrations/prometheus.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics scripts/metricsdocgen/generated_metrics - tmpfile=$$(mktemp -d)/$(notdir $@) && cp "$@" "$$tmpfile" && \ +docs/admin/integrations/prometheus.md: node_modules/.installed scripts/metricsdocgen/main.go scripts/metricsdocgen/metrics scripts/metricsdocgen/generated_metrics | _gen + tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && cp "$@" "$$tmpfile" && \ go run scripts/metricsdocgen/main.go --prometheus-doc-file="$$tmpfile" && \ pnpm exec markdownlint-cli2 --fix "$$tmpfile" && \ pnpm exec markdown-table-formatter "$$tmpfile" && \ - mv "$$tmpfile" "$@" + mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" -docs/reference/cli/index.md: node_modules/.installed scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES) - tmpdir=$$(mktemp -d) && \ +docs/reference/cli/index.md: node_modules/.installed scripts/clidocgen/main.go examples/examples.gen.json $(GO_SRC_FILES) | _gen + tmpdir=$$(mktemp -d -p _gen) && \ + tmpdir=$$(realpath "$$tmpdir") && \ mkdir -p "$$tmpdir/docs/reference/cli" && \ cp docs/manifest.json "$$tmpdir/docs/manifest.json" && \ CI=true DOCS_DIR="$$tmpdir/docs" go run ./scripts/clidocgen && \ pnpm exec markdownlint-cli2 --fix "$$tmpdir/docs/reference/cli/*.md" && \ pnpm exec markdown-table-formatter "$$tmpdir/docs/reference/cli/*.md" && \ - cp "$$tmpdir/docs/reference/cli/"*.md docs/reference/cli/ && \ + for f in "$$tmpdir/docs/reference/cli/"*.md; do mv "$$f" "docs/reference/cli/$$(basename "$$f")"; done && \ rm -rf "$$tmpdir" -docs/admin/security/audit-logs.md: node_modules/.installed coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go - tmpfile=$$(mktemp -d)/$(notdir $@) && cp "$@" "$$tmpfile" && \ +docs/admin/security/audit-logs.md: node_modules/.installed coderd/database/querier.go scripts/auditdocgen/main.go enterprise/audit/table.go coderd/rbac/object_gen.go | _gen + tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && cp "$@" "$$tmpfile" && \ go run scripts/auditdocgen/main.go --audit-doc-file="$$tmpfile" && \ pnpm exec markdownlint-cli2 --fix "$$tmpfile" && \ pnpm exec markdown-table-formatter "$$tmpfile" && \ - mv "$$tmpfile" "$@" + mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" coderd/apidoc/.gen: \ node_modules/.installed \ @@ -1002,25 +1016,28 @@ coderd/apidoc/.gen: \ scripts/apidocgen/generate.sh \ scripts/apidocgen/swaginit/main.go \ $(wildcard scripts/apidocgen/postprocess/*) \ - $(wildcard scripts/apidocgen/markdown-template/*) - tmpdir=$$(mktemp -d) && swagtmp=$$(mktemp -d) && \ + $(wildcard scripts/apidocgen/markdown-template/*) | _gen + tmpdir=$$(mktemp -d -p _gen) && swagtmp=$$(mktemp -d -p _gen) && \ + tmpdir=$$(realpath "$$tmpdir") && swagtmp=$$(realpath "$$swagtmp") && \ mkdir -p "$$tmpdir/reference/api" && \ cp docs/manifest.json "$$tmpdir/manifest.json" && \ SWAG_OUTPUT_DIR="$$swagtmp" APIDOCGEN_DOCS_DIR="$$tmpdir" ./scripts/apidocgen/generate.sh && \ pnpm exec markdownlint-cli2 --fix "$$tmpdir/reference/api/*.md" && \ pnpm exec markdown-table-formatter "$$tmpdir/reference/api/*.md" && \ ./scripts/biome_format.sh "$$swagtmp/swagger.json" && \ - cp "$$tmpdir/reference/api/"*.md docs/reference/api/ && \ - cp "$$tmpdir/manifest.json" docs/manifest.json && \ - cp "$$swagtmp/docs.go" coderd/apidoc/docs.go && \ - cp "$$swagtmp/swagger.json" coderd/apidoc/swagger.json && \ + for f in "$$tmpdir/reference/api/"*.md; do mv "$$f" "docs/reference/api/$$(basename "$$f")"; done && \ + mv "$$tmpdir/manifest.json" _gen/manifest-staging.json && \ + mv "$$swagtmp/docs.go" coderd/apidoc/docs.go && \ + mv "$$swagtmp/swagger.json" coderd/apidoc/swagger.json && \ rm -rf "$$tmpdir" "$$swagtmp" touch "$@" -docs/manifest.json: site/node_modules/.installed coderd/apidoc/.gen docs/reference/cli/index.md - tmpfile=$$(mktemp -d)/$(notdir $@) && cp "$@" "$$tmpfile" && \ +docs/manifest.json: site/node_modules/.installed coderd/apidoc/.gen docs/reference/cli/index.md | _gen + tmpdir=$$(mktemp -d -p _gen) && tmpfile=$$(realpath "$$tmpdir")/$(notdir $@) && \ + cp _gen/manifest-staging.json "$$tmpfile" && \ ./scripts/biome_format.sh "$$tmpfile" && \ - mv "$$tmpfile" "$@" + mv "$$tmpfile" "$@" && rm -rf "$$tmpdir" && \ + rm -f _gen/manifest-staging.json coderd/apidoc/swagger.json: site/node_modules/.installed coderd/apidoc/.gen touch "$@" diff --git a/coderd/database/generate.sh b/coderd/database/generate.sh index ddab225e1b..55dddbb768 100755 --- a/coderd/database/generate.sh +++ b/coderd/database/generate.sh @@ -22,8 +22,12 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") # The logic below depends on the exact version being correct :( sqlc generate - tmpfile=$(mktemp "${TMPDIR:-/tmp}/queries.sql.go.XXXXXX") - trap 'rm -f "$tmpfile"' EXIT + # Work directory for formatting before atomic replacement of + # generated files, ensuring the source tree is never left in a + # partially written state. + mkdir -p ../../_gen + workdir=$(mktemp -d ../../_gen/.dbgen.XXXXXX) + trap 'rm -rf "$workdir"' EXIT first=true files=$(find ./queries/ -type f -name "*.sql.go" | LC_ALL=C sort) @@ -38,32 +42,34 @@ SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") # Copy the header from the first file only, ignoring the source comment. if $first; then - head -n 6 <"$fi" | grep -v "source" >"$tmpfile" + head -n 6 <"$fi" | grep -v "source" >"$workdir/queries.sql.go" first=false fi # Append the file past the imports section into queries.sql.go. - tail -n "+$cut" <"$fi" >>"$tmpfile" + tail -n "+$cut" <"$fi" >>"$workdir/queries.sql.go" done - # Atomically replace the target file. - mv "$tmpfile" queries.sql.go - - # Move the files we want. - mv queries/querier.go . - mv queries/models.go . + # Move sqlc outputs into workdir for formatting. + mv queries/querier.go "$workdir/querier.go" + mv queries/models.go "$workdir/models.go" # Remove temporary go files. rm -f queries/*.go - # Fix struct/interface names. - gofmt -w -r 'Querier -> sqlcQuerier' -- *.go - gofmt -w -r 'Queries -> sqlQuerier' -- *.go + # Fix struct/interface names in the workdir (not the source tree). + gofmt -w -r 'Querier -> sqlcQuerier' -- "$workdir"/*.go + gofmt -w -r 'Queries -> sqlQuerier' -- "$workdir"/*.go - # Ensure correct imports exist. Modules must all be downloaded so we get correct - # suggestions. + # Ensure correct imports exist. Modules must all be downloaded so we + # get correct suggestions. go mod download - go tool golang.org/x/tools/cmd/goimports -w queries.sql.go + go tool golang.org/x/tools/cmd/goimports -w "$workdir/queries.sql.go" + + # Atomically replace all three target files. + mv "$workdir/queries.sql.go" queries.sql.go + mv "$workdir/querier.go" querier.go + mv "$workdir/models.go" models.go go run ../../scripts/dbgen # This will error if a view is broken. This is in it's own package to avoid diff --git a/scripts/atomic_protoc.sh b/scripts/atomic_protoc.sh new file mode 100755 index 0000000000..085c06026c --- /dev/null +++ b/scripts/atomic_protoc.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Runs protoc into a temporary directory, then atomically moves each +# generated file to the source tree. This prevents interrupted builds +# from leaving truncated or deleted .pb.go files. +# +# Usage: atomic_protoc.sh [protoc flags...] ./path/to/file.proto + +set -euo pipefail + +mkdir -p _gen +tmpdir=$(mktemp -d -p _gen) +trap 'rm -rf "$tmpdir"' EXIT + +# Rewrite --go_out=. and --go-drpc_out=. to point at tmpdir. +args=() +for arg in "$@"; do + case "$arg" in + --go_out=.) args+=("--go_out=$tmpdir") ;; + --go-drpc_out=.) args+=("--go-drpc_out=$tmpdir") ;; + *) args+=("$arg") ;; + esac +done + +protoc "${args[@]}" + +# Move all generated .go files from tmpdir back to the source tree. +find "$tmpdir" -name '*.go' -print0 | while IFS= read -r -d '' f; do + dest="${f#"$tmpdir"/}" + mv "$f" "$dest" +done diff --git a/scripts/biome_format.sh b/scripts/biome_format.sh index fe41515260..54bf4881c2 100755 --- a/scripts/biome_format.sh +++ b/scripts/biome_format.sh @@ -16,7 +16,7 @@ trap 'rm -f "$output_file"' EXIT if ( cd "$repo_root/site" - pnpm exec biome format --write "$target" + pnpm exec biome format --write --vcs-enabled=false "$target" ) >"$output_file" 2>&1; then cat "$output_file" exit 0