diff --git a/cli/server.go b/cli/server.go index 1bcce6e233..99daf70168 100644 --- a/cli/server.go +++ b/cli/server.go @@ -865,8 +865,18 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. options.PrometheusRegistry.MustRegister(pgPubsub) } - // Use NATS for pubsub if the experiment is enabled. - if experiments.Enabled(codersdk.ExperimentNATSPubsub) { + useNATSPubsub := !experiments.Enabled(codersdk.ExperimentNoNATSPubsub) + // NATS clustering needs this replica's routable address (clusterHost, + // from --cluster-host or the DERP relay URL). Neither being set is a + // valid legacy config (DERP disabled on the primary), so fall back to + // PG pubsub rather than start a NATS node that can never cluster. + if useNATSPubsub && options.ClusterHost == "" { + logger.Error(ctx, "embedded NATS pubsub is enabled but this replica has no cluster host; "+ + "set --cluster-host (CODER_CLUSTER_HOST) to this replica's routable IP address, "+ + "or configure the DERP relay URL; falling back to PostgreSQL pubsub") + useNATSPubsub = false + } + if useNATSPubsub { token := fmt.Sprintf("%x", sha256.Sum256([]byte(dbURL))) natsps, err := nats.New(ctx, logger.Named("nats_pubsub"), nats.Options{ ClusterAuthToken: token, diff --git a/cli/server_test.go b/cli/server_test.go index 139cc3f054..c83a454d50 100644 --- a/cli/server_test.go +++ b/cli/server_test.go @@ -333,6 +333,10 @@ func TestServer(t *testing.T) { "--http-address", "127.0.0.1:0", "--access-url", "http://localhost:3000/", "--cache-dir", t.TempDir(), + // NATS pubsub is on by default and needs this replica's routable + // address; without it the server logs an error and falls back to + // PostgreSQL pubsub, which would count against the log budget. + "--cluster-host", "127.0.0.1", ) pty := ptytest.New(t).Attach(inv) require.NoError(t, pty.Resize(20, 80)) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index fa75cfa07d..daecc5ee55 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -23548,7 +23548,7 @@ const docTemplate = `{ "mcp-server-http", "mcp-tool-search", "workspace-build-updates", - "nats_pubsub", + "no_nats_pubsub", "workspace-capable-licensing", "ai-gateway-seat-exclusion", "ai-gateway-reverse-proxy", @@ -23566,7 +23566,7 @@ const docTemplate = `{ "ExperimentExample": "This isn't used for anything.", "ExperimentMCPServerHTTP": "Enables the MCP HTTP server functionality.", "ExperimentMCPToolSearch": "Defers MCP tool schemas behind a searchable catalog in agent chats.", - "ExperimentNATSPubsub": "Enables embedded NATS pubsub.", + "ExperimentNoNATSPubsub": "Disables the embedded NATS pubsub, falling back to PostgreSQL pubsub.", "ExperimentNotifications": "Sends notifications via SMTP and webhooks following certain events.", "ExperimentWorkspaceBuildUpdates": "Enables publishing workspace build updates to the all builds pubsub channel.", "ExperimentWorkspaceCapableLicensing": "Counts only users holding the workspace-create permission toward the license seat limit.", @@ -23580,7 +23580,7 @@ const docTemplate = `{ "Enables the MCP HTTP server functionality.", "Defers MCP tool schemas behind a searchable catalog in agent chats.", "Enables publishing workspace build updates to the all builds pubsub channel.", - "Enables embedded NATS pubsub.", + "Disables the embedded NATS pubsub, falling back to PostgreSQL pubsub.", "Counts only users holding the workspace-create permission toward the license seat limit.", "Excludes AI Gateway (AI Bridge) usage from AI Governance seat consumption.", "Uses stateless reverse proxy routing when MCP injection is not configured.", @@ -23596,7 +23596,7 @@ const docTemplate = `{ "ExperimentMCPServerHTTP", "ExperimentMCPToolSearch", "ExperimentWorkspaceBuildUpdates", - "ExperimentNATSPubsub", + "ExperimentNoNATSPubsub", "ExperimentWorkspaceCapableLicensing", "ExperimentAIGatewaySeatExclusion", "ExperimentAIGatewayReverseProxy", diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 313546a597..c4d3dd05eb 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -21434,7 +21434,7 @@ "mcp-server-http", "mcp-tool-search", "workspace-build-updates", - "nats_pubsub", + "no_nats_pubsub", "workspace-capable-licensing", "ai-gateway-seat-exclusion", "ai-gateway-reverse-proxy", @@ -21452,7 +21452,7 @@ "ExperimentExample": "This isn't used for anything.", "ExperimentMCPServerHTTP": "Enables the MCP HTTP server functionality.", "ExperimentMCPToolSearch": "Defers MCP tool schemas behind a searchable catalog in agent chats.", - "ExperimentNATSPubsub": "Enables embedded NATS pubsub.", + "ExperimentNoNATSPubsub": "Disables the embedded NATS pubsub, falling back to PostgreSQL pubsub.", "ExperimentNotifications": "Sends notifications via SMTP and webhooks following certain events.", "ExperimentWorkspaceBuildUpdates": "Enables publishing workspace build updates to the all builds pubsub channel.", "ExperimentWorkspaceCapableLicensing": "Counts only users holding the workspace-create permission toward the license seat limit.", @@ -21466,7 +21466,7 @@ "Enables the MCP HTTP server functionality.", "Defers MCP tool schemas behind a searchable catalog in agent chats.", "Enables publishing workspace build updates to the all builds pubsub channel.", - "Enables embedded NATS pubsub.", + "Disables the embedded NATS pubsub, falling back to PostgreSQL pubsub.", "Counts only users holding the workspace-create permission toward the license seat limit.", "Excludes AI Gateway (AI Bridge) usage from AI Governance seat consumption.", "Uses stateless reverse proxy routing when MCP injection is not configured.", @@ -21482,7 +21482,7 @@ "ExperimentMCPServerHTTP", "ExperimentMCPToolSearch", "ExperimentWorkspaceBuildUpdates", - "ExperimentNATSPubsub", + "ExperimentNoNATSPubsub", "ExperimentWorkspaceCapableLicensing", "ExperimentAIGatewaySeatExclusion", "ExperimentAIGatewayReverseProxy", diff --git a/coderd/coderd.go b/coderd/coderd.go index 2ae2b333b5..8ec3875e4d 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -648,32 +648,17 @@ func New(options *Options) *API { updatesProvider := NewUpdatesProvider(options.Logger.Named("workspace_updates"), options.Pubsub, options.Database, options.Authorizer) - // The NATS cluster CA is only minted and served when NATS pubsub is in use. - // It is experiment-gated, so it is opted into rotation and backed by a real - // signing cache only when the experiment is enabled; otherwise the rotator - // leaves it alone and the cache is a noop, which still answers requests (the - // pubsub treats a missing CA as "mTLS off"). This avoids minting CA private - // keys on deployments that never run NATS clustering. - rotatedFeatures := cryptokeys.DefaultRotatedFeatures() - if experiments.Enabled(codersdk.ExperimentNATSPubsub) { - rotatedFeatures = append(rotatedFeatures, database.CryptoKeyFeatureNATSCA) - } - // Start a background process that rotates keys. We intentionally start this after the caches // are created to force initial requests for a key to populate the caches. This helps catch // bugs that may only occur when a key isn't precached in tests and the latency cost is minimal. - cryptokeys.StartRotator(ctx, options.Logger, options.Database, cryptokeys.WithFeatures(rotatedFeatures)) + cryptokeys.StartRotator(ctx, options.Logger, options.Database) // The NATS CA cache is read-only and depends on the rotator having minted // the nats_ca CA, so it must be constructed after StartRotator. if options.NATSCACache == nil { - if experiments.Enabled(codersdk.ExperimentNATSPubsub) { - options.NATSCACache, err = cryptokeys.NewSigningCache(ctx, options.Logger.Named("nats_ca_cache"), &cryptokeys.DBFetcher{DB: options.Database}, codersdk.CryptoKeyFeatureNATSCA) - if err != nil { - options.Logger.Fatal(ctx, "failed to instantiate NATS CA cache", slog.Error(err)) - } - } else { - options.NATSCACache = cryptokeys.NoopSigningKeycache{} + options.NATSCACache, err = cryptokeys.NewSigningCache(ctx, options.Logger.Named("nats_ca_cache"), &cryptokeys.DBFetcher{DB: options.Database}, codersdk.CryptoKeyFeatureNATSCA) + if err != nil { + options.Logger.Fatal(ctx, "failed to instantiate NATS CA cache", slog.Error(err)) } } diff --git a/coderd/cryptokeys/rotate.go b/coderd/cryptokeys/rotate.go index 7dd72ec459..c99120afb9 100644 --- a/coderd/cryptokeys/rotate.go +++ b/coderd/cryptokeys/rotate.go @@ -51,10 +51,12 @@ var defaultRotatedFeatures = []database.CryptoKeyFeature{ database.CryptoKeyFeatureOIDCConvert, database.CryptoKeyFeatureChatFilesToken, database.CryptoKeyFeatureTailnetResume, + // NATS pubsub is on by default, so its cluster mTLS CA is always rotated. + database.CryptoKeyFeatureNATSCA, } // DefaultRotatedFeatures returns the crypto key features the rotator manages by -// default. It excludes experiment-gated features such as the NATS CA. +// default. func DefaultRotatedFeatures() []database.CryptoKeyFeature { return slices.Clone(defaultRotatedFeatures) } diff --git a/coderd/cryptokeys/rotate_internal_test.go b/coderd/cryptokeys/rotate_internal_test.go index a96281990c..389b3521e7 100644 --- a/coderd/cryptokeys/rotate_internal_test.go +++ b/coderd/cryptokeys/rotate_internal_test.go @@ -160,11 +160,9 @@ func Test_rotateKeys(t *testing.T) { t.Run("IgnoresUnmanagedFeatureKeys", func(t *testing.T) { t.Parallel() - // Regression: a rotator managing a subset of features (e.g. after the - // nats_ca experiment is toggled off) must still rotate its managed - // features even when the DB holds keys for features it does not manage, - // such as nats_ca rows left over from a prior experiment-on run. - // Previously such rows aborted every rotation. + // Regression: a rotator managing a subset of features must still rotate + // its managed features even when the DB holds keys for features it does + // not manage. Previously such rows aborted every rotation. var ( db, _ = dbtestutil.NewDB(t) clock = quartz.NewMock(t) @@ -513,7 +511,7 @@ func Test_rotateKeys(t *testing.T) { keys, err := db.GetCryptoKeys(ctx) require.NoError(t, err) - require.Len(t, keys, 6) + require.Len(t, keys, 7) kbf := keysByFeature(keys, defaultRotatedFeatures) @@ -526,15 +524,19 @@ func Test_rotateKeys(t *testing.T) { require.Len(t, kbf[database.CryptoKeyFeatureTailnetResume], 1) require.Len(t, kbf[database.CryptoKeyFeatureWorkspaceAppsToken], 1) require.Len(t, kbf[database.CryptoKeyFeatureChatFilesToken], 1) + // No existing NATS CA key should've caused one to be inserted. + require.Len(t, kbf[database.CryptoKeyFeatureNATSCA], 1) oidcKey := kbf[database.CryptoKeyFeatureOIDCConvert][0] tailnetKey := kbf[database.CryptoKeyFeatureTailnetResume][0] appTokenKey := kbf[database.CryptoKeyFeatureWorkspaceAppsToken][0] chatFileTokenKey := kbf[database.CryptoKeyFeatureChatFilesToken][0] + natsCAKey := kbf[database.CryptoKeyFeatureNATSCA][0] requireKey(t, oidcKey, database.CryptoKeyFeatureOIDCConvert, now, nullTime, validKey.Sequence) requireKey(t, tailnetKey, database.CryptoKeyFeatureTailnetResume, now, nullTime, deletedKey.Sequence+1) requireKey(t, appTokenKey, database.CryptoKeyFeatureWorkspaceAppsToken, now, nullTime, 1) requireKey(t, chatFileTokenKey, database.CryptoKeyFeatureChatFilesToken, now, nullTime, 1) + requireKey(t, natsCAKey, database.CryptoKeyFeatureNATSCA, now, nullTime, 1) newKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][0] oldKey := kbf[database.CryptoKeyFeatureWorkspaceAppsAPIKey][1] if newKey.Sequence == rotatedKey.Sequence { diff --git a/coderd/x/nats/pubsub.go b/coderd/x/nats/pubsub.go index 96595e0add..6fbe6e2c7e 100644 --- a/coderd/x/nats/pubsub.go +++ b/coderd/x/nats/pubsub.go @@ -357,7 +357,7 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub, } }() - logger.Info(context.Background(), "embedded nats server started", + logger.Debug(context.Background(), "embedded nats server started", slog.F("client_url", ns.ClientURL()), ) diff --git a/codersdk/deployment.go b/codersdk/deployment.go index 5094b6f0ef..c91e52766d 100644 --- a/codersdk/deployment.go +++ b/codersdk/deployment.go @@ -5238,7 +5238,7 @@ const ( ExperimentMCPServerHTTP Experiment = "mcp-server-http" // Enables the MCP HTTP server functionality. ExperimentMCPToolSearch Experiment = "mcp-tool-search" // Defers MCP tool schemas behind a searchable catalog in agent chats. ExperimentWorkspaceBuildUpdates Experiment = "workspace-build-updates" // Enables publishing workspace build updates to the all builds pubsub channel. - ExperimentNATSPubsub Experiment = "nats_pubsub" // Enables embedded NATS pubsub. + ExperimentNoNATSPubsub Experiment = "no_nats_pubsub" // Disables the embedded NATS pubsub, falling back to PostgreSQL pubsub. ExperimentWorkspaceCapableLicensing Experiment = "workspace-capable-licensing" // Counts only users holding the workspace-create permission toward the license seat limit. ExperimentAIGatewaySeatExclusion Experiment = "ai-gateway-seat-exclusion" // Excludes AI Gateway (AI Bridge) usage from AI Governance seat consumption. ExperimentAIGatewayReverseProxy Experiment = "ai-gateway-reverse-proxy" // Uses stateless reverse proxy routing when MCP injection is not configured. @@ -5261,8 +5261,8 @@ func (e Experiment) DisplayName() string { return "MCP HTTP Server Functionality" case ExperimentWorkspaceBuildUpdates: return "Workspace Build Updates Channel" - case ExperimentNATSPubsub: - return "NATS Pubsub" + case ExperimentNoNATSPubsub: + return "No NATS Pubsub" case ExperimentWorkspaceCapableLicensing: return "Workspace-Capable Licensing" case ExperimentAIGatewaySeatExclusion: @@ -5291,7 +5291,7 @@ var ExperimentsKnown = Experiments{ ExperimentWorkspaceUsage, ExperimentMCPServerHTTP, ExperimentMCPToolSearch, - ExperimentNATSPubsub, + ExperimentNoNATSPubsub, ExperimentWorkspaceBuildUpdates, ExperimentWorkspaceCapableLicensing, ExperimentAIGatewaySeatExclusion, diff --git a/docs/admin/networking/high-availability.md b/docs/admin/networking/high-availability.md index ea2f626feb..bb6dcef1ba 100644 --- a/docs/admin/networking/high-availability.md +++ b/docs/admin/networking/high-availability.md @@ -47,13 +47,22 @@ rendezvous for the Coder nodes. `CODER_DERP_SERVER_RELAY_URL` will never be `CODER_ACCESS_URL` because `CODER_ACCESS_URL` is a load balancer to all Coder nodes. +Coder also clusters its inter-node pubsub across HA nodes and needs each +replica's routable address to do so. This address comes from +`CODER_CLUSTER_HOST`, falling back to the host in `CODER_DERP_SERVER_RELAY_URL` +when it is unset. Set `CODER_CLUSTER_HOST` to each node's routable IP address. +Setting it is required when you use [custom relays](./index.md#custom-relays), +because `CODER_DERP_SERVER_RELAY_URL` is ignored and no fallback address is +available. If a node has neither value set, Coder logs an error and falls back +to PostgreSQL for pubsub. + Here's an example 3-node network configuration setup: -| Name | `CODER_HTTP_ADDRESS` | `CODER_DERP_SERVER_RELAY_URL` | `CODER_ACCESS_URL` | -|-----------|----------------------|-------------------------------|--------------------------| -| `coder-1` | `*:80` | `http://10.0.0.1:80` | `https://coder.big.corp` | -| `coder-2` | `*:80` | `http://10.0.0.2:80` | `https://coder.big.corp` | -| `coder-3` | `*:80` | `http://10.0.0.3:80` | `https://coder.big.corp` | +| Name | `CODER_HTTP_ADDRESS` | `CODER_DERP_SERVER_RELAY_URL` | `CODER_CLUSTER_HOST` | `CODER_ACCESS_URL` | +|-----------|----------------------|-------------------------------|----------------------|--------------------------| +| `coder-1` | `*:80` | `http://10.0.0.1:80` | `10.0.0.1` | `https://coder.big.corp` | +| `coder-2` | `*:80` | `http://10.0.0.2:80` | `10.0.0.2` | `https://coder.big.corp` | +| `coder-3` | `*:80` | `http://10.0.0.3:80` | `10.0.0.3` | `https://coder.big.corp` | ## Kubernetes @@ -72,6 +81,8 @@ env: fieldPath: status.podIP - name: CODER_DERP_SERVER_RELAY_URL value: http://$(POD_IP) + - name: CODER_CLUSTER_HOST + value: $(POD_IP) ``` Then, increase the number of pods. diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 8d624028da..1bd10feadd 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -8899,9 +8899,9 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o #### Enumerated Values -| Value(s) | -|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `agent-lifecycle-hooks`, `ai-gateway-reverse-proxy`, `ai-gateway-seat-exclusion`, `auto-fill-parameters`, `chat-advisor`, `chat-virtual-desktop`, `example`, `mcp-server-http`, `mcp-tool-search`, `nats_pubsub`, `notifications`, `workspace-build-updates`, `workspace-capable-licensing`, `workspace-usage` | +| Value(s) | +|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `agent-lifecycle-hooks`, `ai-gateway-reverse-proxy`, `ai-gateway-seat-exclusion`, `auto-fill-parameters`, `chat-advisor`, `chat-virtual-desktop`, `example`, `mcp-server-http`, `mcp-tool-search`, `no_nats_pubsub`, `notifications`, `workspace-build-updates`, `workspace-capable-licensing`, `workspace-usage` | ## codersdk.ExternalAPIKeyScopes diff --git a/enterprise/coderd/coderd_test.go b/enterprise/coderd/coderd_test.go index c2c7a5974f..1e3aed3b5b 100644 --- a/enterprise/coderd/coderd_test.go +++ b/enterprise/coderd/coderd_test.go @@ -647,7 +647,6 @@ func TestMultiReplica_NATSPubsubPeers(t *testing.T) { t.Cleanup(func() { _ = natsA.Close() }) dv := coderdtest.DeploymentValues(t) - dv.Experiments = []string{string(codersdk.ExperimentNATSPubsub)} _, _ = coderdenttest.New(t, &coderdenttest.Options{ EntitlementsUpdateInterval: 25 * time.Millisecond, ReplicaSyncUpdateInterval: 25 * time.Millisecond, diff --git a/enterprise/coderd/licenses.go b/enterprise/coderd/licenses.go index 38549c2511..bbe9393167 100644 --- a/enterprise/coderd/licenses.go +++ b/enterprise/coderd/licenses.go @@ -41,7 +41,7 @@ const ( // re-read licenses from the database and recompute entitlements. // // It is published and subscribed on api.ReplicaSyncPubsub (always Postgres), - // not api.Pubsub. When the NATS pubsub experiment is enabled, api.Pubsub is + // not api.Pubsub. When the embedded NATS pubsub is in use, api.Pubsub is // the embedded NATS pubsub whose cluster mesh only forms once a replica is // HA-licensed, so propagating license changes over it is circular: a fresh // replica could not learn about the license that would let it join the mesh. diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index fb214a077c..e81ee52ad3 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -5028,7 +5028,7 @@ export type Experiment = | "example" | "mcp-server-http" | "mcp-tool-search" - | "nats_pubsub" + | "no_nats_pubsub" | "notifications" | "workspace-build-updates" | "workspace-capable-licensing" @@ -5044,7 +5044,7 @@ export const Experiments: Experiment[] = [ "example", "mcp-server-http", "mcp-tool-search", - "nats_pubsub", + "no_nats_pubsub", "notifications", "workspace-build-updates", "workspace-capable-licensing",