feat: NATS mTLS pubsub implementation (#26902)

coderd/apidoc/docs.go

@@ -19063,13 +19063,15 @@ const docTemplate = `{
                 "workspace_apps_api_key",
                 "workspace_apps_token",
                 "oidc_convert",
-                "tailnet_resume"
+                "tailnet_resume",
+                "nats_ca"
             ],
             "x-enum-varnames": [
                 "CryptoKeyFeatureWorkspaceAppsAPIKey",
                 "CryptoKeyFeatureWorkspaceAppsToken",
                 "CryptoKeyFeatureOIDCConvert",
-                "CryptoKeyFeatureTailnetResume"
+                "CryptoKeyFeatureTailnetResume",
+                "CryptoKeyFeatureNATSCA"
             ]
         },
         "codersdk.CustomNotificationContent": {

coderd/apidoc/swagger.json

@@ -17252,13 +17252,15 @@
 				"workspace_apps_api_key",
 				"workspace_apps_token",
 				"oidc_convert",
-				"tailnet_resume"
+				"tailnet_resume",
+				"nats_ca"
 			],
 			"x-enum-varnames": [
 				"CryptoKeyFeatureWorkspaceAppsAPIKey",
 				"CryptoKeyFeatureWorkspaceAppsToken",
 				"CryptoKeyFeatureOIDCConvert",
-				"CryptoKeyFeatureTailnetResume"
+				"CryptoKeyFeatureTailnetResume",
+				"CryptoKeyFeatureNATSCA"
 			]
 		},
 		"codersdk.CustomNotificationContent": {

coderd/coderd.go

@@ -206,6 +206,13 @@ type Options struct {
 	TLSCertificates    []tls.Certificate
 	TailnetCoordinator tailnet.Coordinator
 	DERPServer         *derp.Server
+	// ClusterHost is this replica's routable cluster address (IP or hostname),
+	// resolved from DeploymentValues.Cluster.Host, falling back to the DERP
+	// relay host for older HA deployments that predate the setting. It is used
+	// as the NATS cluster route host and, when it is an IP, the cluster mTLS
+	// leaf IP SAN. It is consumed by the NATS pubsub (AGPL) and, under
+	// enterprise HA, by replicasync.
+	ClusterHost string
 	// BaseDERPMap is used as the base DERP map for all clients and agents.
 	// Proxies are added to this list.
 	BaseDERPMap                    *tailcfg.DERPMap
@@ -306,7 +313,12 @@ type Options struct {
 	AppSigningKeyCache    cryptokeys.SigningKeycache
 	AppEncryptionKeyCache cryptokeys.EncryptionKeycache
 	OIDCConvertKeyCache   cryptokeys.SigningKeycache
-	Clock                 quartz.Clock
+	// NATSCACache serves the NATS cluster mTLS CA via the generic signing key
+	// cache for the nats_ca feature. SigningKey returns the active CA
+	// (a *NATSCA); VerifyingKey returns a specific CA by sequence. The key
+	// rotator is the sole creator of nats_ca rows, so this cache is read-only.
+	NATSCACache cryptokeys.SigningKeycache
+	Clock       quartz.Clock
 
 	// WebPushDispatcher is a way to send notifications over Web Push.
 	WebPushDispatcher webpush.Dispatcher
@@ -608,10 +620,34 @@ func New(options *Options) *API {
 
 	updatesProvider := NewUpdatesProvider(options.Logger.Named("workspace_updates"), options.Pubsub, options.Database, options.Authorizer)
 
+	// The NATS cluster CA is only minted and served when NATS pubsub is in use.
+	// It is experiment-gated, so it is opted into rotation and backed by a real
+	// signing cache only when the experiment is enabled; otherwise the rotator
+	// leaves it alone and the cache is a noop, which still answers requests (the
+	// pubsub treats a missing CA as "mTLS off"). This avoids minting CA private
+	// keys on deployments that never run NATS clustering.
+	rotatedFeatures := cryptokeys.DefaultRotatedFeatures()
+	if experiments.Enabled(codersdk.ExperimentNATSPubsub) {
+		rotatedFeatures = append(rotatedFeatures, database.CryptoKeyFeatureNATSCA)
+	}
+
 	// Start a background process that rotates keys. We intentionally start this after the caches
 	// are created to force initial requests for a key to populate the caches. This helps catch
 	// bugs that may only occur when a key isn't precached in tests and the latency cost is minimal.
-	cryptokeys.StartRotator(ctx, options.Logger, options.Database)
+	cryptokeys.StartRotator(ctx, options.Logger, options.Database, cryptokeys.WithFeatures(rotatedFeatures))
+
+	// The NATS CA cache is read-only and depends on the rotator having minted
+	// the nats_ca CA, so it must be constructed after StartRotator.
+	if options.NATSCACache == nil {
+		if experiments.Enabled(codersdk.ExperimentNATSPubsub) {
+			options.NATSCACache, err = cryptokeys.NewSigningCache(ctx, options.Logger.Named("nats_ca_cache"), &cryptokeys.DBFetcher{DB: options.Database}, codersdk.CryptoKeyFeatureNATSCA)
+			if err != nil {
+				options.Logger.Fatal(ctx, "failed to instantiate NATS CA cache", slog.Error(err))
+			}
+		} else {
+			options.NATSCACache = cryptokeys.NoopSigningKeycache{}
+		}
+	}
 
 	// Ensure all system role permissions are current.
 	//nolint:gocritic // Startup reconciliation reads/writes system roles. There is
@@ -2413,6 +2449,9 @@ func (api *API) Close() error {
 	_ = api.OIDCConvertKeyCache.Close()
 	_ = api.AppSigningKeyCache.Close()
 	_ = api.AppEncryptionKeyCache.Close()
+	if api.NATSCACache != nil {
+		_ = api.NATSCACache.Close()
+	}
 	_ = api.UpdatesProvider.Close()
 	api.workspaceAgentConnWatcher.Close()
 	api.workspaceBuildOrchestrator.Close()

coderd/cryptokeys/ca.go

@@ -0,0 +1,149 @@
+package cryptokeys
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"time"
+
+	"golang.org/x/xerrors"
+)
+
+const (
+	caCertPEMBlockType = "CERTIFICATE"
+	caKeyPEMBlockType  = "EC PRIVATE KEY"
+
+	// clockSkewTolerance backdates the CA certificate's NotBefore and extends
+	// its NotAfter so that replicas with mildly skewed clocks still accept it.
+	clockSkewTolerance = time.Hour
+)
+
+// NATSCA is the decoded form of a single nats_ca crypto key row, produced by
+// the generic crypto key cache (see idSecret). The CA signs the ephemeral leaf
+// certificates that replicas use for NATS cluster mTLS.
+//
+// The active CA is served by a SigningKeycache.SigningKey call for the nats_ca
+// feature; a specific historical CA (for verifying a peer leaf minted under an
+// earlier CA during a rotation overlap) is served by VerifyingKey with that
+// row's sequence.
+type NATSCA struct {
+	// Sequence is the crypto_keys sequence of the row this CA came from.
+	Sequence int32
+	// Cert is the CA certificate used to sign or verify leaf certificates.
+	Cert *x509.Certificate
+	// Key is the CA private key, used to sign leaves.
+	Key crypto.Signer
+}
+
+// generateCASecret generates a new self-signed CA certificate and private key
+// for signing NATS cluster leaf certificates, PEM-encoded into a single
+// bundle for storage in the crypto_keys secret column.
+//
+// anchorTime is the key row's starts_at (which may be in the future for a
+// rotated-in key). keyDuration is the rotator's key duration: the row stays the
+// active signer for that long. The certificate stays valid for NATSCAOverlap
+// past that window so that, once the next CA becomes the active signer, this CA
+// is still valid while replicas' key caches refresh onto the new one. Leaves
+// are separately clamped to expire before this NotAfter (see coderd/x/nats
+// mintLeaf), so the overlap only needs to cover the cache-refresh transition.
+func generateCASecret(anchorTime time.Time, keyDuration time.Duration) (string, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return "", xerrors.Errorf("generate key: %w", err)
+	}
+
+	// 128-bit random serial per CA/Browser Forum conventions.
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return "", xerrors.Errorf("generate serial: %w", err)
+	}
+
+	template := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName: "coder-nats-ca",
+		},
+		NotBefore:             anchorTime.Add(-clockSkewTolerance),
+		NotAfter:              anchorTime.Add(keyDuration + NATSCAOverlap),
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLenZero:        true,
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	if err != nil {
+		return "", xerrors.Errorf("create certificate: %w", err)
+	}
+
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		return "", xerrors.Errorf("marshal private key: %w", err)
+	}
+
+	var secret []byte
+	secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caCertPEMBlockType, Bytes: der})...)
+	secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caKeyPEMBlockType, Bytes: keyDER})...)
+	return string(secret), nil
+}
+
+// parseCASecret parses a PEM bundle produced by generateCASecret back into
+// the CA certificate and private key.
+func parseCASecret(secret string) (*x509.Certificate, crypto.Signer, error) {
+	var (
+		cert *x509.Certificate
+		key  *ecdsa.PrivateKey
+	)
+	rest := []byte(secret)
+	for {
+		var block *pem.Block
+		block, rest = pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		switch block.Type {
+		case caCertPEMBlockType:
+			if cert != nil {
+				return nil, nil, xerrors.New("multiple certificates in CA secret")
+			}
+			var err error
+			cert, err = x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("parse certificate: %w", err)
+			}
+		case caKeyPEMBlockType:
+			if key != nil {
+				return nil, nil, xerrors.New("multiple private keys in CA secret")
+			}
+			var err error
+			key, err = x509.ParseECPrivateKey(block.Bytes)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("parse private key: %w", err)
+			}
+		default:
+			return nil, nil, xerrors.Errorf("unexpected PEM block type: %q", block.Type)
+		}
+	}
+	if cert == nil {
+		return nil, nil, xerrors.New("no certificate in CA secret")
+	}
+	if key == nil {
+		return nil, nil, xerrors.New("no private key in CA secret")
+	}
+	if !key.PublicKey.Equal(cert.PublicKey) {
+		return nil, nil, xerrors.New("private key does not match certificate")
+	}
+	// Reject a structurally valid bundle whose certificate cannot act as a
+	// signing CA. Without this, a corrupted secret could yield a non-CA cert
+	// that silently becomes the active signer; leaves signed under it would
+	// then fail x509 verification on every replica.
+	if !cert.IsCA || !cert.BasicConstraintsValid || cert.KeyUsage&x509.KeyUsageCertSign == 0 {
+		return nil, nil, xerrors.New("certificate is not a valid signing CA")
+	}
+	return cert, key, nil
+}

coderd/cryptokeys/ca_internal_test.go

@@ -0,0 +1,251 @@
+package cryptokeys
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+)
+
+func TestCASecretRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	// The certificate's NotAfter must track the supplied keyDuration, not a
+	// hardcoded default, so a CA stays valid for as long as it can be the
+	// active signer plus the longest leaf it signs.
+	for _, keyDuration := range []time.Duration{DefaultKeyDuration, DefaultKeyDuration * 3, time.Hour} {
+		now := time.Now().UTC().Truncate(time.Second)
+		secret, err := generateCASecret(now, keyDuration)
+		require.NoError(t, err)
+
+		cert, signer, err := parseCASecret(secret)
+		require.NoError(t, err)
+
+		require.True(t, cert.IsCA)
+		require.True(t, cert.BasicConstraintsValid)
+		require.True(t, cert.MaxPathLenZero)
+		require.Equal(t, x509.KeyUsageCertSign, cert.KeyUsage)
+		require.Equal(t, now.Add(-clockSkewTolerance), cert.NotBefore)
+		require.Equal(t, now.Add(keyDuration+NATSCAOverlap), cert.NotAfter)
+		require.Equal(t, cert.PublicKey, signer.Public())
+
+		// The cert must outlive its active-signer window so leaves signed at
+		// the end of that window still chain to a valid CA.
+		require.True(t, cert.NotAfter.After(now.Add(keyDuration)),
+			"cert must remain valid past the end of its active-signer window")
+
+		// The cert must be able to verify itself as a trust root.
+		pool := x509.NewCertPool()
+		pool.AddCert(cert)
+		_, err = cert.Verify(x509.VerifyOptions{Roots: pool})
+		require.NoError(t, err)
+	}
+}
+
+func TestParseCASecretErrors(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	secretA, err := generateCASecret(now, DefaultKeyDuration)
+	require.NoError(t, err)
+	secretB, err := generateCASecret(now, DefaultKeyDuration)
+	require.NoError(t, err)
+
+	certA, keyA := splitCAPEM(t, secretA)
+	_, keyB := splitCAPEM(t, secretB)
+
+	nonCACert, nonCAKey := generateNonCAPEM(t, now)
+
+	cases := []struct {
+		name    string
+		secret  string
+		errText string
+	}{
+		{"Empty", "", "no certificate"},
+		{"NotPEM", "not pem at all", "no certificate"},
+		{"CertOnly", string(certA), "no private key"},
+		{"KeyCertMismatch", string(certA) + string(keyB), "does not match certificate"},
+		{"MultipleCertificates", string(certA) + string(certA) + string(keyA), "multiple certificates"},
+		{"MultiplePrivateKeys", string(certA) + string(keyA) + string(keyA), "multiple private keys"},
+		{"UnexpectedBlockType", string(pemBlock("RSA PRIVATE KEY", []byte("x"))), "unexpected PEM block type"},
+		{"BadCertificateBytes", string(pemBlock(caCertPEMBlockType, []byte("garbage"))), "parse certificate"},
+		{"BadPrivateKeyBytes", string(certA) + string(pemBlock(caKeyPEMBlockType, []byte("garbage"))), "parse private key"},
+		{"NotASigningCA", string(nonCACert) + string(nonCAKey), "not a valid signing CA"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, _, err := parseCASecret(tc.secret)
+			require.ErrorContains(t, err, tc.errText)
+		})
+	}
+}
+
+// splitCAPEM splits a CA secret bundle into its certificate and private key
+// PEM blocks so tests can recombine them into malformed bundles.
+func splitCAPEM(t *testing.T, secret string) (certPEM, keyPEM []byte) {
+	t.Helper()
+	rest := []byte(secret)
+	for {
+		block, r := pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		rest = r
+		switch block.Type {
+		case caCertPEMBlockType:
+			certPEM = pem.EncodeToMemory(block)
+		case caKeyPEMBlockType:
+			keyPEM = pem.EncodeToMemory(block)
+		}
+	}
+	require.NotNil(t, certPEM)
+	require.NotNil(t, keyPEM)
+	return certPEM, keyPEM
+}
+
+func pemBlock(blockType string, der []byte) []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der})
+}
+
+// generateNonCAPEM produces a structurally valid cert+key bundle whose
+// certificate is not a CA (no IsCA, no KeyUsageCertSign). The key matches the
+// cert, so it passes every parseCASecret check except the signing-CA check.
+func generateNonCAPEM(t *testing.T, now time.Time) (certPEM, keyPEM []byte) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "not-a-ca"},
+		NotBefore:    now.Add(-time.Hour),
+		NotAfter:     now.Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	require.NoError(t, err)
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	return pemBlock(caCertPEMBlockType, der), pemBlock(caKeyPEMBlockType, keyDER)
+}
+
+// TestNATSCASigningCache exercises the nats_ca feature through the generic
+// signing key cache: the PEM secret decodes into a *NATSCA, SigningKey serves
+// the active CA, VerifyingKey serves a specific CA by sequence, and a rotation
+// is picked up on the next refresh.
+func TestNATSCASigningCache(t *testing.T) {
+	t.Parallel()
+
+	t.Run("ActiveAndVerifyingByID", func(t *testing.T) {
+		t.Parallel()
+
+		db, _ := dbtestutil.NewDB(t)
+		ctx := testutil.Context(t, testutil.WaitShort)
+		now := time.Now().UTC()
+
+		current := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: 1,
+			StartsAt: now.Add(-time.Hour),
+		})
+
+		cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA)
+		require.NoError(t, err)
+		defer cache.Close()
+
+		id, key, err := cache.SigningKey(ctx)
+		require.NoError(t, err)
+
+		ca, ok := key.(*NATSCA)
+		require.True(t, ok, "signing key should decode to *NATSCA, got %T", key)
+		require.Equal(t, current.Sequence, ca.Sequence)
+		require.NotNil(t, ca.Cert)
+		require.NotNil(t, ca.Key)
+
+		currentCert, _, err := parseCASecret(current.Secret.String)
+		require.NoError(t, err)
+		require.Equal(t, currentCert.Raw, ca.Cert.Raw)
+
+		// VerifyingKey looks the CA up by the sequence embedded in id, which is
+		// how a peer leaf minted under this CA is verified.
+		verifying, err := cache.VerifyingKey(ctx, id)
+		require.NoError(t, err)
+		vca, ok := verifying.(*NATSCA)
+		require.True(t, ok, "verifying key should decode to *NATSCA, got %T", verifying)
+		require.Equal(t, currentCert.Raw, vca.Cert.Raw)
+	})
+
+	t.Run("RefreshesOnRotation", func(t *testing.T) {
+		t.Parallel()
+
+		db, _ := dbtestutil.NewDB(t)
+		ctx := testutil.Context(t, testutil.WaitShort)
+		clock := quartz.NewMock(t)
+		now := dbtime.Now()
+		clock.Set(now)
+
+		first := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: 1,
+			StartsAt: now.Add(-time.Hour),
+		})
+
+		cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA, WithCacheClock(clock))
+		require.NoError(t, err)
+		defer cache.Close()
+
+		_, key, err := cache.SigningKey(ctx)
+		require.NoError(t, err)
+		require.Equal(t, first.Sequence, key.(*NATSCA).Sequence)
+
+		// Simulate a rotation by inserting a higher-sequence active CA. The old
+		// CA stays valid for verification by its sequence.
+		second := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: 2,
+			StartsAt: now.Add(-time.Minute),
+		})
+
+		// Fire the background refresher; the active CA advances to the new row.
+		clock.Advance(refreshInterval).MustWait(ctx)
+
+		_, key, err = cache.SigningKey(ctx)
+		require.NoError(t, err)
+		require.Equal(t, second.Sequence, key.(*NATSCA).Sequence)
+
+		oldVerifying, err := cache.VerifyingKey(ctx, "1")
+		require.NoError(t, err)
+		require.Equal(t, first.Sequence, oldVerifying.(*NATSCA).Sequence)
+	})
+}
+
+func TestNoopSigningKeycache(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	var cache SigningKeycache = NoopSigningKeycache{}
+
+	_, _, err := cache.SigningKey(ctx)
+	require.ErrorIs(t, err, ErrKeyNotFound)
+
+	_, err = cache.VerifyingKey(ctx, "1")
+	require.ErrorIs(t, err, ErrKeyNotFound)
+
+	require.NoError(t, cache.Close())
+}

coderd/cryptokeys/cache.go

@@ -55,6 +55,26 @@ type SigningKeycache interface {
 	io.Closer
 }
 
+// NoopSigningKeycache is a SigningKeycache that holds no keys: SigningKey and
+// VerifyingKey always report ErrKeyNotFound. It lets a subsystem that only
+// needs real keys once an optional feature is enabled (for example NATS
+// cluster mTLS, which only signs leaves under enterprise HA) be constructed
+// without a database dependency, then be swapped for a real cache when the
+// feature turns on.
+type NoopSigningKeycache struct{}
+
+var _ SigningKeycache = NoopSigningKeycache{}
+
+func (NoopSigningKeycache) SigningKey(context.Context) (string, interface{}, error) {
+	return "", nil, ErrKeyNotFound
+}
+
+func (NoopSigningKeycache) VerifyingKey(context.Context, string) (interface{}, error) {
+	return nil, ErrKeyNotFound
+}
+
+func (NoopSigningKeycache) Close() error { return nil }
+
 const (
 	// latestSequence is a special sequence number that represents the latest key.
 	latestSequence = -1
@@ -213,23 +233,42 @@ func isEncryptionKeyFeature(feature codersdk.CryptoKeyFeature) bool {
 
 func isSigningKeyFeature(feature codersdk.CryptoKeyFeature) bool {
 	switch feature {
-	case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken:
+	case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken, codersdk.CryptoKeyFeatureNATSCA:
 		return true
 	default:
 		return false
 	}
 }
 
-func idSecret(k codersdk.CryptoKey) (string, []byte, error) {
+// idSecret materializes a stored crypto key into the in-memory key object the
+// feature uses, returning it as an interface{} alongside the key's id (its
+// sequence as a decimal string). Most features hex-decode the secret into raw
+// bytes, but nats_ca stores a PEM cert+key bundle and decodes into a *NATSCA.
+//
+// TODO: this hard-coded switch on feature is the simplest way to support a
+// second secret encoding, but it couples this generic cache to nats_ca
+// specifics. Explore abstracting the decode step (for example a per-feature
+// decoder injected at construction) so new key types can be added without
+// editing this function.
+func idSecret(k codersdk.CryptoKey) (string, interface{}, error) {
+	id := strconv.FormatInt(int64(k.Sequence), 10)
+
+	if k.Feature == codersdk.CryptoKeyFeatureNATSCA {
+		cert, signer, err := parseCASecret(k.Secret)
+		if err != nil {
+			return "", nil, xerrors.Errorf("decode nats_ca key: %w", err)
+		}
+		return id, &NATSCA{Sequence: k.Sequence, Cert: cert, Key: signer}, nil
+	}
+
 	key, err := hex.DecodeString(k.Secret)
 	if err != nil {
 		return "", nil, xerrors.Errorf("decode key: %w", err)
 	}
-
-	return strconv.FormatInt(int64(k.Sequence), 10), key, nil
+	return id, key, nil
 }
 
-func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, []byte, error) {
+func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, interface{}, error) {
 	c.logger.Debug(ctx, "request for key", slog.F("sequence", sequence))
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -284,7 +323,7 @@ func (c *cache) key(sequence int32) (codersdk.CryptoKey, bool) {
 	return key, ok
 }
 
-func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, []byte, error) {
+func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, interface{}, error) {
 	if sequence == latestSequence {
 		if !key.CanSign(now) {
 			return "", nil, ErrKeyInvalid

coderd/cryptokeys/rotate.go

@@ -21,6 +21,15 @@ const (
 	WorkspaceAppsTokenDuration = time.Minute
 	OIDCConvertTokenDuration   = time.Minute * 5
 	TailnetResumeTokenDuration = time.Hour * 24
+	// NATSCAOverlap is how long a NATS cluster CA certificate stays valid past
+	// the end of its active-signing window (startsAt + keyDuration). The next CA
+	// becomes the active signer at the window's end, but replicas keep minting
+	// leaves with the old CA until their key cache refreshes onto the new one.
+	// This overlap keeps the old CA valid through that transition, so it must
+	// exceed the cache refresh interval (plus a small leaf clamp buffer). Leaf
+	// lifetime imposes nothing here: leaves are clamped to just before their
+	// signing CA's NotAfter (see coderd/x/nats mintLeaf).
+	NATSCAOverlap = time.Minute * 30
 
 	// defaultRotationInterval is the default interval at which keys are checked for rotation.
 	defaultRotationInterval = time.Minute * 10
@@ -70,6 +79,15 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption {
 	}
 }
 
+// WithFeatures sets the crypto key features the rotator manages, replacing the
+// default set. Use this to opt experiment- or deployment-gated features (such
+// as the NATS cluster CA) into rotation only when their owner is active.
+func WithFeatures(features []database.CryptoKeyFeature) RotatorOption {
+	return func(r *rotator) {
+		r.features = slices.Clone(features)
+	}
+}
+
 // StartRotator starts a background process that rotates keys in the database.
 // It ensures there's at least one valid key per feature prior to returning.
 // Canceling the provided context will stop the background process.
@@ -126,10 +144,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error {
 				return xerrors.Errorf("get keys: %w", err)
 			}
 
-			featureKeys, err := keysByFeature(cryptokeys, k.features)
-			if err != nil {
-				return xerrors.Errorf("keys by feature: %w", err)
-			}
+			featureKeys := keysByFeature(cryptokeys, k.features)
 
 			now := dbtime.Time(k.clock.Now().UTC())
 			for feature, keys := range featureKeys {
@@ -189,7 +204,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error {
 }
 
 func (k *rotator) insertNewKey(ctx context.Context, tx database.Store, feature database.CryptoKeyFeature, startsAt time.Time) (database.CryptoKey, error) {
-	secret, err := generateNewSecret(feature)
+	secret, err := generateNewSecret(feature, startsAt, k.keyDuration)
 	if err != nil {
 		return database.CryptoKey{}, xerrors.Errorf("generate new secret: %w", err)
 	}
@@ -246,7 +261,11 @@ func (k *rotator) rotateKey(ctx context.Context, tx database.Store, key database
 	return []database.CryptoKey{updatedKey, newKey}, nil
 }
 
-func generateNewSecret(feature database.CryptoKeyFeature) (string, error) {
+// generateNewSecret generates the secret for a new key of the given feature.
+// keyDuration is the rotator's key duration; it is only used by features whose
+// secret encodes its own validity window (currently only the NATS CA, whose
+// certificate must outlive the key row's active-signer period).
+func generateNewSecret(feature database.CryptoKeyFeature, startsAt time.Time, keyDuration time.Duration) (string, error) {
 	switch feature {
 	case database.CryptoKeyFeatureWorkspaceAppsAPIKey:
 		return generateKey(32)
@@ -256,6 +275,8 @@ func generateNewSecret(feature database.CryptoKeyFeature) (string, error) {
 		return generateKey(64)
 	case database.CryptoKeyFeatureTailnetResume:
 		return generateKey(64)
+	case database.CryptoKeyFeatureNATSCA:
+		return generateCASecret(startsAt, keyDuration)
 	}
 	return "", xerrors.Errorf("unknown feature: %s", feature)
 }
@@ -279,6 +300,11 @@ func tokenDuration(feature database.CryptoKeyFeature) time.Duration {
 		return OIDCConvertTokenDuration
 	case database.CryptoKeyFeatureTailnetResume:
 		return TailnetResumeTokenDuration
+	case database.CryptoKeyFeatureNATSCA:
+		// The old CA row only needs to outlive its own certificate, which stays
+		// valid for NATSCAOverlap past the active-signing window. Keeping the
+		// row (and thus its trust-root status) beyond cert expiry is pointless.
+		return NATSCAOverlap
 	default:
 		return 0
 	}
@@ -297,19 +323,25 @@ func shouldRotateKey(key database.CryptoKey, keyDuration time.Duration, now time
 	return !now.Add(time.Hour).UTC().Before(expirationTime)
 }
 
-func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) (map[database.CryptoKeyFeature][]database.CryptoKey, error) {
+// keysByFeature groups keys by feature, restricted to the managed feature set.
+// GetCryptoKeys returns rows for every feature, but the rotator only manages a
+// subset (features can be gated, e.g. nats_ca behind an experiment). Keys for
+// features outside the managed set belong to features this rotator is not
+// responsible for and are skipped, so their presence (for example nats_ca rows
+// left over from a prior experiment-on run) does not abort rotation of the
+// managed features.
+func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) map[database.CryptoKeyFeature][]database.CryptoKey {
 	m := map[database.CryptoKeyFeature][]database.CryptoKey{}
 	for _, feature := range features {
 		m[feature] = []database.CryptoKey{}
 	}
 	for _, key := range keys {
 		if _, ok := m[key.Feature]; !ok {
-			return nil, xerrors.Errorf("unknown feature: %s", key.Feature)
+			continue
 		}
-
 		m[key.Feature] = append(m[key.Feature], key)
 	}
-	return m, nil
+	return m
 }
 
 // minStartsAt ensures the minimum starts_at time we use for a new

coderd/cryptokeys/rotate_internal_test.go

@@ -104,6 +104,112 @@ func Test_rotateKeys(t *testing.T) {
 		require.Equal(t, newKey, keys[0])
 	})
 
+	t.Run("RotatesNATSCA", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			db, _       = dbtestutil.NewDB(t)
+			clock       = quartz.NewMock(t)
+			keyDuration = time.Hour * 24 * 7
+			logger      = testutil.Logger(t)
+			ctx         = testutil.Context(t, testutil.WaitShort)
+		)
+
+		kr := &rotator{
+			db:          db,
+			keyDuration: keyDuration,
+			clock:       clock,
+			logger:      logger,
+			features: []database.CryptoKeyFeature{
+				database.CryptoKeyFeatureNATSCA,
+			},
+		}
+
+		now := dbnow(clock)
+
+		oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			StartsAt: now,
+			Sequence: 4,
+		})
+
+		// Advance the window to just inside rotation time.
+		_ = clock.Advance(keyDuration - time.Minute*59)
+		err := kr.rotateKeys(ctx)
+		require.NoError(t, err)
+
+		// The old CA row is retained roughly as long as its certificate is
+		// valid: NATSCAOverlap past the active-signing window, plus the
+		// rotator's standard 1h propagation buffer.
+		expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(NATSCAOverlap + time.Hour)
+		oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
+			Feature:  oldKey.Feature,
+			Sequence: oldKey.Sequence,
+		})
+		require.NoError(t, err)
+		require.Equal(t, expectedDeletesAt, oldKey.DeletesAt.Time.UTC())
+
+		newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: oldKey.Sequence + 1,
+		})
+		require.NoError(t, err)
+		requireKey(t, newKey, database.CryptoKeyFeatureNATSCA, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1)
+	})
+
+	t.Run("IgnoresUnmanagedFeatureKeys", func(t *testing.T) {
+		t.Parallel()
+
+		// Regression: a rotator managing a subset of features (e.g. after the
+		// nats_ca experiment is toggled off) must still rotate its managed
+		// features even when the DB holds keys for features it does not manage,
+		// such as nats_ca rows left over from a prior experiment-on run.
+		// Previously such rows aborted every rotation.
+		var (
+			db, _       = dbtestutil.NewDB(t)
+			clock       = quartz.NewMock(t)
+			keyDuration = time.Hour * 24 * 7
+			logger      = testutil.Logger(t)
+			ctx         = testutil.Context(t, testutil.WaitShort)
+		)
+
+		kr := &rotator{
+			db:          db,
+			keyDuration: keyDuration,
+			clock:       clock,
+			logger:      logger,
+			// Manages only tailnet resume; nats_ca is intentionally not managed,
+			// mirroring the experiment being off.
+			features: []database.CryptoKeyFeature{
+				database.CryptoKeyFeatureTailnetResume,
+			},
+		}
+
+		now := dbnow(clock)
+
+		// A leftover nats_ca row the rotator does not manage.
+		_ = dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			StartsAt: now,
+			Sequence: 1,
+		})
+
+		// No managed key exists yet, so rotation must insert one for the managed
+		// feature and must not error on the unmanaged nats_ca row.
+		err := kr.rotateKeys(ctx)
+		require.NoError(t, err)
+
+		newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureTailnetResume)
+		require.NoError(t, err)
+		require.Equal(t, database.CryptoKeyFeatureTailnetResume, newKey.Feature)
+
+		// The unmanaged nats_ca row is untouched (no rotation, no delete).
+		natsKeys, err := db.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureNATSCA)
+		require.NoError(t, err)
+		require.Len(t, natsKeys, 1)
+		require.False(t, natsKeys[0].DeletesAt.Valid)
+	})
+
 	t.Run("DoesNotRotateValidKeys", func(t *testing.T) {
 		t.Parallel()
 
@@ -409,8 +515,7 @@ func Test_rotateKeys(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, keys, 5)
 
-		kbf, err := keysByFeature(keys, defaultRotatedFeatures)
-		require.NoError(t, err)
+		kbf := keysByFeature(keys, defaultRotatedFeatures)
 
 		// No actions on OIDC convert.
 		require.Len(t, kbf[database.CryptoKeyFeatureOIDCConvert], 1)
@@ -586,6 +691,14 @@ func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKey
 	require.Equal(t, deletesAt.Time.UTC(), key.DeletesAt.Time.UTC())
 	require.Equal(t, sequence, key.Sequence)
 
+	// The NATS CA secret is a PEM bundle rather than hex-encoded bytes.
+	if key.Feature == database.CryptoKeyFeatureNATSCA {
+		cert, _, err := parseCASecret(key.Secret.String)
+		require.NoError(t, err)
+		require.True(t, cert.IsCA)
+		return
+	}
+
 	secret, err := hex.DecodeString(key.Secret.String)
 	require.NoError(t, err)
 

coderd/database/dbgen/dbgen.go

@@ -2,13 +2,19 @@ package dbgen
 
 import (
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"database/sql"
 	"encoding/hex"
 	"encoding/json"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"maps"
+	"math/big"
 	"net"
 	"strings"
 	"testing"
@@ -2223,10 +2229,47 @@ func newCryptoKeySecret(feature database.CryptoKeyFeature) (string, error) {
 		return generateCryptoKey(64)
 	case database.CryptoKeyFeatureTailnetResume:
 		return generateCryptoKey(64)
+	case database.CryptoKeyFeatureNATSCA:
+		return generateCACryptoKeySecret()
 	}
 	return "", xerrors.Errorf("unknown feature: %s", feature)
 }
 
+// generateCACryptoKeySecret generates a self-signed CA certificate and private
+// key as a PEM bundle, matching the secret format that coderd/cryptokeys
+// produces for the nats_ca feature. It intentionally duplicates
+// cryptokeys.generateCASecret rather than calling it: coderd/cryptokeys's
+// internal tests import dbgen, so importing cryptokeys here would create a
+// test-build import cycle.
+func generateCACryptoKeySecret() (string, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return "", xerrors.Errorf("generate key: %w", err)
+	}
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "dbgen-ca"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(24 * time.Hour),
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLenZero:        true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	if err != nil {
+		return "", xerrors.Errorf("create certificate: %w", err)
+	}
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		return "", xerrors.Errorf("marshal private key: %w", err)
+	}
+	var secret []byte
+	secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})...)
+	secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER})...)
+	return string(secret), nil
+}
+
 func generateCryptoKey(length int) (string, error) {
 	b := make([]byte, length)
 	_, err := rand.Read(b)

coderd/database/modelmethods.go

@@ -2,7 +2,6 @@ package database
 
 import (
 	"database/sql"
-	"encoding/hex"
 	"fmt"
 	"slices"
 	"sort"
@@ -873,10 +872,6 @@ func (k CryptoKey) ExpiresAt(keyDuration time.Duration) time.Time {
 	return k.StartsAt.Add(keyDuration).UTC()
 }
 
-func (k CryptoKey) DecodeString() ([]byte, error) {
-	return hex.DecodeString(k.Secret.String)
-}
-
 func (k CryptoKey) CanSign(now time.Time) bool {
 	isAfterStart := !k.StartsAt.IsZero() && !now.Before(k.StartsAt)
 	return isAfterStart && k.CanVerify(now)

coderd/x/nats/cluster.go

@@ -11,6 +11,7 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/cryptokeys"
 )
 
 const defaultClusterTokenUsername = "coder"
@@ -49,6 +50,21 @@ func (p *Pubsub) SetPeerFetcher(fetcher PeerFetcher) {
 	p.RefreshPeers()
 }
 
+// SetCACache swaps the cluster mTLS CA cache, then triggers a peer refresh so
+// any route blocked by the previous (for example noop) cache is retried. It is
+// a no-op unless the pubsub was started with cluster TLS enabled
+// (Options.ClusterCA set, which installs the TLS callbacks). Passing a noop
+// cache reverts to no mTLS: new route handshakes can no longer mint a leaf and
+// will not form. The leaf IP SAN is fixed at construction from ClusterHost, so
+// it is not passed here. It logs the resulting mTLS state.
+func (p *Pubsub) SetCACache(ca cryptokeys.SigningKeycache) {
+	if p.clusterTLS == nil {
+		return
+	}
+	p.clusterTLS.setCACache(ca)
+	p.RefreshPeers()
+}
+
 // RefreshPeers signals the peer refresh worker to fetch and apply the latest
 // peer route addresses. Multiple pending refreshes are coalesced.
 func (p *Pubsub) RefreshPeers() {

coderd/x/nats/pubsub.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"hash/fnv"
+	"net"
 	"net/url"
 	"sync"
 	"time"
@@ -15,7 +16,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/cryptokeys"
 	"github.com/coder/coder/v2/coderd/database/pubsub"
+	"github.com/coder/quartz"
 )
 
 // DefaultServerMaxPendingBytes caps how many bytes the embedded NATS server will
@@ -121,6 +124,24 @@ type Options struct {
 	// clustered embedded NATS servers. Empty disables route auth.
 	ClusterAuthToken string
 
+	// ClusterCA enables mutual TLS on the cluster route listener. When set
+	// (and cluster mode is enabled), each replica mints an ephemeral leaf
+	// certificate from the active nats_ca CA and verifies peers against the
+	// CA fetched from this cache on each handshake. Nil keeps routes
+	// plaintext (token auth only). cryptokeys.SigningKeycache satisfies this.
+	//
+	// The leaf's IP SAN (and the accept-side source binding) is this replica's
+	// ClusterHost, so ClusterHost must be an IP for mTLS to activate.
+	ClusterCA cryptokeys.SigningKeycache
+
+	// clock overrides the cluster TLS clock, for tests.
+	clock quartz.Clock
+
+	// clusterTLSTimeout overrides the cluster route TLS handshake timeout, for
+	// tests. Zero leaves the NATS default (2s). Tests use a longer timeout
+	// because handshakes are flaky under load and in CI.
+	clusterTLSTimeout time.Duration
+
 	// PeerFetcher provides the current set of peer route addresses.
 	// RefreshPeers uses it to update the configured cluster routes.
 	PeerFetcher PeerFetcher
@@ -182,6 +203,8 @@ type Pubsub struct {
 	clustered     bool
 	serverOpts    *natsserver.Options
 	currentRoutes []*url.URL
+	// clusterTLS is non-nil when the cluster route listener runs mutual TLS.
+	clusterTLS *clusterTLS
 
 	peerFetcher PeerFetcher
 	peerRefresh chan struct{}
@@ -304,6 +327,25 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub,
 		return nil, err
 	}
 
+	// When ClusterCA is set, install the cluster TLS callbacks at boot so the
+	// route listener can negotiate mTLS. The callbacks read the CA cache on
+	// each handshake, so the default noop cache keeps routes inert (no leaf can
+	// be minted) until SetCACache swaps in a real cache. The leaf IP SAN is this
+	// replica's ClusterHost, fixed here at construction; leaf minting enforces
+	// that it is an IP. ClusterCA == nil keeps routes plaintext (token auth
+	// only).
+	var ct *clusterTLS
+	if !opts.disableCluster && opts.ClusterCA != nil {
+		selfIP := net.ParseIP(opts.ClusterHost)
+		ct = newClusterTLS(ctx, logger, opts.clock, opts.ClusterCA, selfIP)
+		sopts.Cluster.TLSConfig = ct.tlsConfig()
+		// Leave TLSTimeout unset (NATS defaults to 2s) unless a test overrides
+		// it; the default has not shown a need to change in production.
+		if opts.clusterTLSTimeout > 0 {
+			sopts.Cluster.TLSTimeout = opts.clusterTLSTimeout.Seconds()
+		}
+	}
+
 	ns, err := startEmbeddedServer(sopts)
 	if err != nil {
 		return nil, err
@@ -333,6 +375,7 @@ func New(ctx context.Context, logger slog.Logger, opts Options) (pubSub *Pubsub,
 	p.clustered = !opts.disableCluster
 	p.serverOpts = sopts.Clone()
 	p.currentRoutes = cloneRouteURLs(sopts.Routes)
+	p.clusterTLS = ct
 	handlers := p.buildConnHandlers()
 
 	publishPool, err := newConnPool(ns, opts, handlers, opts.PublishConns, "coder-pubsub-pub")

coderd/x/nats/pubsub_internal_test.go

@@ -613,13 +613,19 @@ func defaultTestOptions() Options {
 	return Options{disableCluster: true}
 }
 
+// testClusterTLSTimeout relaxes the cluster route TLS handshake timeout in
+// tests. NATS defaults to a tight 2s, which is flaky under load and in CI;
+// production keeps the default until it is shown to need changing.
+const testClusterTLSTimeout = 10 * time.Second
+
 func clusterTestOptions(t *testing.T) Options {
 	t.Helper()
 	return Options{
-		ClusterHost:      "127.0.0.1",
-		ClusterPort:      natsserver.RANDOM_PORT,
-		disableCluster:   false,
-		ClusterAuthToken: fmt.Sprintf("shared-token-%d", time.Now().UnixNano()),
+		ClusterHost:       "127.0.0.1",
+		ClusterPort:       natsserver.RANDOM_PORT,
+		disableCluster:    false,
+		ClusterAuthToken:  fmt.Sprintf("shared-token-%d", time.Now().UnixNano()),
+		clusterTLSTimeout: testClusterTLSTimeout,
 	}
 }
 

coderd/x/nats/tls.go

@@ -0,0 +1,429 @@
+package nats
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"net"
+	"slices"
+	"strconv"
+	"sync"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/cryptokeys"
+	"github.com/coder/quartz"
+)
+
+const (
+	// leafSerialBits is the entropy of a leaf certificate serial number.
+	leafSerialBits = 128
+	// clockSkewToleranceTLS backdates a leaf's NotBefore so a peer with a
+	// mildly skewed clock still accepts a freshly minted leaf.
+	clockSkewToleranceTLS = time.Hour
+)
+
+// clusterTLS builds the cluster route *tls.Config. Certificate selection and
+// peer verification are tls.Config callbacks that consult the CA cache on each
+// use, so a CA rotation is tracked without restarting or reloading the server.
+type clusterTLS struct {
+	ctx    context.Context
+	logger slog.Logger
+	clock  quartz.Clock
+
+	mu sync.Mutex
+	// ca is swapped by setCACache: the default noop cache mints no leaf (so no
+	// route forms) until the real cache is installed once cluster mTLS is
+	// enabled. ip is this replica's cluster host, fixed at construction and
+	// embedded as the leaf IP SAN.
+	ca cryptokeys.SigningKeycache
+	ip net.IP
+	// leaf is the cached leaf certificate, reused until it expires (its NotAfter
+	// equals the signing CA's) or setCACache clears it on a cache swap.
+	leaf *tls.Certificate
+	// verifyPools caches the root pool used to verify a peer leaf, keyed by the
+	// CA sequence stamped in the leaf. A CA cert is immutable for a given
+	// sequence, so the pool is built once and reused across handshakes. Expired
+	// entries are pruned on insert to bound the map across rotations.
+	verifyPools map[string]cachedVerifyPool
+}
+
+// cachedVerifyPool is a verify root pool plus the NotAfter of the CA cert it
+// holds; the entry is dropped once the clock passes notAfter.
+type cachedVerifyPool struct {
+	pool     *x509.CertPool
+	notAfter time.Time
+}
+
+func newClusterTLS(ctx context.Context, logger slog.Logger, clock quartz.Clock, ca cryptokeys.SigningKeycache, ip net.IP) *clusterTLS {
+	if clock == nil {
+		clock = quartz.NewReal()
+	}
+	return &clusterTLS{
+		ctx:    ctx,
+		logger: logger.Named("cluster_tls"),
+		clock:  clock,
+		ca:     ca,
+		ip:     ip,
+	}
+}
+
+// setCACache swaps the CA cache. Because the tls.Config callbacks read it on
+// each handshake, the swap takes effect without a server restart or route
+// reload: installing the real cache lets routes negotiate mTLS, and reverting
+// to a noop cache makes leaf minting fail so no new route can form. The leaf IP
+// SAN is fixed at construction (this replica's cluster host does not change), so
+// it is not touched here. A swap clears the cached leaf so the next handshake
+// re-mints under the new CA.
+func (t *clusterTLS) setCACache(ca cryptokeys.SigningKeycache) {
+	t.mu.Lock()
+	t.ca = ca
+	t.leaf = nil
+	// Verify pools follow the CA source: drop them so stale roots are not
+	// reused after a swap to a noop or different CA.
+	t.verifyPools = nil
+	ip := t.ip
+	t.mu.Unlock()
+
+	// Log the resulting mTLS state. A noop cache disables mTLS (no leaf can be
+	// minted); a real cache with a valid self IP enables it; a real cache
+	// without an IP cluster host leaves routes plaintext (token auth only).
+	switch {
+	case isNoopSigningCache(ca):
+		t.logger.Info(t.ctx, "nats cluster mTLS disabled")
+	case len(ip) == 0:
+		t.logger.Warn(t.ctx, "nats cluster mTLS inactive: cluster host is not an IP; cluster routes use token auth only")
+	default:
+		t.logger.Info(t.ctx, "nats cluster mTLS enabled")
+	}
+}
+
+// isNoopSigningCache reports whether ca is the no-op cache used to disable
+// cluster mTLS.
+func isNoopSigningCache(ca cryptokeys.SigningKeycache) bool {
+	_, ok := ca.(cryptokeys.NoopSigningKeycache)
+	return ok
+}
+
+// caCache returns the current CA cache under lock so callers do not hold the
+// lock across cache I/O.
+func (t *clusterTLS) caCache() cryptokeys.SigningKeycache {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	return t.ca
+}
+
+// tlsConfig returns the *tls.Config for the embedded server's cluster route
+// listener. The same config is used by NATS for both accepting inbound routes
+// (TLS server) and soliciting outbound routes (TLS client), so it sets both
+// GetCertificate and GetClientCertificate.
+//
+// Verification is done in VerifyConnection against the CA fetched fresh from
+// the cache, not against a static RootCAs/ClientCAs pool that cannot follow a
+// rotating CA. InsecureSkipVerify disables Go's default static-root check on
+// the dialing side ONLY so verifyConnection can run instead; it does not make
+// the connection unauthenticated. Every connection is still mutually verified
+// (ClientAuth requires a peer certificate) against live CA material.
+//
+// GetConfigForClient runs only when accepting a route (TLS server side), where
+// the dialing peer's source IP is available on the underlying connection. It
+// returns a per-connection config whose VerifyConnection additionally requires
+// the peer leaf's IP SAN to match that source IP, binding the certificate to
+// the network origin. The dialing side has no equivalent hook (Go does not
+// expose the connection in client-certificate callbacks), so it relies on the
+// base VerifyConnection: chain + membership against the known peer set.
+func (t *clusterTLS) tlsConfig() *tls.Config {
+	return &tls.Config{
+		MinVersion: tls.VersionTLS13,
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			leaf, err := t.currentLeaf()
+			if err != nil {
+				t.logger.Warn(t.ctx, "get nats cluster leaf for GetCertificate", slog.Error(err))
+			}
+			return leaf, err
+		},
+		GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			leaf, err := t.currentLeaf()
+			if err != nil {
+				t.logger.Warn(t.ctx, "get nats cluster leaf for GetClientCertificate", slog.Error(err))
+			}
+			return leaf, err
+		},
+		ClientAuth: tls.RequireAnyClientCert,
+		//nolint:gosec // Not insecure: verify performs full chain verification
+		// against the live CA cache. Go's static RootCAs cannot track a rotating
+		// CA, so default verification is replaced, not removed.
+		InsecureSkipVerify: true,
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			err := t.verify(cs, nil)
+			if err != nil {
+				t.logger.Warn(t.ctx, "verify nats cluster peer for VerifyConnection", slog.Error(err))
+			}
+			return err
+		},
+		GetConfigForClient: t.configForClient,
+	}
+}
+
+// configForClient builds the per-connection config used when accepting a route.
+// It captures the dialing peer's source IP from the underlying connection so
+// VerifyConnection can require the peer leaf's IP SAN to match it. NATS calls
+// this on each inbound handshake, so a fresh config is allocated per accepted
+// connection; that is fine at cluster-route cardinality (a handful of peers).
+func (t *clusterTLS) configForClient(chi *tls.ClientHelloInfo) (*tls.Config, error) {
+	// The accept side must bind the peer leaf to the address it connected from,
+	// so a source IP is required. Fail closed if it cannot be determined rather
+	// than silently skipping the binding in verify.
+	sourceIP, err := clientSourceIP(chi)
+	if err != nil {
+		t.logger.Warn(t.ctx, "reject nats cluster route: no source IP", slog.Error(err))
+		return nil, err
+	}
+
+	cfg := &tls.Config{
+		MinVersion: tls.VersionTLS13,
+		GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+			leaf, err := t.currentLeaf()
+			if err != nil {
+				t.logger.Warn(t.ctx, "get nats cluster leaf for GetCertificate", slog.Error(err))
+			}
+			return leaf, err
+		},
+		ClientAuth: tls.RequireAnyClientCert,
+		//nolint:gosec // See tlsConfig: verification is performed in VerifyConnection.
+		InsecureSkipVerify: true,
+		VerifyConnection: func(cs tls.ConnectionState) error {
+			err := t.verify(cs, sourceIP)
+			if err != nil {
+				t.logger.Warn(t.ctx, "verify nats cluster peer for VerifyConnection", slog.Error(err))
+			}
+			return err
+		},
+	}
+	return cfg, nil
+}
+
+// clientSourceIP extracts the dialing peer's source IP from the accepted
+// connection. The accept side requires it, so every failure is an error rather
+// than a nil that would bypass source binding in verify.
+func clientSourceIP(chi *tls.ClientHelloInfo) (net.IP, error) {
+	if chi.Conn == nil {
+		return nil, xerrors.New("no underlying connection")
+	}
+	remote := chi.Conn.RemoteAddr()
+	if remote == nil {
+		return nil, xerrors.New("no remote address")
+	}
+	host, _, err := net.SplitHostPort(remote.String())
+	if err != nil {
+		return nil, xerrors.Errorf("split remote address %q: %w", remote.String(), err)
+	}
+	ip := net.ParseIP(host)
+	if ip == nil {
+		return nil, xerrors.Errorf("remote host %q is not an IP", host)
+	}
+	return ip, nil
+}
+
+// currentLeaf returns the cached leaf, re-minting it when it is missing or
+// expired. A leaf carries no independent lifetime: its NotAfter equals its
+// signing CA's (see mintLeaf), so re-minting is driven purely by CA rotation.
+//
+// The whole method holds t.mu so the CA cache, IP, and cached leaf are read as
+// a consistent set: a concurrent setCACache cannot swap the CA out from under
+// the IP we mint with. The lock is held across the SigningKey lookup and the
+// (rare) mint; both are cheap (an in-memory cache hit and, only on a miss, a
+// keygen+sign).
+func (t *clusterTLS) currentLeaf() (*tls.Certificate, error) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	// Reuse the cached leaf while it is still within its validity window,
+	// before consulting the signing cache. A leaf's NotAfter equals its signing
+	// CA's, and the previous CA stays trusted by peers through the rotation
+	// overlap, so a still-valid cached leaf always chains to a CA peers accept.
+	// A new CA is picked up when the leaf expires (forcing a re-mint) or when
+	// setCACache swaps the cache and clears the leaf.
+	now := t.clock.Now()
+	if t.leaf != nil && now.Before(t.leaf.Leaf.NotAfter) {
+		return t.leaf, nil
+	}
+
+	id, key, err := t.ca.SigningKey(t.ctx)
+	if err != nil {
+		return nil, xerrors.Errorf("get signing CA: %w", err)
+	}
+	ca, ok := key.(*cryptokeys.NATSCA)
+	if !ok {
+		return nil, xerrors.Errorf("unexpected signing key type %T", key)
+	}
+
+	leaf, err := mintLeaf(ca, t.ip, now)
+	if err != nil {
+		return nil, xerrors.Errorf("mint leaf: %w", err)
+	}
+	t.leaf = leaf
+	t.logger.Debug(t.ctx, "minted nats cluster leaf", slog.F("ca_sequence", id))
+	return leaf, nil
+}
+
+// mintLeaf creates an ephemeral leaf certificate signed by the active CA. The
+// signing CA's sequence is stamped into the leaf's Subject SerialNumber so a
+// verifying peer can look up exactly that CA (see verifyConnection), and the
+// replica's relay IP is embedded as an IP SAN so a dialing peer can confirm it
+// reached the host it intended. The leaf is usable as both a TLS server and
+// client certificate because each replica both accepts and dials cluster
+// routes.
+func mintLeaf(ca *cryptokeys.NATSCA, ip net.IP, now time.Time) (*tls.Certificate, error) {
+	if len(ip) == 0 {
+		return nil, xerrors.New("leaf IP SAN is required")
+	}
+
+	leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, xerrors.Errorf("generate leaf key: %w", err)
+	}
+
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), leafSerialBits))
+	if err != nil {
+		return nil, xerrors.Errorf("generate serial: %w", err)
+	}
+
+	// A leaf is only ever used to authenticate a handshake, so it need only be
+	// valid as long as the CA that signed it. Tie the leaf's NotAfter to the
+	// CA's so a leaf never outlives its CA and carries no independent lifetime.
+	// An expired active CA means a fully-dead rotator; fail loud rather than
+	// mint a dead leaf.
+	if !ca.Cert.NotAfter.After(now) {
+		return nil, xerrors.Errorf("signing CA (seq %d) is expired: NotAfter %s",
+			ca.Sequence, ca.Cert.NotAfter)
+	}
+	notAfter := ca.Cert.NotAfter
+
+	template := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName: "coder-nats-cluster-leaf",
+			// SerialNumber carries the sequence of the CA that signed this
+			// leaf, letting a verifier fetch exactly that CA from its cache.
+			SerialNumber: strconv.FormatInt(int64(ca.Sequence), 10),
+		},
+		IPAddresses:           []net.IP{ip},
+		NotBefore:             now.Add(-clockSkewToleranceTLS),
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+	}
+
+	leafDER, err := x509.CreateCertificate(rand.Reader, template, ca.Cert, &leafKey.PublicKey, ca.Key)
+	if err != nil {
+		return nil, xerrors.Errorf("create leaf certificate: %w", err)
+	}
+	leaf, err := x509.ParseCertificate(leafDER)
+	if err != nil {
+		return nil, xerrors.Errorf("parse leaf certificate: %w", err)
+	}
+
+	return &tls.Certificate{
+		Certificate: [][]byte{leafDER},
+		PrivateKey:  leafKey,
+		Leaf:        leaf,
+	}, nil
+}
+
+// verify verifies a peer's leaf certificate. It reads the signing CA sequence
+// the peer stamped into its leaf, fetches that exact CA from the cache, and
+// confirms the leaf chains to it. Using the stamped sequence is not a trust
+// decision: the leaf must still chain to OUR trusted copy of that CA, and a CA
+// that has been retired is no longer returned by the cache, so leaves from a
+// deleted CA are rejected.
+//
+// It then enforces source binding: when sourceIP is set (the accept side, where
+// the dialing peer's connection address is available), the leaf must carry that
+// source IP as an IP SAN, binding the certificate to the network origin. Go's
+// default hostname verification, which InsecureSkipVerify disables, cannot do
+// this because Go does not populate cs.ServerName for IP-based routes. On the
+// dial side sourceIP is nil (Go does not expose the connection in the
+// client-certificate callbacks), so only the chain is verified there.
+func (t *clusterTLS) verify(cs tls.ConnectionState, sourceIP net.IP) error {
+	if len(cs.PeerCertificates) == 0 {
+		return xerrors.New("no peer certificate presented")
+	}
+	leaf := cs.PeerCertificates[0]
+
+	seq := leaf.Subject.SerialNumber
+	if seq == "" {
+		return xerrors.New("peer leaf missing signing CA sequence")
+	}
+
+	key, err := t.caCache().VerifyingKey(t.ctx, seq)
+	if err != nil {
+		return xerrors.Errorf("get CA for sequence %q: %w", seq, err)
+	}
+	ca, ok := key.(*cryptokeys.NATSCA)
+	if !ok {
+		return xerrors.Errorf("unexpected verifying key type %T", key)
+	}
+
+	// Leaves carry both ServerAuth and ClientAuth, since each replica is both a
+	// route server and client. Requiring those specific usages rejects a leaf
+	// with some unexpected EKU rather than accepting any usage.
+	if _, err := leaf.Verify(x509.VerifyOptions{
+		Roots:       t.verifyPool(seq, ca.Cert),
+		KeyUsages:   []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		CurrentTime: t.clock.Now(),
+	}); err != nil {
+		return xerrors.Errorf("verify peer leaf against CA sequence %q: %w", seq, err)
+	}
+
+	// On the accept side, confirm the leaf's IP SAN matches the address the
+	// peer actually connected from.
+	if len(sourceIP) != 0 && !slices.ContainsFunc(leaf.IPAddresses, sourceIP.Equal) {
+		return xerrors.Errorf("peer leaf IP SANs %v do not match source IP %s", leaf.IPAddresses, sourceIP)
+	}
+	return nil
+}
+
+// verifyPool returns the root pool used to verify a peer leaf minted under the
+// given CA sequence, building it once and caching it for reuse. It is called
+// from verify on every route handshake; cluster routes are long-lived, so a
+// handshake is a rare event, and the common case here is a cache hit (a single
+// map lookup).
+//
+// A miss occurs only the first time a sequence is seen (startup, and once per
+// CA rotation), which is the only moment the map can grow, so pruning of expired
+// entries is attached to the miss path rather than run on every handshake. An
+// entry is dropped once the clock passes the CA cert's NotAfter: no valid leaf
+// can chain to an expired CA, and the CA outlives every leaf it signed, so this
+// is always safe and bounds the map across rotations.
+func (t *clusterTLS) verifyPool(seq string, cert *x509.Certificate) *x509.CertPool {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if cp, ok := t.verifyPools[seq]; ok {
+		return cp.pool
+	}
+
+	now := t.clock.Now()
+	for s, cp := range t.verifyPools {
+		if now.After(cp.notAfter) {
+			delete(t.verifyPools, s)
+		}
+	}
+
+	pool := x509.NewCertPool()
+	pool.AddCert(cert)
+	if t.verifyPools == nil {
+		t.verifyPools = map[string]cachedVerifyPool{}
+	}
+	t.verifyPools[seq] = cachedVerifyPool{pool: pool, notAfter: cert.NotAfter}
+	return pool
+}

coderd/x/nats/tls_internal_test.go

@@ -0,0 +1,530 @@
+package nats
+
+import (
+	"context"
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"math/big"
+	"net"
+	"strconv"
+	"testing"
+	"time"
+
+	natsserver "github.com/nats-io/nats-server/v2/server"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog/v3/sloggers/slogtest"
+	"github.com/coder/coder/v2/coderd/cryptokeys"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+)
+
+// fakeCACache is an in-memory cryptokeys.SigningKeycache for tests. active is returned
+// by SigningKey (the CA this replica mints leaves under); byID is consulted by
+// VerifyingKey (the CAs this replica trusts when verifying peers).
+type fakeCACache struct {
+	active *cryptokeys.NATSCA
+	byID   map[string]*cryptokeys.NATSCA
+}
+
+func (f *fakeCACache) SigningKey(context.Context) (string, interface{}, error) {
+	if f.active == nil {
+		return "", nil, cryptokeys.ErrKeyNotFound
+	}
+	return strconv.FormatInt(int64(f.active.Sequence), 10), f.active, nil
+}
+
+func (f *fakeCACache) VerifyingKey(_ context.Context, id string) (interface{}, error) {
+	ca, ok := f.byID[id]
+	if !ok {
+		return nil, cryptokeys.ErrKeyNotFound
+	}
+	return ca, nil
+}
+
+func (*fakeCACache) Close() error { return nil }
+
+func generateTestCA(t *testing.T, sequence int32) *cryptokeys.NATSCA {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	now := time.Now()
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(int64(sequence)),
+		Subject:               pkix.Name{CommonName: "coder-nats-ca-test"},
+		NotBefore:             now.Add(-time.Hour),
+		NotAfter:              now.Add(72 * time.Hour),
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	require.NoError(t, err)
+	cert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+
+	return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)}
+}
+
+// newTLSPubsub builds a clustered pubsub whose route listener requires mTLS,
+// using the supplied CA cache. ip is this node's cluster host: the route
+// listener bind host and the leaf IP SAN. Peers dial each other on that host
+// (clusterRouteAddress), so ip must be 127.0.0.1 for routes to form.
+func newTLSPubsub(t *testing.T, ca cryptokeys.SigningKeycache, ip net.IP) *Pubsub {
+	t.Helper()
+	logger := slogtest.Make(t, nil)
+	ctx := testutil.Context(t, testutil.WaitLong)
+	ps, err := New(ctx, logger, Options{
+		ClusterHost:       ip.String(),
+		ClusterPort:       natsserver.RANDOM_PORT,
+		disableCluster:    false,
+		ClusterCA:         ca,
+		clusterTLSTimeout: testClusterTLSTimeout,
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = ps.Close() })
+	return ps
+}
+
+// setLeafSAN overrides a node's leaf IP SAN after construction, for tests that
+// need the minted SAN to differ from the loopback address the node binds and
+// connects on (which are otherwise both the node's ClusterHost). Clearing the
+// cached leaf forces the next handshake to re-mint under the new SAN.
+func setLeafSAN(ps *Pubsub, ip net.IP) {
+	ps.clusterTLS.mu.Lock()
+	defer ps.clusterTLS.mu.Unlock()
+	ps.clusterTLS.ip = ip
+	ps.clusterTLS.leaf = nil
+}
+
+func numRoutes(t *testing.T, ps *Pubsub) int {
+	t.Helper()
+	routes, err := ps.Server.Routez(&natsserver.RoutezOptions{})
+	require.NoError(t, err)
+	return routes.NumRoutes
+}
+
+// TestPubsub_ClusterTLS validates that the embedded NATS server honors the
+// tls.Config callbacks on cluster routes: leaves minted from the CA cache form
+// a verified mesh, peers under unrelated CAs are rejected, and peers on either
+// side of a CA rotation still verify each other.
+func TestPubsub_ClusterTLS(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Mesh", func(t *testing.T) {
+		t.Parallel()
+
+		ca := generateTestCA(t, 1)
+		cache := func() *fakeCACache {
+			return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}
+		}
+		a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1))
+		b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1))
+		c := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1))
+
+		addrA := clusterRouteAddress(t, a)
+		addrB := clusterRouteAddress(t, b)
+		addrC := clusterRouteAddress(t, c)
+		// Full symmetric mesh: every node must know a peer to accept a route
+		// from it (accept-side membership), so each is given the other two.
+		// Drive peers through fetchers, as production does: a fetcher re-applies
+		// the same peers on every refresh, so the startup refresh (which runs
+		// with the boot-time noop fetcher) cannot race a manual call and wipe
+		// the route/known-peer set.
+		a.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrB, addrC}})
+		b.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrC}})
+		c.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrB}})
+
+		event := "tls-mesh"
+		got := make(chan []byte, 8)
+		cancel, err := c.Subscribe(event, func(_ context.Context, msg []byte) { got <- msg })
+		require.NoError(t, err)
+		defer cancel()
+
+		// Retry publishes until the route subscription has propagated.
+		require.Eventually(t, func() bool {
+			if err := b.Publish(event, []byte("hello")); err != nil {
+				return false
+			}
+			if err := b.Flush(); err != nil {
+				return false
+			}
+			select {
+			case msg := <-got:
+				return string(msg) == "hello"
+			case <-time.After(testutil.IntervalMedium):
+				return false
+			}
+		}, testutil.WaitLong, testutil.IntervalFast)
+	})
+
+	t.Run("WrongCARejected", func(t *testing.T) {
+		t.Parallel()
+
+		caX := generateTestCA(t, 1)
+		caY := generateTestCA(t, 1)
+		a := newTLSPubsub(t, &fakeCACache{active: caX, byID: map[string]*cryptokeys.NATSCA{"1": caX}}, net.IPv4(127, 0, 0, 1))
+		b := newTLSPubsub(t, &fakeCACache{active: caY, byID: map[string]*cryptokeys.NATSCA{"1": caY}}, net.IPv4(127, 0, 0, 1))
+
+		require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)}))
+
+		// Each side only trusts its own CA, so the route handshake never
+		// completes and no route is established.
+		require.Never(t, func() bool {
+			return numRoutes(t, a) > 0 || numRoutes(t, b) > 0
+		}, testutil.WaitShort, testutil.IntervalFast)
+	})
+
+	t.Run("RotationOverlap", func(t *testing.T) {
+		t.Parallel()
+
+		ca1 := generateTestCA(t, 1)
+		ca2 := generateTestCA(t, 2)
+		bundle := map[string]*cryptokeys.NATSCA{"1": ca1, "2": ca2}
+		// a still mints under the old CA; b has already rotated to the new CA.
+		// Both trust both CAs, so the mesh forms across the rotation overlap.
+		a := newTLSPubsub(t, &fakeCACache{active: ca1, byID: bundle}, net.IPv4(127, 0, 0, 1))
+		b := newTLSPubsub(t, &fakeCACache{active: ca2, byID: bundle}, net.IPv4(127, 0, 0, 1))
+
+		// Symmetric peers so each side accepts a route from the other, driven
+		// through fetchers (see Mesh) so the startup noop refresh cannot race a
+		// manual call and wipe the route/known-peer set.
+		a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}})
+		b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}})
+
+		require.Eventually(t, func() bool {
+			return numRoutes(t, a) > 0 && numRoutes(t, b) > 0
+		}, testutil.WaitLong, testutil.IntervalFast)
+	})
+
+	t.Run("SANMismatch", func(t *testing.T) {
+		t.Parallel()
+
+		ca := generateTestCA(t, 1)
+		cache := func() *fakeCACache {
+			return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}
+		}
+		// Both nodes bind and connect on loopback and know each other as peers,
+		// so the CA and source-membership checks pass. But both mint their leaf
+		// with a SAN that does not match the loopback address they connect from,
+		// so every handshake is rejected on the SAN binding alone and no route
+		// forms. This isolates the SAN check: a valid CA-signed leaf presented
+		// from a known replica is still rejected when the cert is not bound to
+		// the address it connects from (e.g. a stolen or mis-minted leaf).
+		a := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1))
+		b := newTLSPubsub(t, cache(), net.IPv4(127, 0, 0, 1))
+		setLeafSAN(a, net.IPv4(10, 99, 99, 99))
+		setLeafSAN(b, net.IPv4(10, 99, 99, 99))
+
+		require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)}))
+		require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)}))
+
+		require.Never(t, func() bool {
+			return numRoutes(t, a) > 0 || numRoutes(t, b) > 0
+		}, testutil.WaitShort, testutil.IntervalFast)
+	})
+
+	t.Run("MixedTLSAndPlaintext", func(t *testing.T) {
+		t.Parallel()
+
+		ca := generateTestCA(t, 1)
+		// a requires mTLS on its route listener; b is a plaintext node
+		// (newTestPubsub leaves ClusterCA nil). Routes must not form in either
+		// direction: a rollout has to enable TLS on every replica at once.
+		a := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1))
+		b := newTestPubsub(t, clusterTestOptions(t))
+
+		require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)}))
+		require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)}))
+
+		require.Never(t, func() bool {
+			return numRoutes(t, a) > 0 || numRoutes(t, b) > 0
+		}, testutil.WaitShort, testutil.IntervalFast)
+	})
+}
+
+// TestPubsub_ClusterTLS_CacheSwap covers the Part C optional-mTLS model: a node
+// that boots with the noop CA cache forms no route, and swapping in a real cache
+// via SetCACache lets routes form over mTLS with no server restart.
+func TestPubsub_ClusterTLS_CacheSwap(t *testing.T) {
+	t.Parallel()
+
+	t.Run("NoopFormsNoRoute", func(t *testing.T) {
+		t.Parallel()
+
+		ca := generateTestCA(t, 1)
+		// a boots with the noop cache (production default); b has a real cache.
+		// a cannot mint a leaf, so its route handshakes fail and no route forms.
+		a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1))
+		b := newTLSPubsub(t, &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}, net.IPv4(127, 0, 0, 1))
+
+		require.NoError(t, a.setPeerAddresses([]string{clusterRouteAddress(t, b)}))
+		require.NoError(t, b.setPeerAddresses([]string{clusterRouteAddress(t, a)}))
+
+		require.Never(t, func() bool {
+			return numRoutes(t, a) > 0 || numRoutes(t, b) > 0
+		}, testutil.WaitShort, testutil.IntervalFast)
+	})
+
+	t.Run("SwapToRealFormsRoute", func(t *testing.T) {
+		t.Parallel()
+
+		ca := generateTestCA(t, 1)
+		realCache := func() *fakeCACache {
+			return &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}
+		}
+		// Both boot with the noop cache, then both get the real cache swapped in
+		// (mirroring the enterprise HA enable path) without a server restart.
+		a := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1))
+		b := newTLSPubsub(t, cryptokeys.NoopSigningKeycache{}, net.IPv4(127, 0, 0, 1))
+
+		// Drive peers through fetchers, as production does, rather than calling
+		// setPeerAddresses directly: SetCACache and SetPeerFetcher both trigger
+		// a peer refresh that reads the current fetcher, so routes converge on
+		// the fetcher's addresses without racing a manual call.
+		a.SetCACache(realCache())
+		b.SetCACache(realCache())
+
+		a.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, b)}})
+		b.SetPeerFetcher(&testPeerFetcher{addresses: []string{clusterRouteAddress(t, a)}})
+
+		require.Eventually(t, func() bool {
+			return numRoutes(t, a) > 0 && numRoutes(t, b) > 0
+		}, testutil.WaitLong, testutil.IntervalFast)
+	})
+}
+
+// TestClusterTLS_configForClient_RequiresSourceIP asserts the accept side fails
+// closed when it cannot determine the peer's source IP, rather than skipping the
+// source-binding check.
+func TestClusterTLS_configForClient_RequiresSourceIP(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	ca := generateTestCA(t, 1)
+	cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}
+	ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(127, 0, 0, 1))
+
+	// No underlying connection: the source IP cannot be determined, so the
+	// accept-side config is refused and the handshake aborts.
+	_, err := ct.configForClient(&tls.ClientHelloInfo{})
+	require.Error(t, err)
+}
+
+// TestClusterTLS_verify unit-tests the verifier directly, isolating chain
+// verification and source-IP binding that the mesh tests exercise only
+// indirectly.
+func TestClusterTLS_verify(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	ca := generateTestCA(t, 1)
+	cache := &fakeCACache{active: ca, byID: map[string]*cryptokeys.NATSCA{"1": ca}}
+
+	leafIP := net.IPv4(10, 0, 0, 5)
+	ct := newClusterTLS(ctx, slogtest.Make(t, nil), nil, cache, net.IPv4(10, 0, 0, 1))
+
+	// A leaf bound to leafIP, signed by the trusted CA.
+	leafCert, err := mintLeaf(ca, leafIP, time.Now())
+	require.NoError(t, err)
+	leaf, err := x509.ParseCertificate(leafCert.Certificate[0])
+	require.NoError(t, err)
+	cs := tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}}
+
+	t.Run("DialSideChainOnly", func(t *testing.T) {
+		t.Parallel()
+		// No source IP (dial side): only the chain is verified.
+		require.NoError(t, ct.verify(cs, nil))
+	})
+
+	t.Run("AcceptSideSourceMatches", func(t *testing.T) {
+		t.Parallel()
+		// Source IP equals the leaf SAN: accepted.
+		require.NoError(t, ct.verify(cs, leafIP))
+	})
+
+	t.Run("AcceptSideSourceMismatch", func(t *testing.T) {
+		t.Parallel()
+		// The leaf is bound to leafIP, so a connection from a different source
+		// is rejected even though the chain is valid.
+		err := ct.verify(cs, net.IPv4(10, 0, 0, 1))
+		require.ErrorContains(t, err, "do not match source IP")
+	})
+
+	t.Run("UntrustedCARejected", func(t *testing.T) {
+		t.Parallel()
+		otherCA := generateTestCA(t, 9)
+		strangerCert, err := mintLeaf(otherCA, leafIP, time.Now())
+		require.NoError(t, err)
+		stranger, err := x509.ParseCertificate(strangerCert.Certificate[0])
+		require.NoError(t, err)
+		// The stamped sequence (9) is not in the cache, so the CA lookup fails.
+		err = ct.verify(tls.ConnectionState{PeerCertificates: []*x509.Certificate{stranger}}, nil)
+		require.Error(t, err)
+	})
+}
+
+// TestClusterTLS_verifyPool asserts the verify-pool cache reuses a pool for a
+// given CA sequence and prunes entries whose CA cert has expired, so the map
+// does not grow unbounded across rotations.
+func TestClusterTLS_verifyPool(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	clock := quartz.NewMock(t)
+	clock.Set(time.Now())
+
+	ca1 := generateTestCA(t, 1)
+	ca2 := generateTestCA(t, 2)
+	cache := &fakeCACache{byID: map[string]*cryptokeys.NATSCA{"1": ca1, "2": ca2}}
+	ct := newClusterTLS(ctx, slogtest.Make(t, nil), clock, cache, net.IPv4(10, 0, 0, 1))
+
+	// First build for seq 1 caches the pool; a second call returns the same one.
+	p1 := ct.verifyPool("1", ca1.Cert)
+	require.Same(t, p1, ct.verifyPool("1", ca1.Cert))
+	require.Len(t, ct.verifyPools, 1)
+
+	// Advance past ca1's NotAfter. Building a pool for a new sequence prunes the
+	// now-expired seq 1 entry, leaving only seq 2.
+	clock.Set(ca1.Cert.NotAfter.Add(time.Minute))
+	ct.verifyPool("2", ca2.Cert)
+	require.Len(t, ct.verifyPools, 1)
+	_, ok := ct.verifyPools["1"]
+	require.False(t, ok, "expired seq 1 pool should be pruned")
+	_, ok = ct.verifyPools["2"]
+	require.True(t, ok)
+}
+
+// generateTestCAWithValidity is like generateTestCA but lets a test control the
+// CA certificate's NotAfter, so leaf-clamp behavior near CA expiry is testable.
+func generateTestCAWithValidity(t *testing.T, sequence int32, notAfter time.Time) *cryptokeys.NATSCA {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+
+	template := &x509.Certificate{
+		SerialNumber:          big.NewInt(int64(sequence)),
+		Subject:               pkix.Name{CommonName: "coder-nats-ca-test"},
+		NotBefore:             notAfter.Add(-90 * 24 * time.Hour),
+		NotAfter:              notAfter,
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	require.NoError(t, err)
+	cert, err := x509.ParseCertificate(der)
+	require.NoError(t, err)
+
+	return &cryptokeys.NATSCA{Sequence: sequence, Cert: cert, Key: crypto.Signer(key)}
+}
+
+// TestMintLeaf asserts a leaf's NotAfter is exactly its signing CA's NotAfter
+// (a leaf carries no independent lifetime), and that minting against an
+// already-expired CA fails rather than emitting a dead leaf.
+func TestMintLeaf(t *testing.T) {
+	t.Parallel()
+
+	ip := net.IPv4(127, 0, 0, 1)
+
+	t.Run("MatchesCAValidity", func(t *testing.T) {
+		t.Parallel()
+		now := time.Now()
+		// generateTestCA mints a 72h CA; the leaf's NotAfter tracks it exactly.
+		ca := generateTestCA(t, 1)
+		leaf, err := mintLeaf(ca, ip, now)
+		require.NoError(t, err)
+		require.WithinDuration(t, ca.Cert.NotAfter, leaf.Leaf.NotAfter, time.Second)
+		require.WithinDuration(t, now.Add(-clockSkewToleranceTLS), leaf.Leaf.NotBefore, time.Second)
+	})
+
+	t.Run("ErrorsWhenCAExpired", func(t *testing.T) {
+		t.Parallel()
+		now := time.Now()
+		// CA's NotAfter is already in the past: no usable leaf can be minted.
+		ca := generateTestCAWithValidity(t, 1, now.Add(-time.Minute))
+		_, err := mintLeaf(ca, ip, now)
+		require.Error(t, err)
+		require.ErrorContains(t, err, "expired")
+	})
+}
+
+// TestPubsub_ClusterTLS_RealCA stands up a three-node TLS mesh whose trust root
+// is a real CA served by the cryptokeys signing cache against a real DB, then
+// verifies a cross-route publish/subscribe round-trip. This exercises the
+// integration seam between the cryptokeys CA cache and the x/nats cluster TLS
+// callbacks, including the real PEM/x509 round-trip that the synthetic
+// generateTestCA helper does not cover. Nodes form a direct full mesh to avoid
+// depending on multi-hop route gossip.
+func TestPubsub_ClusterTLS_RealCA(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	// Seed an active nats_ca crypto key, mirroring the row the key rotator
+	// mints in production. The signing cache decodes the PEM secret into a
+	// *cryptokeys.NATSCA the same way production reads it.
+	dbgen.CryptoKey(t, db, database.CryptoKey{
+		Feature:  database.CryptoKeyFeatureNATSCA,
+		Sequence: 1,
+		StartsAt: time.Now().UTC().Add(-time.Hour),
+	})
+
+	newNode := func() *Pubsub {
+		// A real signing cache per node, as each replica builds in coderd.New.
+		cache, err := cryptokeys.NewSigningCache(ctx, slogtest.Make(t, nil), &cryptokeys.DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = cache.Close() })
+		// Nodes mesh on loopback, so the leaf IP SAN must be 127.0.0.1.
+		return newTLSPubsub(t, cache, net.IPv4(127, 0, 0, 1))
+	}
+
+	a := newNode()
+	b := newNode()
+	c := newNode()
+
+	addrA := clusterRouteAddress(t, a)
+	addrB := clusterRouteAddress(t, b)
+	addrC := clusterRouteAddress(t, c)
+	// Drive peers through fetchers, as production does, so the startup noop
+	// refresh cannot race a manual call and wipe the route/known-peer set.
+	a.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrB, addrC}})
+	b.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrC}})
+	c.SetPeerFetcher(&testPeerFetcher{addresses: []string{addrA, addrB}})
+
+	received := make(chan string, 4)
+	cancelSub, err := c.Subscribe("tls-realca", func(_ context.Context, msg []byte) {
+		select {
+		case received <- string(msg):
+		default:
+		}
+	})
+	require.NoError(t, err)
+	defer cancelSub()
+
+	// Routes and subscription interest propagate asynchronously after the
+	// servers report ready, so retry rather than gate on a one-shot check.
+	require.Eventually(t, func() bool {
+		if err := b.Publish("tls-realca", []byte("hello")); err != nil {
+			return false
+		}
+		select {
+		case msg := <-received:
+			require.Equal(t, "hello", msg)
+			return true
+		case <-time.After(testutil.IntervalMedium):
+			return false
+		}
+	}, testutil.WaitLong, testutil.IntervalFast)
+}