feat: NATS mTLS pubsub implementation (#26902)

coderd/cryptokeys/ca.go

@@ -0,0 +1,149 @@
+package cryptokeys
+
+import (
+	"crypto"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"time"
+
+	"golang.org/x/xerrors"
+)
+
+const (
+	caCertPEMBlockType = "CERTIFICATE"
+	caKeyPEMBlockType  = "EC PRIVATE KEY"
+
+	// clockSkewTolerance backdates the CA certificate's NotBefore and extends
+	// its NotAfter so that replicas with mildly skewed clocks still accept it.
+	clockSkewTolerance = time.Hour
+)
+
+// NATSCA is the decoded form of a single nats_ca crypto key row, produced by
+// the generic crypto key cache (see idSecret). The CA signs the ephemeral leaf
+// certificates that replicas use for NATS cluster mTLS.
+//
+// The active CA is served by a SigningKeycache.SigningKey call for the nats_ca
+// feature; a specific historical CA (for verifying a peer leaf minted under an
+// earlier CA during a rotation overlap) is served by VerifyingKey with that
+// row's sequence.
+type NATSCA struct {
+	// Sequence is the crypto_keys sequence of the row this CA came from.
+	Sequence int32
+	// Cert is the CA certificate used to sign or verify leaf certificates.
+	Cert *x509.Certificate
+	// Key is the CA private key, used to sign leaves.
+	Key crypto.Signer
+}
+
+// generateCASecret generates a new self-signed CA certificate and private key
+// for signing NATS cluster leaf certificates, PEM-encoded into a single
+// bundle for storage in the crypto_keys secret column.
+//
+// anchorTime is the key row's starts_at (which may be in the future for a
+// rotated-in key). keyDuration is the rotator's key duration: the row stays the
+// active signer for that long. The certificate stays valid for NATSCAOverlap
+// past that window so that, once the next CA becomes the active signer, this CA
+// is still valid while replicas' key caches refresh onto the new one. Leaves
+// are separately clamped to expire before this NotAfter (see coderd/x/nats
+// mintLeaf), so the overlap only needs to cover the cache-refresh transition.
+func generateCASecret(anchorTime time.Time, keyDuration time.Duration) (string, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return "", xerrors.Errorf("generate key: %w", err)
+	}
+
+	// 128-bit random serial per CA/Browser Forum conventions.
+	serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
+	if err != nil {
+		return "", xerrors.Errorf("generate serial: %w", err)
+	}
+
+	template := &x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			CommonName: "coder-nats-ca",
+		},
+		NotBefore:             anchorTime.Add(-clockSkewTolerance),
+		NotAfter:              anchorTime.Add(keyDuration + NATSCAOverlap),
+		KeyUsage:              x509.KeyUsageCertSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+		MaxPathLenZero:        true,
+	}
+
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	if err != nil {
+		return "", xerrors.Errorf("create certificate: %w", err)
+	}
+
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		return "", xerrors.Errorf("marshal private key: %w", err)
+	}
+
+	var secret []byte
+	secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caCertPEMBlockType, Bytes: der})...)
+	secret = append(secret, pem.EncodeToMemory(&pem.Block{Type: caKeyPEMBlockType, Bytes: keyDER})...)
+	return string(secret), nil
+}
+
+// parseCASecret parses a PEM bundle produced by generateCASecret back into
+// the CA certificate and private key.
+func parseCASecret(secret string) (*x509.Certificate, crypto.Signer, error) {
+	var (
+		cert *x509.Certificate
+		key  *ecdsa.PrivateKey
+	)
+	rest := []byte(secret)
+	for {
+		var block *pem.Block
+		block, rest = pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		switch block.Type {
+		case caCertPEMBlockType:
+			if cert != nil {
+				return nil, nil, xerrors.New("multiple certificates in CA secret")
+			}
+			var err error
+			cert, err = x509.ParseCertificate(block.Bytes)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("parse certificate: %w", err)
+			}
+		case caKeyPEMBlockType:
+			if key != nil {
+				return nil, nil, xerrors.New("multiple private keys in CA secret")
+			}
+			var err error
+			key, err = x509.ParseECPrivateKey(block.Bytes)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("parse private key: %w", err)
+			}
+		default:
+			return nil, nil, xerrors.Errorf("unexpected PEM block type: %q", block.Type)
+		}
+	}
+	if cert == nil {
+		return nil, nil, xerrors.New("no certificate in CA secret")
+	}
+	if key == nil {
+		return nil, nil, xerrors.New("no private key in CA secret")
+	}
+	if !key.PublicKey.Equal(cert.PublicKey) {
+		return nil, nil, xerrors.New("private key does not match certificate")
+	}
+	// Reject a structurally valid bundle whose certificate cannot act as a
+	// signing CA. Without this, a corrupted secret could yield a non-CA cert
+	// that silently becomes the active signer; leaves signed under it would
+	// then fail x509 verification on every replica.
+	if !cert.IsCA || !cert.BasicConstraintsValid || cert.KeyUsage&x509.KeyUsageCertSign == 0 {
+		return nil, nil, xerrors.New("certificate is not a valid signing CA")
+	}
+	return cert, key, nil
+}

coderd/cryptokeys/ca_internal_test.go

@@ -0,0 +1,251 @@
+package cryptokeys
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+)
+
+func TestCASecretRoundTrip(t *testing.T) {
+	t.Parallel()
+
+	// The certificate's NotAfter must track the supplied keyDuration, not a
+	// hardcoded default, so a CA stays valid for as long as it can be the
+	// active signer plus the longest leaf it signs.
+	for _, keyDuration := range []time.Duration{DefaultKeyDuration, DefaultKeyDuration * 3, time.Hour} {
+		now := time.Now().UTC().Truncate(time.Second)
+		secret, err := generateCASecret(now, keyDuration)
+		require.NoError(t, err)
+
+		cert, signer, err := parseCASecret(secret)
+		require.NoError(t, err)
+
+		require.True(t, cert.IsCA)
+		require.True(t, cert.BasicConstraintsValid)
+		require.True(t, cert.MaxPathLenZero)
+		require.Equal(t, x509.KeyUsageCertSign, cert.KeyUsage)
+		require.Equal(t, now.Add(-clockSkewTolerance), cert.NotBefore)
+		require.Equal(t, now.Add(keyDuration+NATSCAOverlap), cert.NotAfter)
+		require.Equal(t, cert.PublicKey, signer.Public())
+
+		// The cert must outlive its active-signer window so leaves signed at
+		// the end of that window still chain to a valid CA.
+		require.True(t, cert.NotAfter.After(now.Add(keyDuration)),
+			"cert must remain valid past the end of its active-signer window")
+
+		// The cert must be able to verify itself as a trust root.
+		pool := x509.NewCertPool()
+		pool.AddCert(cert)
+		_, err = cert.Verify(x509.VerifyOptions{Roots: pool})
+		require.NoError(t, err)
+	}
+}
+
+func TestParseCASecretErrors(t *testing.T) {
+	t.Parallel()
+
+	now := time.Now()
+	secretA, err := generateCASecret(now, DefaultKeyDuration)
+	require.NoError(t, err)
+	secretB, err := generateCASecret(now, DefaultKeyDuration)
+	require.NoError(t, err)
+
+	certA, keyA := splitCAPEM(t, secretA)
+	_, keyB := splitCAPEM(t, secretB)
+
+	nonCACert, nonCAKey := generateNonCAPEM(t, now)
+
+	cases := []struct {
+		name    string
+		secret  string
+		errText string
+	}{
+		{"Empty", "", "no certificate"},
+		{"NotPEM", "not pem at all", "no certificate"},
+		{"CertOnly", string(certA), "no private key"},
+		{"KeyCertMismatch", string(certA) + string(keyB), "does not match certificate"},
+		{"MultipleCertificates", string(certA) + string(certA) + string(keyA), "multiple certificates"},
+		{"MultiplePrivateKeys", string(certA) + string(keyA) + string(keyA), "multiple private keys"},
+		{"UnexpectedBlockType", string(pemBlock("RSA PRIVATE KEY", []byte("x"))), "unexpected PEM block type"},
+		{"BadCertificateBytes", string(pemBlock(caCertPEMBlockType, []byte("garbage"))), "parse certificate"},
+		{"BadPrivateKeyBytes", string(certA) + string(pemBlock(caKeyPEMBlockType, []byte("garbage"))), "parse private key"},
+		{"NotASigningCA", string(nonCACert) + string(nonCAKey), "not a valid signing CA"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			_, _, err := parseCASecret(tc.secret)
+			require.ErrorContains(t, err, tc.errText)
+		})
+	}
+}
+
+// splitCAPEM splits a CA secret bundle into its certificate and private key
+// PEM blocks so tests can recombine them into malformed bundles.
+func splitCAPEM(t *testing.T, secret string) (certPEM, keyPEM []byte) {
+	t.Helper()
+	rest := []byte(secret)
+	for {
+		block, r := pem.Decode(rest)
+		if block == nil {
+			break
+		}
+		rest = r
+		switch block.Type {
+		case caCertPEMBlockType:
+			certPEM = pem.EncodeToMemory(block)
+		case caKeyPEMBlockType:
+			keyPEM = pem.EncodeToMemory(block)
+		}
+	}
+	require.NotNil(t, certPEM)
+	require.NotNil(t, keyPEM)
+	return certPEM, keyPEM
+}
+
+func pemBlock(blockType string, der []byte) []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der})
+}
+
+// generateNonCAPEM produces a structurally valid cert+key bundle whose
+// certificate is not a CA (no IsCA, no KeyUsageCertSign). The key matches the
+// cert, so it passes every parseCASecret check except the signing-CA check.
+func generateNonCAPEM(t *testing.T, now time.Time) (certPEM, keyPEM []byte) {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: "not-a-ca"},
+		NotBefore:    now.Add(-time.Hour),
+		NotAfter:     now.Add(time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, key.Public(), key)
+	require.NoError(t, err)
+	keyDER, err := x509.MarshalECPrivateKey(key)
+	require.NoError(t, err)
+	return pemBlock(caCertPEMBlockType, der), pemBlock(caKeyPEMBlockType, keyDER)
+}
+
+// TestNATSCASigningCache exercises the nats_ca feature through the generic
+// signing key cache: the PEM secret decodes into a *NATSCA, SigningKey serves
+// the active CA, VerifyingKey serves a specific CA by sequence, and a rotation
+// is picked up on the next refresh.
+func TestNATSCASigningCache(t *testing.T) {
+	t.Parallel()
+
+	t.Run("ActiveAndVerifyingByID", func(t *testing.T) {
+		t.Parallel()
+
+		db, _ := dbtestutil.NewDB(t)
+		ctx := testutil.Context(t, testutil.WaitShort)
+		now := time.Now().UTC()
+
+		current := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: 1,
+			StartsAt: now.Add(-time.Hour),
+		})
+
+		cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA)
+		require.NoError(t, err)
+		defer cache.Close()
+
+		id, key, err := cache.SigningKey(ctx)
+		require.NoError(t, err)
+
+		ca, ok := key.(*NATSCA)
+		require.True(t, ok, "signing key should decode to *NATSCA, got %T", key)
+		require.Equal(t, current.Sequence, ca.Sequence)
+		require.NotNil(t, ca.Cert)
+		require.NotNil(t, ca.Key)
+
+		currentCert, _, err := parseCASecret(current.Secret.String)
+		require.NoError(t, err)
+		require.Equal(t, currentCert.Raw, ca.Cert.Raw)
+
+		// VerifyingKey looks the CA up by the sequence embedded in id, which is
+		// how a peer leaf minted under this CA is verified.
+		verifying, err := cache.VerifyingKey(ctx, id)
+		require.NoError(t, err)
+		vca, ok := verifying.(*NATSCA)
+		require.True(t, ok, "verifying key should decode to *NATSCA, got %T", verifying)
+		require.Equal(t, currentCert.Raw, vca.Cert.Raw)
+	})
+
+	t.Run("RefreshesOnRotation", func(t *testing.T) {
+		t.Parallel()
+
+		db, _ := dbtestutil.NewDB(t)
+		ctx := testutil.Context(t, testutil.WaitShort)
+		clock := quartz.NewMock(t)
+		now := dbtime.Now()
+		clock.Set(now)
+
+		first := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: 1,
+			StartsAt: now.Add(-time.Hour),
+		})
+
+		cache, err := NewSigningCache(ctx, testutil.Logger(t), &DBFetcher{DB: db}, codersdk.CryptoKeyFeatureNATSCA, WithCacheClock(clock))
+		require.NoError(t, err)
+		defer cache.Close()
+
+		_, key, err := cache.SigningKey(ctx)
+		require.NoError(t, err)
+		require.Equal(t, first.Sequence, key.(*NATSCA).Sequence)
+
+		// Simulate a rotation by inserting a higher-sequence active CA. The old
+		// CA stays valid for verification by its sequence.
+		second := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: 2,
+			StartsAt: now.Add(-time.Minute),
+		})
+
+		// Fire the background refresher; the active CA advances to the new row.
+		clock.Advance(refreshInterval).MustWait(ctx)
+
+		_, key, err = cache.SigningKey(ctx)
+		require.NoError(t, err)
+		require.Equal(t, second.Sequence, key.(*NATSCA).Sequence)
+
+		oldVerifying, err := cache.VerifyingKey(ctx, "1")
+		require.NoError(t, err)
+		require.Equal(t, first.Sequence, oldVerifying.(*NATSCA).Sequence)
+	})
+}
+
+func TestNoopSigningKeycache(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	var cache SigningKeycache = NoopSigningKeycache{}
+
+	_, _, err := cache.SigningKey(ctx)
+	require.ErrorIs(t, err, ErrKeyNotFound)
+
+	_, err = cache.VerifyingKey(ctx, "1")
+	require.ErrorIs(t, err, ErrKeyNotFound)
+
+	require.NoError(t, cache.Close())
+}

coderd/cryptokeys/cache.go

@@ -55,6 +55,26 @@ type SigningKeycache interface {
 	io.Closer
 }
 
+// NoopSigningKeycache is a SigningKeycache that holds no keys: SigningKey and
+// VerifyingKey always report ErrKeyNotFound. It lets a subsystem that only
+// needs real keys once an optional feature is enabled (for example NATS
+// cluster mTLS, which only signs leaves under enterprise HA) be constructed
+// without a database dependency, then be swapped for a real cache when the
+// feature turns on.
+type NoopSigningKeycache struct{}
+
+var _ SigningKeycache = NoopSigningKeycache{}
+
+func (NoopSigningKeycache) SigningKey(context.Context) (string, interface{}, error) {
+	return "", nil, ErrKeyNotFound
+}
+
+func (NoopSigningKeycache) VerifyingKey(context.Context, string) (interface{}, error) {
+	return nil, ErrKeyNotFound
+}
+
+func (NoopSigningKeycache) Close() error { return nil }
+
 const (
 	// latestSequence is a special sequence number that represents the latest key.
 	latestSequence = -1
@@ -213,23 +233,42 @@ func isEncryptionKeyFeature(feature codersdk.CryptoKeyFeature) bool {
 
 func isSigningKeyFeature(feature codersdk.CryptoKeyFeature) bool {
 	switch feature {
-	case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken:
+	case codersdk.CryptoKeyFeatureTailnetResume, codersdk.CryptoKeyFeatureOIDCConvert, codersdk.CryptoKeyFeatureWorkspaceAppsToken, codersdk.CryptoKeyFeatureNATSCA:
 		return true
 	default:
 		return false
 	}
 }
 
-func idSecret(k codersdk.CryptoKey) (string, []byte, error) {
+// idSecret materializes a stored crypto key into the in-memory key object the
+// feature uses, returning it as an interface{} alongside the key's id (its
+// sequence as a decimal string). Most features hex-decode the secret into raw
+// bytes, but nats_ca stores a PEM cert+key bundle and decodes into a *NATSCA.
+//
+// TODO: this hard-coded switch on feature is the simplest way to support a
+// second secret encoding, but it couples this generic cache to nats_ca
+// specifics. Explore abstracting the decode step (for example a per-feature
+// decoder injected at construction) so new key types can be added without
+// editing this function.
+func idSecret(k codersdk.CryptoKey) (string, interface{}, error) {
+	id := strconv.FormatInt(int64(k.Sequence), 10)
+
+	if k.Feature == codersdk.CryptoKeyFeatureNATSCA {
+		cert, signer, err := parseCASecret(k.Secret)
+		if err != nil {
+			return "", nil, xerrors.Errorf("decode nats_ca key: %w", err)
+		}
+		return id, &NATSCA{Sequence: k.Sequence, Cert: cert, Key: signer}, nil
+	}
+
 	key, err := hex.DecodeString(k.Secret)
 	if err != nil {
 		return "", nil, xerrors.Errorf("decode key: %w", err)
 	}
-
-	return strconv.FormatInt(int64(k.Sequence), 10), key, nil
+	return id, key, nil
 }
 
-func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, []byte, error) {
+func (c *cache) cryptoKey(ctx context.Context, sequence int32) (string, interface{}, error) {
 	c.logger.Debug(ctx, "request for key", slog.F("sequence", sequence))
 	c.mu.Lock()
 	defer c.mu.Unlock()
@@ -284,7 +323,7 @@ func (c *cache) key(sequence int32) (codersdk.CryptoKey, bool) {
 	return key, ok
 }
 
-func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, []byte, error) {
+func checkKey(key codersdk.CryptoKey, sequence int32, now time.Time) (string, interface{}, error) {
 	if sequence == latestSequence {
 		if !key.CanSign(now) {
 			return "", nil, ErrKeyInvalid

coderd/cryptokeys/rotate.go

@@ -21,6 +21,15 @@ const (
 	WorkspaceAppsTokenDuration = time.Minute
 	OIDCConvertTokenDuration   = time.Minute * 5
 	TailnetResumeTokenDuration = time.Hour * 24
+	// NATSCAOverlap is how long a NATS cluster CA certificate stays valid past
+	// the end of its active-signing window (startsAt + keyDuration). The next CA
+	// becomes the active signer at the window's end, but replicas keep minting
+	// leaves with the old CA until their key cache refreshes onto the new one.
+	// This overlap keeps the old CA valid through that transition, so it must
+	// exceed the cache refresh interval (plus a small leaf clamp buffer). Leaf
+	// lifetime imposes nothing here: leaves are clamped to just before their
+	// signing CA's NotAfter (see coderd/x/nats mintLeaf).
+	NATSCAOverlap = time.Minute * 30
 
 	// defaultRotationInterval is the default interval at which keys are checked for rotation.
 	defaultRotationInterval = time.Minute * 10
@@ -70,6 +79,15 @@ func WithKeyDuration(keyDuration time.Duration) RotatorOption {
 	}
 }
 
+// WithFeatures sets the crypto key features the rotator manages, replacing the
+// default set. Use this to opt experiment- or deployment-gated features (such
+// as the NATS cluster CA) into rotation only when their owner is active.
+func WithFeatures(features []database.CryptoKeyFeature) RotatorOption {
+	return func(r *rotator) {
+		r.features = slices.Clone(features)
+	}
+}
+
 // StartRotator starts a background process that rotates keys in the database.
 // It ensures there's at least one valid key per feature prior to returning.
 // Canceling the provided context will stop the background process.
@@ -126,10 +144,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error {
 				return xerrors.Errorf("get keys: %w", err)
 			}
 
-			featureKeys, err := keysByFeature(cryptokeys, k.features)
-			if err != nil {
-				return xerrors.Errorf("keys by feature: %w", err)
-			}
+			featureKeys := keysByFeature(cryptokeys, k.features)
 
 			now := dbtime.Time(k.clock.Now().UTC())
 			for feature, keys := range featureKeys {
@@ -189,7 +204,7 @@ func (k *rotator) rotateKeys(ctx context.Context) error {
 }
 
 func (k *rotator) insertNewKey(ctx context.Context, tx database.Store, feature database.CryptoKeyFeature, startsAt time.Time) (database.CryptoKey, error) {
-	secret, err := generateNewSecret(feature)
+	secret, err := generateNewSecret(feature, startsAt, k.keyDuration)
 	if err != nil {
 		return database.CryptoKey{}, xerrors.Errorf("generate new secret: %w", err)
 	}
@@ -246,7 +261,11 @@ func (k *rotator) rotateKey(ctx context.Context, tx database.Store, key database
 	return []database.CryptoKey{updatedKey, newKey}, nil
 }
 
-func generateNewSecret(feature database.CryptoKeyFeature) (string, error) {
+// generateNewSecret generates the secret for a new key of the given feature.
+// keyDuration is the rotator's key duration; it is only used by features whose
+// secret encodes its own validity window (currently only the NATS CA, whose
+// certificate must outlive the key row's active-signer period).
+func generateNewSecret(feature database.CryptoKeyFeature, startsAt time.Time, keyDuration time.Duration) (string, error) {
 	switch feature {
 	case database.CryptoKeyFeatureWorkspaceAppsAPIKey:
 		return generateKey(32)
@@ -256,6 +275,8 @@ func generateNewSecret(feature database.CryptoKeyFeature) (string, error) {
 		return generateKey(64)
 	case database.CryptoKeyFeatureTailnetResume:
 		return generateKey(64)
+	case database.CryptoKeyFeatureNATSCA:
+		return generateCASecret(startsAt, keyDuration)
 	}
 	return "", xerrors.Errorf("unknown feature: %s", feature)
 }
@@ -279,6 +300,11 @@ func tokenDuration(feature database.CryptoKeyFeature) time.Duration {
 		return OIDCConvertTokenDuration
 	case database.CryptoKeyFeatureTailnetResume:
 		return TailnetResumeTokenDuration
+	case database.CryptoKeyFeatureNATSCA:
+		// The old CA row only needs to outlive its own certificate, which stays
+		// valid for NATSCAOverlap past the active-signing window. Keeping the
+		// row (and thus its trust-root status) beyond cert expiry is pointless.
+		return NATSCAOverlap
 	default:
 		return 0
 	}
@@ -297,19 +323,25 @@ func shouldRotateKey(key database.CryptoKey, keyDuration time.Duration, now time
 	return !now.Add(time.Hour).UTC().Before(expirationTime)
 }
 
-func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) (map[database.CryptoKeyFeature][]database.CryptoKey, error) {
+// keysByFeature groups keys by feature, restricted to the managed feature set.
+// GetCryptoKeys returns rows for every feature, but the rotator only manages a
+// subset (features can be gated, e.g. nats_ca behind an experiment). Keys for
+// features outside the managed set belong to features this rotator is not
+// responsible for and are skipped, so their presence (for example nats_ca rows
+// left over from a prior experiment-on run) does not abort rotation of the
+// managed features.
+func keysByFeature(keys []database.CryptoKey, features []database.CryptoKeyFeature) map[database.CryptoKeyFeature][]database.CryptoKey {
 	m := map[database.CryptoKeyFeature][]database.CryptoKey{}
 	for _, feature := range features {
 		m[feature] = []database.CryptoKey{}
 	}
 	for _, key := range keys {
 		if _, ok := m[key.Feature]; !ok {
-			return nil, xerrors.Errorf("unknown feature: %s", key.Feature)
+			continue
 		}
-
 		m[key.Feature] = append(m[key.Feature], key)
 	}
-	return m, nil
+	return m
 }
 
 // minStartsAt ensures the minimum starts_at time we use for a new

coderd/cryptokeys/rotate_internal_test.go

@@ -104,6 +104,112 @@ func Test_rotateKeys(t *testing.T) {
 		require.Equal(t, newKey, keys[0])
 	})
 
+	t.Run("RotatesNATSCA", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			db, _       = dbtestutil.NewDB(t)
+			clock       = quartz.NewMock(t)
+			keyDuration = time.Hour * 24 * 7
+			logger      = testutil.Logger(t)
+			ctx         = testutil.Context(t, testutil.WaitShort)
+		)
+
+		kr := &rotator{
+			db:          db,
+			keyDuration: keyDuration,
+			clock:       clock,
+			logger:      logger,
+			features: []database.CryptoKeyFeature{
+				database.CryptoKeyFeatureNATSCA,
+			},
+		}
+
+		now := dbnow(clock)
+
+		oldKey := dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			StartsAt: now,
+			Sequence: 4,
+		})
+
+		// Advance the window to just inside rotation time.
+		_ = clock.Advance(keyDuration - time.Minute*59)
+		err := kr.rotateKeys(ctx)
+		require.NoError(t, err)
+
+		// The old CA row is retained roughly as long as its certificate is
+		// valid: NATSCAOverlap past the active-signing window, plus the
+		// rotator's standard 1h propagation buffer.
+		expectedDeletesAt := oldKey.ExpiresAt(keyDuration).Add(NATSCAOverlap + time.Hour)
+		oldKey, err = db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
+			Feature:  oldKey.Feature,
+			Sequence: oldKey.Sequence,
+		})
+		require.NoError(t, err)
+		require.Equal(t, expectedDeletesAt, oldKey.DeletesAt.Time.UTC())
+
+		newKey, err := db.GetCryptoKeyByFeatureAndSequence(ctx, database.GetCryptoKeyByFeatureAndSequenceParams{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			Sequence: oldKey.Sequence + 1,
+		})
+		require.NoError(t, err)
+		requireKey(t, newKey, database.CryptoKeyFeatureNATSCA, oldKey.ExpiresAt(keyDuration), nullTime, oldKey.Sequence+1)
+	})
+
+	t.Run("IgnoresUnmanagedFeatureKeys", func(t *testing.T) {
+		t.Parallel()
+
+		// Regression: a rotator managing a subset of features (e.g. after the
+		// nats_ca experiment is toggled off) must still rotate its managed
+		// features even when the DB holds keys for features it does not manage,
+		// such as nats_ca rows left over from a prior experiment-on run.
+		// Previously such rows aborted every rotation.
+		var (
+			db, _       = dbtestutil.NewDB(t)
+			clock       = quartz.NewMock(t)
+			keyDuration = time.Hour * 24 * 7
+			logger      = testutil.Logger(t)
+			ctx         = testutil.Context(t, testutil.WaitShort)
+		)
+
+		kr := &rotator{
+			db:          db,
+			keyDuration: keyDuration,
+			clock:       clock,
+			logger:      logger,
+			// Manages only tailnet resume; nats_ca is intentionally not managed,
+			// mirroring the experiment being off.
+			features: []database.CryptoKeyFeature{
+				database.CryptoKeyFeatureTailnetResume,
+			},
+		}
+
+		now := dbnow(clock)
+
+		// A leftover nats_ca row the rotator does not manage.
+		_ = dbgen.CryptoKey(t, db, database.CryptoKey{
+			Feature:  database.CryptoKeyFeatureNATSCA,
+			StartsAt: now,
+			Sequence: 1,
+		})
+
+		// No managed key exists yet, so rotation must insert one for the managed
+		// feature and must not error on the unmanaged nats_ca row.
+		err := kr.rotateKeys(ctx)
+		require.NoError(t, err)
+
+		newKey, err := db.GetLatestCryptoKeyByFeature(ctx, database.CryptoKeyFeatureTailnetResume)
+		require.NoError(t, err)
+		require.Equal(t, database.CryptoKeyFeatureTailnetResume, newKey.Feature)
+
+		// The unmanaged nats_ca row is untouched (no rotation, no delete).
+		natsKeys, err := db.GetCryptoKeysByFeature(ctx, database.CryptoKeyFeatureNATSCA)
+		require.NoError(t, err)
+		require.Len(t, natsKeys, 1)
+		require.False(t, natsKeys[0].DeletesAt.Valid)
+	})
+
 	t.Run("DoesNotRotateValidKeys", func(t *testing.T) {
 		t.Parallel()
 
@@ -409,8 +515,7 @@ func Test_rotateKeys(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, keys, 5)
 
-		kbf, err := keysByFeature(keys, defaultRotatedFeatures)
-		require.NoError(t, err)
+		kbf := keysByFeature(keys, defaultRotatedFeatures)
 
 		// No actions on OIDC convert.
 		require.Len(t, kbf[database.CryptoKeyFeatureOIDCConvert], 1)
@@ -586,6 +691,14 @@ func requireKey(t *testing.T, key database.CryptoKey, feature database.CryptoKey
 	require.Equal(t, deletesAt.Time.UTC(), key.DeletesAt.Time.UTC())
 	require.Equal(t, sequence, key.Sequence)
 
+	// The NATS CA secret is a PEM bundle rather than hex-encoded bytes.
+	if key.Feature == database.CryptoKeyFeatureNATSCA {
+		cert, _, err := parseCASecret(key.Secret.String)
+		require.NoError(t, err)
+		require.True(t, cert.IsCA)
+		return
+	}
+
 	secret, err := hex.DecodeString(key.Secret.String)
 	require.NoError(t, err)