feat: Add high availability for multiple replicas (#4555)

* feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* remove printlns

* close all connections on coordinator

* impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* Add replicas

* Add DERP meshing to arbitrary addresses

* Move packages to highavailability folder

* Move coordinator to high availability package

* Add flags for HA

* Rename to replicasync

* Denest packages for replicas

* Add test for multiple replicas

* Fix coordination test

* Add HA to the helm chart

* Rename function pointer

* Add warnings for HA

* Add the ability to block endpoints

* Add flag to disable P2P connections

* Wow, I made the tests pass

* Add replicas endpoint

* Ensure close kills replica

* Update sql

* Add database latency to high availability

* Pipe TLS to DERP mesh

* Fix DERP mesh with TLS

* Add tests for TLS

* Fix replica sync TLS

* Fix RootCA for replica meshing

* Remove ID from replicasync

* Fix getting certificates for meshing

* Remove excessive locking

* Fix linting

* Store mesh key in the database

* Fix replica key for tests

* Fix types gen

* Fix unlocking unlocked

* Fix race in tests

* Update enterprise/derpmesh/derpmesh.go

Co-authored-by: Colin Adler <colin1adler@gmail.com>

* Rename to syncReplicas

* Reuse http client

* Delete old replicas on a CRON

* Fix race condition in connection tests

* Fix linting

* Fix nil type

* Move pubsub to in-memory for twenty test

* Add comment for configuration tweaking

* Fix leak with transport

* Fix close leak in derpmesh

* Fix race when creating server

* Remove handler update

* Skip test on Windows

* Fix DERP mesh test

* Wrap HTTP handler replacement in mutex

* Fix error message for relay

* Fix API handler for normal tests

* Fix speedtest

* Fix replica resend

* Fix derpmesh send

* Ping async

* Increase wait time of template version jobd

* Fix race when closing replica sync

* Add name to client

* Log the derpmap being used

* Don't connect if DERP is empty

* Improve agent coordinator logging

* Fix lock in coordinator

* Fix relay addr

* Fix race when updating durations

* Fix client publish race

* Run pubsub loop in a queue

* Store agent nodes in order

* Fix coordinator locking

* Check for closed pipe

Co-authored-by: Colin Adler <colin1adler@gmail.com>

enterprise/coderd/authorize_test.go

@@ -28,7 +28,7 @@ func TestCheckACLPermissions(t *testing.T) {
 	// Create adminClient, member, and org adminClient
 	adminUser := coderdtest.CreateFirstUser(t, adminClient)
 	_ = coderdenttest.AddLicense(t, adminClient, coderdenttest.LicenseOptions{
-		TemplateRBACEnabled: true,
+		TemplateRBAC: true,
 	})
 
 	memberClient := coderdtest.CreateAnotherUser(t, adminClient, adminUser.OrganizationID)

enterprise/coderd/coderd.go

@@ -3,6 +3,8 @@ package coderd
 import (
 	"context"
 	"crypto/ed25519"
+	"crypto/tls"
+	"crypto/x509"
 	"net/http"
 	"sync"
 	"time"
@@ -23,6 +25,10 @@ import (
 	"github.com/coder/coder/enterprise/audit"
 	"github.com/coder/coder/enterprise/audit/backends"
 	"github.com/coder/coder/enterprise/coderd/license"
+	"github.com/coder/coder/enterprise/derpmesh"
+	"github.com/coder/coder/enterprise/replicasync"
+	"github.com/coder/coder/enterprise/tailnet"
+	agpltailnet "github.com/coder/coder/tailnet"
 )
 
 // New constructs an Enterprise coderd API instance.
@@ -47,6 +53,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
 		Options:                options,
 		cancelEntitlementsLoop: cancelFunc,
 	}
+
 	oauthConfigs := &httpmw.OAuth2Configs{
 		Github: options.GithubOAuth2Config,
 		OIDC:   options.OIDCConfig,
@@ -59,6 +66,10 @@ func New(ctx context.Context, options *Options) (*API, error) {
 
 	api.AGPL.APIHandler.Group(func(r chi.Router) {
 		r.Get("/entitlements", api.serveEntitlements)
+		r.Route("/replicas", func(r chi.Router) {
+			r.Use(apiKeyMiddleware)
+			r.Get("/", api.replicas)
+		})
 		r.Route("/licenses", func(r chi.Router) {
 			r.Use(apiKeyMiddleware)
 			r.Post("/", api.postLicense)
@@ -117,7 +128,40 @@ func New(ctx context.Context, options *Options) (*API, error) {
 		})
 	}
 
-	err := api.updateEntitlements(ctx)
+	meshRootCA := x509.NewCertPool()
+	for _, certificate := range options.TLSCertificates {
+		for _, certificatePart := range certificate.Certificate {
+			certificate, err := x509.ParseCertificate(certificatePart)
+			if err != nil {
+				return nil, xerrors.Errorf("parse certificate %s: %w", certificate.Subject.CommonName, err)
+			}
+			meshRootCA.AddCert(certificate)
+		}
+	}
+	// This TLS configuration spoofs access from the access URL hostname
+	// assuming that the certificates provided will cover that hostname.
+	//
+	// Replica sync and DERP meshing require accessing replicas via their
+	// internal IP addresses, and if TLS is configured we use the same
+	// certificates.
+	meshTLSConfig := &tls.Config{
+		MinVersion:   tls.VersionTLS12,
+		Certificates: options.TLSCertificates,
+		RootCAs:      meshRootCA,
+		ServerName:   options.AccessURL.Hostname(),
+	}
+	var err error
+	api.replicaManager, err = replicasync.New(ctx, options.Logger, options.Database, options.Pubsub, &replicasync.Options{
+		RelayAddress: options.DERPServerRelayAddress,
+		RegionID:     int32(options.DERPServerRegionID),
+		TLSConfig:    meshTLSConfig,
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("initialize replica: %w", err)
+	}
+	api.derpMesh = derpmesh.New(options.Logger.Named("derpmesh"), api.DERPServer, meshTLSConfig)
+
+	err = api.updateEntitlements(ctx)
 	if err != nil {
 		return nil, xerrors.Errorf("update entitlements: %w", err)
 	}
@@ -129,13 +173,17 @@ func New(ctx context.Context, options *Options) (*API, error) {
 type Options struct {
 	*coderd.Options
 
-	RBACEnabled  bool
+	RBAC         bool
 	AuditLogging bool
 	// Whether to block non-browser connections.
 	BrowserOnly        bool
 	SCIMAPIKey         []byte
 	UserWorkspaceQuota int
 
+	// Used for high availability.
+	DERPServerRelayAddress string
+	DERPServerRegionID     int
+
 	EntitlementsUpdateInterval time.Duration
 	Keys                       map[string]ed25519.PublicKey
 }
@@ -144,6 +192,11 @@ type API struct {
 	AGPL *coderd.API
 	*Options
 
+	// Detects multiple Coder replicas running at the same time.
+	replicaManager *replicasync.Manager
+	// Meshes DERP connections from multiple replicas.
+	derpMesh *derpmesh.Mesh
+
 	cancelEntitlementsLoop func()
 	entitlementsMu         sync.RWMutex
 	entitlements           codersdk.Entitlements
@@ -151,6 +204,8 @@ type API struct {
 
 func (api *API) Close() error {
 	api.cancelEntitlementsLoop()
+	_ = api.replicaManager.Close()
+	_ = api.derpMesh.Close()
 	return api.AGPL.Close()
 }
 
@@ -158,12 +213,13 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 	api.entitlementsMu.Lock()
 	defer api.entitlementsMu.Unlock()
 
-	entitlements, err := license.Entitlements(ctx, api.Database, api.Logger, api.Keys, map[string]bool{
-		codersdk.FeatureAuditLog:       api.AuditLogging,
-		codersdk.FeatureBrowserOnly:    api.BrowserOnly,
-		codersdk.FeatureSCIM:           len(api.SCIMAPIKey) != 0,
-		codersdk.FeatureWorkspaceQuota: api.UserWorkspaceQuota != 0,
-		codersdk.FeatureTemplateRBAC:   api.RBACEnabled,
+	entitlements, err := license.Entitlements(ctx, api.Database, api.Logger, len(api.replicaManager.All()), api.Keys, map[string]bool{
+		codersdk.FeatureAuditLog:         api.AuditLogging,
+		codersdk.FeatureBrowserOnly:      api.BrowserOnly,
+		codersdk.FeatureSCIM:             len(api.SCIMAPIKey) != 0,
+		codersdk.FeatureWorkspaceQuota:   api.UserWorkspaceQuota != 0,
+		codersdk.FeatureHighAvailability: api.DERPServerRelayAddress != "",
+		codersdk.FeatureTemplateRBAC:     api.RBAC,
 	})
 	if err != nil {
 		return err
@@ -209,6 +265,46 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 		api.AGPL.WorkspaceQuotaEnforcer.Store(&enforcer)
 	}
 
+	if changed, enabled := featureChanged(codersdk.FeatureHighAvailability); changed {
+		coordinator := agpltailnet.NewCoordinator()
+		if enabled {
+			haCoordinator, err := tailnet.NewCoordinator(api.Logger, api.Pubsub)
+			if err != nil {
+				api.Logger.Error(ctx, "unable to set up high availability coordinator", slog.Error(err))
+				// If we try to setup the HA coordinator and it fails, nothing
+				// is actually changing.
+				changed = false
+			} else {
+				coordinator = haCoordinator
+			}
+
+			api.replicaManager.SetCallback(func() {
+				addresses := make([]string, 0)
+				for _, replica := range api.replicaManager.Regional() {
+					addresses = append(addresses, replica.RelayAddress)
+				}
+				api.derpMesh.SetAddresses(addresses, false)
+				_ = api.updateEntitlements(ctx)
+			})
+		} else {
+			api.derpMesh.SetAddresses([]string{}, false)
+			api.replicaManager.SetCallback(func() {
+				// If the amount of replicas change, so should our entitlements.
+				// This is to display a warning in the UI if the user is unlicensed.
+				_ = api.updateEntitlements(ctx)
+			})
+		}
+
+		// Recheck changed in case the HA coordinator failed to set up.
+		if changed {
+			oldCoordinator := *api.AGPL.TailnetCoordinator.Swap(&coordinator)
+			err := oldCoordinator.Close()
+			if err != nil {
+				api.Logger.Error(ctx, "close old tailnet coordinator", slog.Error(err))
+			}
+		}
+	}
+
 	api.entitlements = entitlements
 
 	return nil

enterprise/coderd/coderd_test.go

@@ -41,9 +41,9 @@ func TestEntitlements(t *testing.T) {
 		})
 		_ = coderdtest.CreateFirstUser(t, client)
 		coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			UserLimit:           100,
-			AuditLog:            true,
-			TemplateRBACEnabled: true,
+			UserLimit:    100,
+			AuditLog:     true,
+			TemplateRBAC: true,
 		})
 		res, err := client.Entitlements(context.Background())
 		require.NoError(t, err)
@@ -85,7 +85,7 @@ func TestEntitlements(t *testing.T) {
 		assert.False(t, res.HasLicense)
 		al = res.Features[codersdk.FeatureAuditLog]
 		assert.Equal(t, codersdk.EntitlementNotEntitled, al.Entitlement)
-		assert.True(t, al.Enabled)
+		assert.False(t, al.Enabled)
 	})
 	t.Run("Pubsub", func(t *testing.T) {
 		t.Parallel()

enterprise/coderd/coderdenttest/coderdenttest.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/ed25519"
 	"crypto/rand"
+	"crypto/tls"
 	"io"
+	"net/http"
 	"testing"
 	"time"
 
@@ -60,19 +62,21 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
 	if options.Options == nil {
 		options.Options = &coderdtest.Options{}
 	}
-	srv, cancelFunc, oop := coderdtest.NewOptions(t, options.Options)
+	setHandler, cancelFunc, oop := coderdtest.NewOptions(t, options.Options)
 	coderAPI, err := coderd.New(context.Background(), &coderd.Options{
-		RBACEnabled:                true,
+		RBAC:                       true,
 		AuditLogging:               options.AuditLogging,
 		BrowserOnly:                options.BrowserOnly,
 		SCIMAPIKey:                 options.SCIMAPIKey,
+		DERPServerRelayAddress:     oop.AccessURL.String(),
+		DERPServerRegionID:         oop.DERPMap.RegionIDs()[0],
 		UserWorkspaceQuota:         options.UserWorkspaceQuota,
 		Options:                    oop,
 		EntitlementsUpdateInterval: options.EntitlementsUpdateInterval,
 		Keys:                       Keys,
 	})
 	assert.NoError(t, err)
-	srv.Config.Handler = coderAPI.AGPL.RootHandler
+	setHandler(coderAPI.AGPL.RootHandler)
 	var provisionerCloser io.Closer = nopcloser{}
 	if options.IncludeProvisionerDaemon {
 		provisionerCloser = coderdtest.NewProvisionerDaemon(t, coderAPI.AGPL)
@@ -83,22 +87,32 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
 		_ = provisionerCloser.Close()
 		_ = coderAPI.Close()
 	})
-	return codersdk.New(coderAPI.AccessURL), provisionerCloser, coderAPI
+	client := codersdk.New(coderAPI.AccessURL)
+	client.HTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				//nolint:gosec
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	return client, provisionerCloser, coderAPI
 }
 
 type LicenseOptions struct {
-	AccountType         string
-	AccountID           string
-	Trial               bool
-	AllFeatures         bool
-	GraceAt             time.Time
-	ExpiresAt           time.Time
-	UserLimit           int64
-	AuditLog            bool
-	BrowserOnly         bool
-	SCIM                bool
-	WorkspaceQuota      bool
-	TemplateRBACEnabled bool
+	AccountType      string
+	AccountID        string
+	Trial            bool
+	AllFeatures      bool
+	GraceAt          time.Time
+	ExpiresAt        time.Time
+	UserLimit        int64
+	AuditLog         bool
+	BrowserOnly      bool
+	SCIM             bool
+	WorkspaceQuota   bool
+	TemplateRBAC     bool
+	HighAvailability bool
 }
 
 // AddLicense generates a new license with the options provided and inserts it.
@@ -134,9 +148,13 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string {
 	if options.WorkspaceQuota {
 		workspaceQuota = 1
 	}
+	highAvailability := int64(0)
+	if options.HighAvailability {
+		highAvailability = 1
+	}
 
 	rbacEnabled := int64(0)
-	if options.TemplateRBACEnabled {
+	if options.TemplateRBAC {
 		rbacEnabled = 1
 	}
 
@@ -154,12 +172,13 @@ func GenerateLicense(t *testing.T, options LicenseOptions) string {
 		Version:        license.CurrentVersion,
 		AllFeatures:    options.AllFeatures,
 		Features: license.Features{
-			UserLimit:      options.UserLimit,
-			AuditLog:       auditLog,
-			BrowserOnly:    browserOnly,
-			SCIM:           scim,
-			WorkspaceQuota: workspaceQuota,
-			TemplateRBAC:   rbacEnabled,
+			UserLimit:        options.UserLimit,
+			AuditLog:         auditLog,
+			BrowserOnly:      browserOnly,
+			SCIM:             scim,
+			WorkspaceQuota:   workspaceQuota,
+			HighAvailability: highAvailability,
+			TemplateRBAC:     rbacEnabled,
 		},
 	}
 	tok := jwt.NewWithClaims(jwt.SigningMethodEdDSA, c)

enterprise/coderd/coderdenttest/coderdenttest_test.go

@@ -33,7 +33,7 @@ func TestAuthorizeAllEndpoints(t *testing.T) {
 	ctx, _ := testutil.Context(t)
 	admin := coderdtest.CreateFirstUser(t, client)
 	license := coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-		TemplateRBACEnabled: true,
+		TemplateRBAC: true,
 	})
 	group, err := client.CreateGroup(ctx, admin.OrganizationID, codersdk.CreateGroupRequest{
 		Name: "testgroup",
@@ -58,6 +58,10 @@ func TestAuthorizeAllEndpoints(t *testing.T) {
 		AssertAction: rbac.ActionRead,
 		AssertObject: rbac.ResourceLicense,
 	}
+	assertRoute["GET:/api/v2/replicas"] = coderdtest.RouteCheck{
+		AssertAction: rbac.ActionRead,
+		AssertObject: rbac.ResourceReplicas,
+	}
 	assertRoute["DELETE:/api/v2/licenses/{id}"] = coderdtest.RouteCheck{
 		AssertAction: rbac.ActionDelete,
 		AssertObject: rbac.ResourceLicense,

enterprise/coderd/groups_test.go

@@ -24,7 +24,7 @@ func TestCreateGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -43,7 +43,7 @@ func TestCreateGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		_, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -67,7 +67,7 @@ func TestCreateGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		_, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -90,7 +90,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -112,7 +112,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
 		_, user3 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -138,7 +138,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
 		_, user3 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -173,7 +173,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -197,7 +197,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -221,7 +221,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
 		ctx, _ := testutil.Context(t)
@@ -247,7 +247,7 @@ func TestPatchGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -276,7 +276,7 @@ func TestGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -296,7 +296,7 @@ func TestGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
 		_, user3 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -326,7 +326,7 @@ func TestGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		client1, _ := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
 
@@ -347,7 +347,7 @@ func TestGroup(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -380,7 +380,7 @@ func TestGroup(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -421,7 +421,7 @@ func TestGroups(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
 		_, user3 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -467,7 +467,7 @@ func TestDeleteGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		group1, err := client.CreateGroup(ctx, user.OrganizationID, codersdk.CreateGroupRequest{
@@ -492,7 +492,7 @@ func TestDeleteGroup(t *testing.T) {
 		user := coderdtest.CreateFirstUser(t, client)
 
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 		ctx, _ := testutil.Context(t)
 		err := client.DeleteGroup(ctx, user.OrganizationID)

enterprise/coderd/license/license.go

@@ -17,12 +17,20 @@ import (
 )
 
 // Entitlements processes licenses to return whether features are enabled or not.
-func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, keys map[string]ed25519.PublicKey, enablements map[string]bool) (codersdk.Entitlements, error) {
+func Entitlements(
+	ctx context.Context,
+	db database.Store,
+	logger slog.Logger,
+	replicaCount int,
+	keys map[string]ed25519.PublicKey,
+	enablements map[string]bool,
+) (codersdk.Entitlements, error) {
 	now := time.Now()
 	// Default all entitlements to be disabled.
 	entitlements := codersdk.Entitlements{
 		Features: map[string]codersdk.Feature{},
 		Warnings: []string{},
+		Errors:   []string{},
 	}
 	for _, featureName := range codersdk.FeatureNames {
 		entitlements.Features[featureName] = codersdk.Feature{
@@ -96,6 +104,12 @@ func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, ke
 				Enabled:     enablements[codersdk.FeatureWorkspaceQuota],
 			}
 		}
+		if claims.Features.HighAvailability > 0 {
+			entitlements.Features[codersdk.FeatureHighAvailability] = codersdk.Feature{
+				Entitlement: entitlement,
+				Enabled:     enablements[codersdk.FeatureHighAvailability],
+			}
+		}
 		if claims.Features.TemplateRBAC > 0 {
 			entitlements.Features[codersdk.FeatureTemplateRBAC] = codersdk.Feature{
 				Entitlement: entitlement,
@@ -132,6 +146,10 @@ func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, ke
 			if featureName == codersdk.FeatureUserLimit {
 				continue
 			}
+			// High availability has it's own warnings based on replica count!
+			if featureName == codersdk.FeatureHighAvailability {
+				continue
+			}
 			feature := entitlements.Features[featureName]
 			if !feature.Enabled {
 				continue
@@ -141,9 +159,6 @@ func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, ke
 			case codersdk.EntitlementNotEntitled:
 				entitlements.Warnings = append(entitlements.Warnings,
 					fmt.Sprintf("%s is enabled but your license is not entitled to this feature.", niceName))
-				// Disable the feature and add a warning...
-				feature.Enabled = false
-				entitlements.Features[featureName] = feature
 			case codersdk.EntitlementGracePeriod:
 				entitlements.Warnings = append(entitlements.Warnings,
 					fmt.Sprintf("%s is enabled but your license for this feature is expired.", niceName))
@@ -152,6 +167,32 @@ func Entitlements(ctx context.Context, db database.Store, logger slog.Logger, ke
 		}
 	}
 
+	if replicaCount > 1 {
+		feature := entitlements.Features[codersdk.FeatureHighAvailability]
+
+		switch feature.Entitlement {
+		case codersdk.EntitlementNotEntitled:
+			if entitlements.HasLicense {
+				entitlements.Errors = append(entitlements.Warnings,
+					"You have multiple replicas but your license is not entitled to high availability. You will be unable to connect to workspaces.")
+			} else {
+				entitlements.Errors = append(entitlements.Warnings,
+					"You have multiple replicas but high availability is an Enterprise feature. You will be unable to connect to workspaces.")
+			}
+		case codersdk.EntitlementGracePeriod:
+			entitlements.Warnings = append(entitlements.Warnings,
+				"You have multiple replicas but your license for high availability is expired. Reduce to one replica or workspace connections will stop working.")
+		}
+	}
+
+	for _, featureName := range codersdk.FeatureNames {
+		feature := entitlements.Features[featureName]
+		if feature.Entitlement == codersdk.EntitlementNotEntitled {
+			feature.Enabled = false
+			entitlements.Features[featureName] = feature
+		}
+	}
+
 	return entitlements, nil
 }
 
@@ -171,12 +212,13 @@ var (
 )
 
 type Features struct {
-	UserLimit      int64 `json:"user_limit"`
-	AuditLog       int64 `json:"audit_log"`
-	BrowserOnly    int64 `json:"browser_only"`
-	SCIM           int64 `json:"scim"`
-	WorkspaceQuota int64 `json:"workspace_quota"`
-	TemplateRBAC   int64 `json:"template_rbac"`
+	UserLimit        int64 `json:"user_limit"`
+	AuditLog         int64 `json:"audit_log"`
+	BrowserOnly      int64 `json:"browser_only"`
+	SCIM             int64 `json:"scim"`
+	WorkspaceQuota   int64 `json:"workspace_quota"`
+	TemplateRBAC     int64 `json:"template_rbac"`
+	HighAvailability int64 `json:"high_availability"`
 }
 
 type Claims struct {

enterprise/coderd/license/license_test.go

@@ -20,17 +20,18 @@ import (
 func TestEntitlements(t *testing.T) {
 	t.Parallel()
 	all := map[string]bool{
-		codersdk.FeatureAuditLog:       true,
-		codersdk.FeatureBrowserOnly:    true,
-		codersdk.FeatureSCIM:           true,
-		codersdk.FeatureWorkspaceQuota: true,
-		codersdk.FeatureTemplateRBAC:   true,
+		codersdk.FeatureAuditLog:         true,
+		codersdk.FeatureBrowserOnly:      true,
+		codersdk.FeatureSCIM:             true,
+		codersdk.FeatureWorkspaceQuota:   true,
+		codersdk.FeatureHighAvailability: true,
+		codersdk.FeatureTemplateRBAC:     true,
 	}
 
 	t.Run("Defaults", func(t *testing.T) {
 		t.Parallel()
 		db := databasefake.New()
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all)
 		require.NoError(t, err)
 		require.False(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -46,7 +47,7 @@ func TestEntitlements(t *testing.T) {
 			JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{}),
 			Exp: time.Now().Add(time.Hour),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{})
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -60,16 +61,17 @@ func TestEntitlements(t *testing.T) {
 		db := databasefake.New()
 		db.InsertLicense(context.Background(), database.InsertLicenseParams{
 			JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{
-				UserLimit:           100,
-				AuditLog:            true,
-				BrowserOnly:         true,
-				SCIM:                true,
-				WorkspaceQuota:      true,
-				TemplateRBACEnabled: true,
+				UserLimit:        100,
+				AuditLog:         true,
+				BrowserOnly:      true,
+				SCIM:             true,
+				WorkspaceQuota:   true,
+				HighAvailability: true,
+				TemplateRBAC:     true,
 			}),
 			Exp: time.Now().Add(time.Hour),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{})
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -82,18 +84,19 @@ func TestEntitlements(t *testing.T) {
 		db := databasefake.New()
 		db.InsertLicense(context.Background(), database.InsertLicenseParams{
 			JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{
-				UserLimit:           100,
-				AuditLog:            true,
-				BrowserOnly:         true,
-				SCIM:                true,
-				WorkspaceQuota:      true,
-				TemplateRBACEnabled: true,
-				GraceAt:             time.Now().Add(-time.Hour),
-				ExpiresAt:           time.Now().Add(time.Hour),
+				UserLimit:        100,
+				AuditLog:         true,
+				BrowserOnly:      true,
+				SCIM:             true,
+				WorkspaceQuota:   true,
+				HighAvailability: true,
+				TemplateRBAC:     true,
+				GraceAt:          time.Now().Add(-time.Hour),
+				ExpiresAt:        time.Now().Add(time.Hour),
 			}),
 			Exp: time.Now().Add(time.Hour),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all)
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all)
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -101,6 +104,9 @@ func TestEntitlements(t *testing.T) {
 			if featureName == codersdk.FeatureUserLimit {
 				continue
 			}
+			if featureName == codersdk.FeatureHighAvailability {
+				continue
+			}
 			niceName := strings.Title(strings.ReplaceAll(featureName, "_", " "))
 			require.Equal(t, codersdk.EntitlementGracePeriod, entitlements.Features[featureName].Entitlement)
 			require.Contains(t, entitlements.Warnings, fmt.Sprintf("%s is enabled but your license for this feature is expired.", niceName))
@@ -113,7 +119,7 @@ func TestEntitlements(t *testing.T) {
 			JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{}),
 			Exp: time.Now().Add(time.Hour),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all)
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all)
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -121,6 +127,9 @@ func TestEntitlements(t *testing.T) {
 			if featureName == codersdk.FeatureUserLimit {
 				continue
 			}
+			if featureName == codersdk.FeatureHighAvailability {
+				continue
+			}
 			niceName := strings.Title(strings.ReplaceAll(featureName, "_", " "))
 			// Ensures features that are not entitled are properly disabled.
 			require.False(t, entitlements.Features[featureName].Enabled)
@@ -139,7 +148,7 @@ func TestEntitlements(t *testing.T) {
 			}),
 			Exp: time.Now().Add(time.Hour),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{})
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.Contains(t, entitlements.Warnings, "Your deployment has 2 active users but is only licensed for 1.")
@@ -161,7 +170,7 @@ func TestEntitlements(t *testing.T) {
 			}),
 			Exp: time.Now().Add(time.Hour),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{})
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.Empty(t, entitlements.Warnings)
@@ -184,7 +193,7 @@ func TestEntitlements(t *testing.T) {
 			}),
 		})
 
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, map[string]bool{})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, map[string]bool{})
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -199,7 +208,7 @@ func TestEntitlements(t *testing.T) {
 				AllFeatures: true,
 			}),
 		})
-		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, coderdenttest.Keys, all)
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 1, coderdenttest.Keys, all)
 		require.NoError(t, err)
 		require.True(t, entitlements.HasLicense)
 		require.False(t, entitlements.Trial)
@@ -211,4 +220,52 @@ func TestEntitlements(t *testing.T) {
 			require.Equal(t, codersdk.EntitlementEntitled, entitlements.Features[featureName].Entitlement)
 		}
 	})
+
+	t.Run("MultipleReplicasNoLicense", func(t *testing.T) {
+		t.Parallel()
+		db := databasefake.New()
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, all)
+		require.NoError(t, err)
+		require.False(t, entitlements.HasLicense)
+		require.Len(t, entitlements.Errors, 1)
+		require.Equal(t, "You have multiple replicas but high availability is an Enterprise feature. You will be unable to connect to workspaces.", entitlements.Errors[0])
+	})
+
+	t.Run("MultipleReplicasNotEntitled", func(t *testing.T) {
+		t.Parallel()
+		db := databasefake.New()
+		db.InsertLicense(context.Background(), database.InsertLicenseParams{
+			Exp: time.Now().Add(time.Hour),
+			JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{
+				AuditLog: true,
+			}),
+		})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, map[string]bool{
+			codersdk.FeatureHighAvailability: true,
+		})
+		require.NoError(t, err)
+		require.True(t, entitlements.HasLicense)
+		require.Len(t, entitlements.Errors, 1)
+		require.Equal(t, "You have multiple replicas but your license is not entitled to high availability. You will be unable to connect to workspaces.", entitlements.Errors[0])
+	})
+
+	t.Run("MultipleReplicasGrace", func(t *testing.T) {
+		t.Parallel()
+		db := databasefake.New()
+		db.InsertLicense(context.Background(), database.InsertLicenseParams{
+			JWT: coderdenttest.GenerateLicense(t, coderdenttest.LicenseOptions{
+				HighAvailability: true,
+				GraceAt:          time.Now().Add(-time.Hour),
+				ExpiresAt:        time.Now().Add(time.Hour),
+			}),
+			Exp: time.Now().Add(time.Hour),
+		})
+		entitlements, err := license.Entitlements(context.Background(), db, slog.Logger{}, 2, coderdenttest.Keys, map[string]bool{
+			codersdk.FeatureHighAvailability: true,
+		})
+		require.NoError(t, err)
+		require.True(t, entitlements.HasLicense)
+		require.Len(t, entitlements.Warnings, 1)
+		require.Equal(t, "You have multiple replicas but your license for high availability is expired. Reduce to one replica or workspace connections will stop working.", entitlements.Warnings[0])
+	})
 }

enterprise/coderd/licenses_test.go

@@ -78,21 +78,21 @@ func TestGetLicense(t *testing.T) {
 		defer cancel()
 
 		coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			AccountID:           "testing",
-			AuditLog:            true,
-			SCIM:                true,
-			BrowserOnly:         true,
-			TemplateRBACEnabled: true,
+			AccountID:    "testing",
+			AuditLog:     true,
+			SCIM:         true,
+			BrowserOnly:  true,
+			TemplateRBAC: true,
 		})
 
 		coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			AccountID:           "testing2",
-			AuditLog:            true,
-			SCIM:                true,
-			BrowserOnly:         true,
-			Trial:               true,
-			UserLimit:           200,
-			TemplateRBACEnabled: false,
+			AccountID:    "testing2",
+			AuditLog:     true,
+			SCIM:         true,
+			BrowserOnly:  true,
+			Trial:        true,
+			UserLimit:    200,
+			TemplateRBAC: false,
 		})
 
 		licenses, err := client.Licenses(ctx)
@@ -101,23 +101,25 @@ func TestGetLicense(t *testing.T) {
 		assert.Equal(t, int32(1), licenses[0].ID)
 		assert.Equal(t, "testing", licenses[0].Claims["account_id"])
 		assert.Equal(t, map[string]interface{}{
-			codersdk.FeatureUserLimit:      json.Number("0"),
-			codersdk.FeatureAuditLog:       json.Number("1"),
-			codersdk.FeatureSCIM:           json.Number("1"),
-			codersdk.FeatureBrowserOnly:    json.Number("1"),
-			codersdk.FeatureWorkspaceQuota: json.Number("0"),
-			codersdk.FeatureTemplateRBAC:   json.Number("1"),
+			codersdk.FeatureUserLimit:        json.Number("0"),
+			codersdk.FeatureAuditLog:         json.Number("1"),
+			codersdk.FeatureSCIM:             json.Number("1"),
+			codersdk.FeatureBrowserOnly:      json.Number("1"),
+			codersdk.FeatureWorkspaceQuota:   json.Number("0"),
+			codersdk.FeatureHighAvailability: json.Number("0"),
+			codersdk.FeatureTemplateRBAC:     json.Number("1"),
 		}, licenses[0].Claims["features"])
 		assert.Equal(t, int32(2), licenses[1].ID)
 		assert.Equal(t, "testing2", licenses[1].Claims["account_id"])
 		assert.Equal(t, true, licenses[1].Claims["trial"])
 		assert.Equal(t, map[string]interface{}{
-			codersdk.FeatureUserLimit:      json.Number("200"),
-			codersdk.FeatureAuditLog:       json.Number("1"),
-			codersdk.FeatureSCIM:           json.Number("1"),
-			codersdk.FeatureBrowserOnly:    json.Number("1"),
-			codersdk.FeatureWorkspaceQuota: json.Number("0"),
-			codersdk.FeatureTemplateRBAC:   json.Number("0"),
+			codersdk.FeatureUserLimit:        json.Number("200"),
+			codersdk.FeatureAuditLog:         json.Number("1"),
+			codersdk.FeatureSCIM:             json.Number("1"),
+			codersdk.FeatureBrowserOnly:      json.Number("1"),
+			codersdk.FeatureWorkspaceQuota:   json.Number("0"),
+			codersdk.FeatureHighAvailability: json.Number("0"),
+			codersdk.FeatureTemplateRBAC:     json.Number("0"),
 		}, licenses[1].Claims["features"])
 	})
 }

enterprise/coderd/replicas.go

@@ -0,0 +1,37 @@
+package coderd
+
+import (
+	"net/http"
+
+	"github.com/coder/coder/coderd/database"
+	"github.com/coder/coder/coderd/httpapi"
+	"github.com/coder/coder/coderd/rbac"
+	"github.com/coder/coder/codersdk"
+)
+
+// replicas returns the number of replicas that are active in Coder.
+func (api *API) replicas(rw http.ResponseWriter, r *http.Request) {
+	if !api.AGPL.Authorize(r, rbac.ActionRead, rbac.ResourceReplicas) {
+		httpapi.ResourceNotFound(rw)
+		return
+	}
+
+	replicas := api.replicaManager.All()
+	res := make([]codersdk.Replica, 0, len(replicas))
+	for _, replica := range replicas {
+		res = append(res, convertReplica(replica))
+	}
+	httpapi.Write(r.Context(), rw, http.StatusOK, res)
+}
+
+func convertReplica(replica database.Replica) codersdk.Replica {
+	return codersdk.Replica{
+		ID:              replica.ID,
+		Hostname:        replica.Hostname,
+		CreatedAt:       replica.CreatedAt,
+		RelayAddress:    replica.RelayAddress,
+		RegionID:        replica.RegionID,
+		Error:           replica.Error,
+		DatabaseLatency: replica.DatabaseLatency,
+	}
+}

enterprise/coderd/replicas_test.go

@@ -0,0 +1,138 @@
+package coderd_test
+
+import (
+	"context"
+	"crypto/tls"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+
+	"github.com/coder/coder/coderd/coderdtest"
+	"github.com/coder/coder/coderd/database/dbtestutil"
+	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/enterprise/coderd/coderdenttest"
+	"github.com/coder/coder/testutil"
+)
+
+func TestReplicas(t *testing.T) {
+	t.Parallel()
+	t.Run("ErrorWithoutLicense", func(t *testing.T) {
+		t.Parallel()
+		db, pubsub := dbtestutil.NewDB(t)
+		firstClient := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				IncludeProvisionerDaemon: true,
+				Database:                 db,
+				Pubsub:                   pubsub,
+			},
+		})
+		_ = coderdtest.CreateFirstUser(t, firstClient)
+		secondClient, _, secondAPI := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				Database: db,
+				Pubsub:   pubsub,
+			},
+		})
+		secondClient.SessionToken = firstClient.SessionToken
+		ents, err := secondClient.Entitlements(context.Background())
+		require.NoError(t, err)
+		require.Len(t, ents.Errors, 1)
+		_ = secondAPI.Close()
+
+		ents, err = firstClient.Entitlements(context.Background())
+		require.NoError(t, err)
+		require.Len(t, ents.Warnings, 0)
+	})
+	t.Run("ConnectAcrossMultiple", func(t *testing.T) {
+		t.Parallel()
+		db, pubsub := dbtestutil.NewDB(t)
+		firstClient := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				IncludeProvisionerDaemon: true,
+				Database:                 db,
+				Pubsub:                   pubsub,
+			},
+		})
+		firstUser := coderdtest.CreateFirstUser(t, firstClient)
+		coderdenttest.AddLicense(t, firstClient, coderdenttest.LicenseOptions{
+			HighAvailability: true,
+		})
+
+		secondClient := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				Database: db,
+				Pubsub:   pubsub,
+			},
+		})
+		secondClient.SessionToken = firstClient.SessionToken
+		replicas, err := secondClient.Replicas(context.Background())
+		require.NoError(t, err)
+		require.Len(t, replicas, 2)
+
+		_, agent := setupWorkspaceAgent(t, firstClient, firstUser, 0)
+		conn, err := secondClient.DialWorkspaceAgent(context.Background(), agent.ID, &codersdk.DialWorkspaceAgentOptions{
+			BlockEndpoints: true,
+			Logger:         slogtest.Make(t, nil).Leveled(slog.LevelDebug),
+		})
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitShort)
+			defer cancelFunc()
+			_, err = conn.Ping(ctx)
+			return err == nil
+		}, testutil.WaitLong, testutil.IntervalFast)
+		_ = conn.Close()
+	})
+	t.Run("ConnectAcrossMultipleTLS", func(t *testing.T) {
+		t.Parallel()
+		db, pubsub := dbtestutil.NewDB(t)
+		certificates := []tls.Certificate{testutil.GenerateTLSCertificate(t, "localhost")}
+		firstClient := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				IncludeProvisionerDaemon: true,
+				Database:                 db,
+				Pubsub:                   pubsub,
+				TLSCertificates:          certificates,
+			},
+		})
+		firstUser := coderdtest.CreateFirstUser(t, firstClient)
+		coderdenttest.AddLicense(t, firstClient, coderdenttest.LicenseOptions{
+			HighAvailability: true,
+		})
+
+		secondClient := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				Database:        db,
+				Pubsub:          pubsub,
+				TLSCertificates: certificates,
+			},
+		})
+		secondClient.SessionToken = firstClient.SessionToken
+		replicas, err := secondClient.Replicas(context.Background())
+		require.NoError(t, err)
+		require.Len(t, replicas, 2)
+
+		_, agent := setupWorkspaceAgent(t, firstClient, firstUser, 0)
+		conn, err := secondClient.DialWorkspaceAgent(context.Background(), agent.ID, &codersdk.DialWorkspaceAgentOptions{
+			BlockEndpoints: true,
+			Logger:         slogtest.Make(t, nil).Named("client").Leveled(slog.LevelDebug),
+		})
+		require.NoError(t, err)
+		require.Eventually(t, func() bool {
+			ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.IntervalSlow)
+			defer cancelFunc()
+			_, err = conn.Ping(ctx)
+			return err == nil
+		}, testutil.WaitLong, testutil.IntervalFast)
+		_ = conn.Close()
+		replicas, err = secondClient.Replicas(context.Background())
+		require.NoError(t, err)
+		require.Len(t, replicas, 2)
+		for _, replica := range replicas {
+			require.Empty(t, replica.Error)
+		}
+	})
+}

enterprise/coderd/templates_test.go

@@ -23,7 +23,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -64,7 +64,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -88,7 +88,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		client1, _ := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -138,7 +138,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -176,7 +176,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -214,7 +214,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
@@ -262,7 +262,7 @@ func TestTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		client1, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -318,7 +318,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -361,7 +361,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -422,7 +422,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
@@ -447,7 +447,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
@@ -472,7 +472,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		_, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -498,7 +498,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		client2, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -533,7 +533,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		client2, user2 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -575,7 +575,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
@@ -597,7 +597,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		client1, user1 := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)
@@ -662,7 +662,7 @@ func TestUpdateTemplateACL(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		client1, _ := coderdtest.CreateAnotherUserWithUser(t, client, user.OrganizationID)

enterprise/coderd/workspaceagents_test.go

@@ -2,6 +2,7 @@ package coderd_test
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net/http"
 	"testing"
@@ -9,7 +10,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
-	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/agent"
 	"github.com/coder/coder/coderd/coderdtest"
@@ -42,7 +42,7 @@ func TestBlockNonBrowser(t *testing.T) {
 			BrowserOnly: true,
 		})
 		_, agent := setupWorkspaceAgent(t, client, user, 0)
-		_, err := client.DialWorkspaceAgentTailnet(context.Background(), slog.Logger{}, agent.ID)
+		_, err := client.DialWorkspaceAgent(context.Background(), agent.ID, nil)
 		var apiErr *codersdk.Error
 		require.ErrorAs(t, err, &apiErr)
 		require.Equal(t, http.StatusConflict, apiErr.StatusCode())
@@ -59,7 +59,7 @@ func TestBlockNonBrowser(t *testing.T) {
 			BrowserOnly: false,
 		})
 		_, agent := setupWorkspaceAgent(t, client, user, 0)
-		conn, err := client.DialWorkspaceAgentTailnet(context.Background(), slog.Logger{}, agent.ID)
+		conn, err := client.DialWorkspaceAgent(context.Background(), agent.ID, nil)
 		require.NoError(t, err)
 		_ = conn.Close()
 	})
@@ -109,6 +109,14 @@ func setupWorkspaceAgent(t *testing.T, client *codersdk.Client, user codersdk.Cr
 	workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
 	coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
 	agentClient := codersdk.New(client.URL)
+	agentClient.HTTPClient = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				//nolint:gosec
+				InsecureSkipVerify: true,
+			},
+		},
+	}
 	agentClient.SessionToken = authToken
 	agentCloser := agent.New(agent.Options{
 		FetchMetadata:     agentClient.WorkspaceAgentMetadata,

enterprise/coderd/workspaces_test.go

@@ -26,7 +26,7 @@ func TestCreateWorkspace(t *testing.T) {
 		client := coderdenttest.New(t, nil)
 		user := coderdtest.CreateFirstUser(t, client)
 		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
-			TemplateRBACEnabled: true,
+			TemplateRBAC: true,
 		})
 
 		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)