feat: Add high availability for multiple replicas (#4555)

* feat: HA tailnet coordinator * fixup! feat: HA tailnet coordinator * fixup! feat: HA tailnet coordinator * remove printlns * close all connections on coordinator * impelement high availability feature * fixup! impelement high availability feature * fixup! impelement high availability feature * fixup! impelement high availability feature * fixup! impelement high availability feature * Add replicas * Add DERP meshing to arbitrary addresses * Move packages to highavailability folder * Move coordinator to high availability package * Add flags for HA * Rename to replicasync * Denest packages for replicas * Add test for multiple replicas * Fix coordination test * Add HA to the helm chart * Rename function pointer * Add warnings for HA * Add the ability to block endpoints * Add flag to disable P2P connections * Wow, I made the tests pass * Add replicas endpoint * Ensure close kills replica * Update sql * Add database latency to high availability * Pipe TLS to DERP mesh * Fix DERP mesh with TLS * Add tests for TLS * Fix replica sync TLS * Fix RootCA for replica meshing * Remove ID from replicasync * Fix getting certificates for meshing * Remove excessive locking * Fix linting * Store mesh key in the database * Fix replica key for tests * Fix types gen * Fix unlocking unlocked * Fix race in tests * Update enterprise/derpmesh/derpmesh.go Co-authored-by: Colin Adler <colin1adler@gmail.com> * Rename to syncReplicas * Reuse http client * Delete old replicas on a CRON * Fix race condition in connection tests * Fix linting * Fix nil type * Move pubsub to in-memory for twenty test * Add comment for configuration tweaking * Fix leak with transport * Fix close leak in derpmesh * Fix race when creating server * Remove handler update * Skip test on Windows * Fix DERP mesh test * Wrap HTTP handler replacement in mutex * Fix error message for relay * Fix API handler for normal tests * Fix speedtest * Fix replica resend * Fix derpmesh send * Ping async * Increase wait time of template version jobd * Fix race when closing replica sync * Add name to client * Log the derpmap being used * Don't connect if DERP is empty * Improve agent coordinator logging * Fix lock in coordinator * Fix relay addr * Fix race when updating durations * Fix client publish race * Run pubsub loop in a queue * Store agent nodes in order * Fix coordinator locking * Check for closed pipe Co-authored-by: Colin Adler <colin1adler@gmail.com>
2026-09-24 15:04:27 +08:00 · 2022-10-17 13:43:30 +00:00
parent dc3519e973
commit 2ba4a62a0d
76 changed files with 3437 additions and 404 deletions
@@ -2,11 +2,20 @@ package cli

 import (
 	"context"
+	"database/sql"
+	"errors"
+	"io"
+	"net/url"

 	"github.com/spf13/cobra"
+	"golang.org/x/xerrors"
+	"tailscale.com/derp"
+	"tailscale.com/types/key"

 	"github.com/coder/coder/cli/deployment"
+	"github.com/coder/coder/cryptorand"
 	"github.com/coder/coder/enterprise/coderd"
+	"github.com/coder/coder/tailnet"

 	agpl "github.com/coder/coder/cli"
 	agplcoderd "github.com/coder/coder/coderd"
@@ -14,23 +23,49 @@ import (

 func server() *cobra.Command {
 	dflags := deployment.Flags()
-	cmd := agpl.Server(dflags, func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, error) {
+	cmd := agpl.Server(dflags, func(ctx context.Context, options *agplcoderd.Options) (*agplcoderd.API, io.Closer, error) {
+		if dflags.DerpServerRelayAddress.Value != "" {
+			_, err := url.Parse(dflags.DerpServerRelayAddress.Value)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("derp-server-relay-address must be a valid HTTP URL: %w", err)
+			}
+		}
+
+		options.DERPServer = derp.NewServer(key.NewNode(), tailnet.Logger(options.Logger.Named("derp")))
+		meshKey, err := options.Database.GetDERPMeshKey(ctx)
+		if err != nil {
+			if !errors.Is(err, sql.ErrNoRows) {
+				return nil, nil, xerrors.Errorf("get mesh key: %w", err)
+			}
+			meshKey, err = cryptorand.String(32)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("generate mesh key: %w", err)
+			}
+			err = options.Database.InsertDERPMeshKey(ctx, meshKey)
+			if err != nil {
+				return nil, nil, xerrors.Errorf("insert mesh key: %w", err)
+			}
+		}
+		options.DERPServer.SetMeshKey(meshKey)
+
 		o := &coderd.Options{
-			AuditLogging:       dflags.AuditLogging.Value,
-			BrowserOnly:        dflags.BrowserOnly.Value,
-			SCIMAPIKey:         []byte(dflags.SCIMAuthHeader.Value),
-			UserWorkspaceQuota: dflags.UserWorkspaceQuota.Value,
-			RBACEnabled:        true,
-			Options:            options,
+			AuditLogging:           dflags.AuditLogging.Value,
+			BrowserOnly:            dflags.BrowserOnly.Value,
+			SCIMAPIKey:             []byte(dflags.SCIMAuthHeader.Value),
+			UserWorkspaceQuota:     dflags.UserWorkspaceQuota.Value,
+			RBAC:                   true,
+			DERPServerRelayAddress: dflags.DerpServerRelayAddress.Value,
+			DERPServerRegionID:     dflags.DerpServerRegionID.Value,
+
+			Options: options,
 		}
 		api, err := coderd.New(ctx, o)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
-		return api.AGPL, nil
+		return api.AGPL, api, nil
 	})

 	deployment.AttachFlags(cmd.Flags(), dflags, true)
-
 	return cmd
 }