opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-22 05:05:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add high availability for multiple replicas (#4555)

Browse Source 
* feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* fixup! feat: HA tailnet coordinator

* remove printlns

* close all connections on coordinator

* impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* fixup! impelement high availability feature

* Add replicas

* Add DERP meshing to arbitrary addresses

* Move packages to highavailability folder

* Move coordinator to high availability package

* Add flags for HA

* Rename to replicasync

* Denest packages for replicas

* Add test for multiple replicas

* Fix coordination test

* Add HA to the helm chart

* Rename function pointer

* Add warnings for HA

* Add the ability to block endpoints

* Add flag to disable P2P connections

* Wow, I made the tests pass

* Add replicas endpoint

* Ensure close kills replica

* Update sql

* Add database latency to high availability

* Pipe TLS to DERP mesh

* Fix DERP mesh with TLS

* Add tests for TLS

* Fix replica sync TLS

* Fix RootCA for replica meshing

* Remove ID from replicasync

* Fix getting certificates for meshing

* Remove excessive locking

* Fix linting

* Store mesh key in the database

* Fix replica key for tests

* Fix types gen

* Fix unlocking unlocked

* Fix race in tests

* Update enterprise/derpmesh/derpmesh.go

Co-authored-by: Colin Adler <colin1adler@gmail.com>

* Rename to syncReplicas

* Reuse http client

* Delete old replicas on a CRON

* Fix race condition in connection tests

* Fix linting

* Fix nil type

* Move pubsub to in-memory for twenty test

* Add comment for configuration tweaking

* Fix leak with transport

* Fix close leak in derpmesh

* Fix race when creating server

* Remove handler update

* Skip test on Windows

* Fix DERP mesh test

* Wrap HTTP handler replacement in mutex

* Fix error message for relay

* Fix API handler for normal tests

* Fix speedtest

* Fix replica resend

* Fix derpmesh send

* Ping async

* Increase wait time of template version jobd

* Fix race when closing replica sync

* Add name to client

* Log the derpmap being used

* Don't connect if DERP is empty

* Improve agent coordinator logging

* Fix lock in coordinator

* Fix relay addr

* Fix race when updating durations

* Fix client publish race

* Run pubsub loop in a queue

* Store agent nodes in order

* Fix coordinator locking

* Check for closed pipe

Co-authored-by: Colin Adler <colin1adler@gmail.com>
This commit is contained in:
Kyle Carberry
2022-10-17 13:43:30 +00:00
committed by GitHub
co-authored by Colin Adler
parent dc3519e973
commit 2ba4a62a0d
76 changed files with 3437 additions and 404 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/agent_test.go
+6 -8
View File
@@ -7,8 +7,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"cdr.dev/slog"
"github.com/coder/coder/cli/clitest"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/provisioner/echo"
@@ -67,11 +65,11 @@ func TestWorkspaceAgent(t *testing.T) {
if assert.NotEmpty(t, workspace.LatestBuild.Resources) && assert.NotEmpty(t, resources[0].Agents) {
assert.NotEmpty(t, resources[0].Agents[0].Version)
}
dialer, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer dialer.Close()
require.Eventually(t, func() bool {
_, err := dialer.Ping()
_, err := dialer.Ping(ctx)
return err == nil
}, testutil.WaitMedium, testutil.IntervalFast)
cancelFunc()
@@ -128,11 +126,11 @@ func TestWorkspaceAgent(t *testing.T) {
if assert.NotEmpty(t, resources) && assert.NotEmpty(t, resources[0].Agents) {
assert.NotEmpty(t, resources[0].Agents[0].Version)
}
dialer, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer dialer.Close()
require.Eventually(t, func() bool {
_, err := dialer.Ping()
_, err := dialer.Ping(ctx)
return err == nil
}, testutil.WaitMedium, testutil.IntervalFast)
cancelFunc()
@@ -189,11 +187,11 @@ func TestWorkspaceAgent(t *testing.T) {
if assert.NotEmpty(t, resources) && assert.NotEmpty(t, resources[0].Agents) {
assert.NotEmpty(t, resources[0].Agents[0].Version)
}
dialer, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, resources[0].Agents[0].ID)
dialer, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer dialer.Close()
require.Eventually(t, func() bool {
_, err := dialer.Ping()
_, err := dialer.Ping(ctx)
return err == nil
}, testutil.WaitMedium, testutil.IntervalFast)
cancelFunc()
cli/config/file.go
+5
View File
@@ -13,6 +13,11 @@ func (r Root) Session() File {
return File(filepath.Join(string(r), "session"))
}
// ReplicaID is a unique identifier for the Coder server.
func (r Root) ReplicaID() File {
return File(filepath.Join(string(r), "replica_id"))
}
func (r Root) URL() File {
return File(filepath.Join(string(r), "url"))
}
cli/configssh_test.go
+1 -2
View File
@@ -19,7 +19,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
"github.com/coder/coder/agent"
@@ -115,7 +114,7 @@ func TestConfigSSH(t *testing.T) {
_ = agentCloser.Close()
}()
resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
agentConn, err := client.DialWorkspaceAgentTailnet(context.Background(), slog.Logger{}, resources[0].Agents[0].ID)
agentConn, err := client.DialWorkspaceAgent(context.Background(), resources[0].Agents[0].ID, nil)
require.NoError(t, err)
defer agentConn.Close()
cli/deployment/flags.go
+7
View File
@@ -85,6 +85,13 @@ func Flags() *codersdk.DeploymentFlags {
Description: "Addresses for STUN servers to establish P2P connections. Set empty to disable P2P connections.",
Default: []string{"stun.l.google.com:19302"},
},
DerpServerRelayAddress: &codersdk.StringFlag{
Name: "DERP Server Relay Address",
Flag: "derp-server-relay-address",
EnvVar: "CODER_DERP_SERVER_RELAY_ADDRESS",
Description: "An HTTP address that is accessible by other replicas to relay DERP traffic. Required for high availability.",
Enterprise: true,
},
DerpConfigURL: &codersdk.StringFlag{
Name: "DERP Config URL",
Flag: "derp-config-url",
cli/portforward.go
+2 -3
View File
@@ -16,7 +16,6 @@ import (
"github.com/spf13/cobra"
"golang.org/x/xerrors"
"cdr.dev/slog"
"github.com/coder/coder/agent"
"github.com/coder/coder/cli/cliflag"
"github.com/coder/coder/cli/cliui"
@@ -96,7 +95,7 @@ func portForward() *cobra.Command {
return xerrors.Errorf("await agent: %w", err)
}
conn, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, workspaceAgent.ID)
conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil)
if err != nil {
return err
}
@@ -156,7 +155,7 @@ func portForward() *cobra.Command {
case <-ticker.C:
}
_, err = conn.Ping()
_, err = conn.Ping(ctx)
if err != nil {
continue
}
cli/root.go
+4 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
@@ -100,8 +101,9 @@ func Core() []*cobra.Command {
}
func AGPL() []*cobra.Command {
all := append(Core(), Server(deployment.Flags(), func(_ context.Context, o *coderd.Options) (*coderd.API, error) {
return coderd.New(o), nil
all := append(Core(), Server(deployment.Flags(), func(_ context.Context, o *coderd.Options) (*coderd.API, io.Closer, error) {
api := coderd.New(o)
return api, api, nil
}))
return all
}
cli/server.go
+16 -7
View File
@@ -69,7 +69,7 @@ import (
)
// nolint:gocyclo
func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *coderd.Options) (*coderd.API, error)) *cobra.Command {
func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *coderd.Options) (*coderd.API, io.Closer, error)) *cobra.Command {
root := &cobra.Command{
Use: "server",
Short: "Start a Coder server",
@@ -167,9 +167,10 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code
}
defer listener.Close()
var tlsConfig *tls.Config
if dflags.TLSEnable.Value {
listener, err = configureServerTLS(
listener, dflags.TLSMinVersion.Value,
tlsConfig, err = configureTLS(
dflags.TLSMinVersion.Value,
dflags.TLSClientAuth.Value,
dflags.TLSCertFiles.Value,
dflags.TLSKeyFiles.Value,
@@ -178,6 +179,7 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code
if err != nil {
return xerrors.Errorf("configure tls: %w", err)
}
listener = tls.NewListener(listener, tlsConfig)
}
tcpAddr, valid := listener.Addr().(*net.TCPAddr)
@@ -328,6 +330,9 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code
Experimental: ExperimentalEnabled(cmd),
DeploymentFlags: dflags,
}
if tlsConfig != nil {
options.TLSCertificates = tlsConfig.Certificates
}
if dflags.OAuth2GithubClientSecret.Value != "" {
options.GithubOAuth2Config, err = configureGithubOAuth2(accessURLParsed,
@@ -471,11 +476,14 @@ func Server(dflags *codersdk.DeploymentFlags, newAPI func(context.Context, *code
), dflags.PromAddress.Value, "prometheus")()
}
coderAPI, err := newAPI(ctx, options)
// We use a separate closer so the Enterprise API
// can have it's own close functions. This is cleaner
// than abstracting the Coder API itself.
coderAPI, closer, err := newAPI(ctx, options)
if err != nil {
return err
}
defer coderAPI.Close()
defer closer.Close()
client := codersdk.New(localURL)
if dflags.TLSEnable.Value {
@@ -893,7 +901,7 @@ func loadCertificates(tlsCertFiles, tlsKeyFiles []string) ([]tls.Certificate, er
return certs, nil
}
func configureServerTLS(listener net.Listener, tlsMinVersion, tlsClientAuth string, tlsCertFiles, tlsKeyFiles []string, tlsClientCAFile string) (net.Listener, error) {
func configureTLS(tlsMinVersion, tlsClientAuth string, tlsCertFiles, tlsKeyFiles []string, tlsClientCAFile string) (*tls.Config, error) {
tlsConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
}
@@ -929,6 +937,7 @@ func configureServerTLS(listener net.Listener, tlsMinVersion, tlsClientAuth stri
if err != nil {
return nil, xerrors.Errorf("load certificates: %w", err)
}
tlsConfig.Certificates = certs
tlsConfig.GetCertificate = func(hi *tls.ClientHelloInfo) (*tls.Certificate, error) {
// If there's only one certificate, return it.
if len(certs) == 1 {
@@ -963,7 +972,7 @@ func configureServerTLS(listener net.Listener, tlsMinVersion, tlsClientAuth stri
tlsConfig.ClientCAs = caPool
}
return tls.NewListener(listener, tlsConfig), nil
return tlsConfig, nil
}
func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, allowSignups bool, allowOrgs []string, rawTeams []string, enterpriseBaseURL string) (*coderd.GithubOAuth2Config, error) {
cli/speedtest.go
+4 -2
View File
@@ -55,7 +55,9 @@ func speedtest() *cobra.Command {
if cliflag.IsSetBool(cmd, varVerbose) {
logger = logger.Leveled(slog.LevelDebug)
}
conn, err := client.DialWorkspaceAgentTailnet(ctx, logger, workspaceAgent.ID)
conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, &codersdk.DialWorkspaceAgentOptions{
Logger: logger,
})
if err != nil {
return err
}
@@ -68,7 +70,7 @@ func speedtest() *cobra.Command {
return ctx.Err()
case <-ticker.C:
}
dur, err := conn.Ping()
dur, err := conn.Ping(ctx)
if err != nil {
continue
}
cli/ssh.go
+1 -3
View File
@@ -20,8 +20,6 @@ import (
"golang.org/x/term"
"golang.org/x/xerrors"
"cdr.dev/slog"
"github.com/coder/coder/cli/cliflag"
"github.com/coder/coder/cli/cliui"
"github.com/coder/coder/coderd/autobuild/notify"
@@ -86,7 +84,7 @@ func ssh() *cobra.Command {
return xerrors.Errorf("await agent: %w", err)
}
conn, err := client.DialWorkspaceAgentTailnet(ctx, slog.Logger{}, workspaceAgent.ID)
conn, err := client.DialWorkspaceAgent(ctx, workspaceAgent.ID, nil)
if err != nil {
return err
}