opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: add derpserver to wsproxy, add proxies to derpmap (#7311)

Browse Source
This commit is contained in:
Dean Sheather
2023-07-27 02:21:04 +10:00
committed by GitHub
parent 70692c2e4e
commit 2f0a9996e7
58 changed files with 3001 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
enterprise/wsproxy/wsproxy.go
+140 -30
View File
@@ -2,9 +2,12 @@ package wsproxy
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"net/url"
"os"
"reflect"
"regexp"
"strings"
@@ -12,9 +15,13 @@ import (
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/hashicorp/go-multierror"
"github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/trace"
"golang.org/x/xerrors"
"tailscale.com/derp"
"tailscale.com/derp/derphttp"
"tailscale.com/types/key"
"cdr.dev/slog"
"github.com/coder/coder/buildinfo"
@@ -25,9 +32,10 @@ import (
"github.com/coder/coder/coderd/workspaceapps"
"github.com/coder/coder/coderd/wsconncache"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/enterprise/derpmesh"
"github.com/coder/coder/enterprise/wsproxy/wsproxysdk"
"github.com/coder/coder/site"
agpl "github.com/coder/coder/tailnet"
"github.com/coder/coder/tailnet"
)
type Options struct {
@@ -52,14 +60,16 @@ type Options struct {
// options.AppHostname is set.
AppHostnameRegex *regexp.Regexp
RealIPConfig *httpmw.RealIPConfig
RealIPConfig *httpmw.RealIPConfig
Tracing trace.TracerProvider
PrometheusRegistry *prometheus.Registry
TLSCertificates []tls.Certificate
APIRateLimit int
SecureAuthCookie bool
DisablePathApps bool
APIRateLimit int
SecureAuthCookie bool
DisablePathApps bool
DERPEnabled bool
DERPServerRelayAddress string
ProxySessionToken string
// AllowAllCors will set all CORs headers to '*'.
@@ -103,12 +113,14 @@ type Server struct {
// the moon's token.
SDKClient *wsproxysdk.Client
// TODO: Missing:
// - derpserver
// DERP
derpMesh *derpmesh.Mesh
// Used for graceful shutdown. Required for the dialer.
ctx context.Context
cancel context.CancelFunc
ctx context.Context
cancel context.CancelFunc
derpCloseFunc func()
registerDone <-chan struct{}
}
// New creates a new workspace proxy server. This requires a primary coderd
@@ -143,21 +155,33 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
return nil, xerrors.Errorf("%q is a workspace proxy, not a primary coderd instance", opts.DashboardURL)
}
regResp, err := client.RegisterWorkspaceProxy(ctx, wsproxysdk.RegisterWorkspaceProxyRequest{
AccessURL: opts.AccessURL.String(),
WildcardHostname: opts.AppHostname,
})
if err != nil {
return nil, xerrors.Errorf("register proxy: %w", err)
meshRootCA := x509.NewCertPool()
for _, certificate := range opts.TLSCertificates {
for _, certificatePart := range certificate.Certificate {
certificate, err := x509.ParseCertificate(certificatePart)
if err != nil {
return nil, xerrors.Errorf("parse certificate %s: %w", certificate.Subject.CommonName, err)
}
meshRootCA.AddCert(certificate)
}
}
// This TLS configuration spoofs access from the access URL hostname
// assuming that the certificates provided will cover that hostname.
//
// Replica sync and DERP meshing require accessing replicas via their
// internal IP addresses, and if TLS is configured we use the same
// certificates.
meshTLSConfig := &tls.Config{
MinVersion: tls.VersionTLS12,
Certificates: opts.TLSCertificates,
RootCAs: meshRootCA,
ServerName: opts.AccessURL.Hostname(),
}
secKey, err := workspaceapps.KeyFromString(regResp.AppSecurityKey)
if err != nil {
return nil, xerrors.Errorf("parse app security key: %w", err)
}
derpServer := derp.NewServer(key.NewNode(), tailnet.Logger(opts.Logger.Named("derp")))
r := chi.NewRouter()
ctx, cancel := context.WithCancel(context.Background())
r := chi.NewRouter()
s := &Server{
Options: opts,
Handler: r,
@@ -166,11 +190,50 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
TracerProvider: opts.Tracing,
PrometheusRegistry: opts.PrometheusRegistry,
SDKClient: client,
derpMesh: derpmesh.New(opts.Logger.Named("derpmesh"), derpServer, meshTLSConfig),
ctx: ctx,
cancel: cancel,
}
connInfo, err := client.SDKClient.WorkspaceAgentConnectionInfo(ctx)
// Register the workspace proxy with the primary coderd instance and start a
// goroutine to periodically re-register.
replicaID := uuid.New()
osHostname, err := os.Hostname()
if err != nil {
return nil, xerrors.Errorf("get OS hostname: %w", err)
}
regResp, registerDone, err := client.RegisterWorkspaceProxyLoop(ctx, wsproxysdk.RegisterWorkspaceProxyLoopOpts{
Logger: opts.Logger,
Request: wsproxysdk.RegisterWorkspaceProxyRequest{
AccessURL: opts.AccessURL.String(),
WildcardHostname: opts.AppHostname,
DerpEnabled: opts.DERPEnabled,
ReplicaID: replicaID,
ReplicaHostname: osHostname,
ReplicaError: "",
ReplicaRelayAddress: opts.DERPServerRelayAddress,
Version: buildinfo.Version(),
},
MutateFn: s.mutateRegister,
CallbackFn: s.handleRegister,
FailureFn: s.handleRegisterFailure,
})
if err != nil {
return nil, xerrors.Errorf("register proxy: %w", err)
}
s.registerDone = registerDone
err = s.handleRegister(ctx, regResp)
if err != nil {
return nil, xerrors.Errorf("handle register: %w", err)
}
derpServer.SetMeshKey(regResp.DERPMeshKey)
secKey, err := workspaceapps.KeyFromString(regResp.AppSecurityKey)
if err != nil {
return nil, xerrors.Errorf("parse app security key: %w", err)
}
connInfo, err := client.SDKClient.WorkspaceAgentConnectionInfoGeneric(ctx)
if err != nil {
return nil, xerrors.Errorf("get derpmap: %w", err)
}
@@ -216,6 +279,9 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
SecureAuthCookie: opts.SecureAuthCookie,
}
derpHandler := derphttp.Handler(derpServer)
derpHandler, s.derpCloseFunc = tailnet.WithWebsocketSupport(derpServer, derpHandler)
// The primary coderd dashboard needs to make some GET requests to
// the workspace proxies to check latency.
corsMW := httpmw.Cors(opts.AllowAllCors, opts.DashboardURL.String())
@@ -266,6 +332,14 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
s.AppServer.Attach(r)
})
r.Route("/derp", func(r chi.Router) {
r.Get("/", derpHandler.ServeHTTP)
// This is used when UDP is blocked, and latency must be checked via HTTP(s).
r.Get("/latency-check", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
})
})
r.Get("/api/v2/buildinfo", s.buildInfo)
r.Get("/healthz", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("OK")) })
// TODO: @emyrk should this be authenticated or debounced?
@@ -295,20 +369,56 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
func (s *Server) Close() error {
s.cancel()
// A timeout to prevent the SDK from blocking the server shutdown.
tmp, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
_ = s.SDKClient.WorkspaceProxyGoingAway(tmp)
_ = s.AppServer.AgentProvider.Close()
return s.AppServer.Close()
var err error
registerDoneWaitTicker := time.NewTicker(11 * time.Second) // the attempt timeout is 10s
select {
case <-registerDoneWaitTicker.C:
err = multierror.Append(err, xerrors.New("timed out waiting for registerDone"))
case <-s.registerDone:
}
s.derpCloseFunc()
appServerErr := s.AppServer.Close()
if appServerErr != nil {
err = multierror.Append(err, appServerErr)
}
agentProviderErr := s.AppServer.AgentProvider.Close()
if agentProviderErr != nil {
err = multierror.Append(err, agentProviderErr)
}
s.SDKClient.SDKClient.HTTPClient.CloseIdleConnections()
return err
}
func (s *Server) DialWorkspaceAgent(id uuid.UUID) (*codersdk.WorkspaceAgentConn, error) {
return s.SDKClient.DialWorkspaceAgent(s.ctx, id, nil)
}
func (s *Server) DialCoordinator(ctx context.Context) (agpl.MultiAgentConn, error) {
func (*Server) mutateRegister(_ *wsproxysdk.RegisterWorkspaceProxyRequest) {
// TODO: we should probably ping replicas similarly to the replicasync
// package in the primary and update req.ReplicaError accordingly.
}
func (s *Server) handleRegister(_ context.Context, res wsproxysdk.RegisterWorkspaceProxyResponse) error {
addresses := make([]string, len(res.SiblingReplicas))
for i, replica := range res.SiblingReplicas {
addresses[i] = replica.RelayAddress
}
s.derpMesh.SetAddresses(addresses, false)
return nil
}
func (s *Server) handleRegisterFailure(err error) {
if s.ctx.Err() != nil {
return
}
s.Logger.Fatal(s.ctx,
"failed to periodically re-register workspace proxy with primary Coder deployment",
slog.Error(err),
)
}
func (s *Server) DialCoordinator(ctx context.Context) (tailnet.MultiAgentConn, error) {
return s.SDKClient.DialCoordinator(ctx)
}