chore: Add workspace proxy enterprise cli commands (#7176)

* feat: Add workspace proxy enterprise cli commands
* chore: Handle custom workspace proxy options. Remove excess
* chore: Add endpoint to register workspace proxies

enterprise/cli/proxyserver.go

@@ -0,0 +1,346 @@
+//go:build !slim
+
+package cli
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"os/signal"
+	"regexp"
+	rpprof "runtime/pprof"
+	"time"
+
+	"github.com/coreos/go-systemd/daemon"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/cli"
+	"github.com/coder/coder/cli/clibase"
+	"github.com/coder/coder/cli/cliui"
+	"github.com/coder/coder/coderd/httpapi"
+	"github.com/coder/coder/coderd/httpmw"
+	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/enterprise/wsproxy"
+)
+
+type closers []func()
+
+func (c closers) Close() {
+	for _, closeF := range c {
+		closeF()
+	}
+}
+
+func (c *closers) Add(f func()) {
+	*c = append(*c, f)
+}
+
+func (*RootCmd) proxyServer() *clibase.Cmd {
+	var (
+		cfg = new(codersdk.DeploymentValues)
+		// Filter options for only relevant ones.
+		opts = cfg.Options().Filter(codersdk.IsWorkspaceProxies)
+
+		externalProxyOptionGroup = clibase.Group{
+			Name: "External Workspace Proxy",
+			YAML: "externalWorkspaceProxy",
+		}
+		proxySessionToken clibase.String
+		primaryAccessURL  clibase.URL
+	)
+	opts.Add(
+		// Options only for external workspace proxies
+
+		clibase.Option{
+			Name:        "Proxy Session Token",
+			Description: "Authentication token for the workspace proxy to communicate with coderd.",
+			Flag:        "proxy-session-token",
+			Env:         "CODER_PROXY_SESSION_TOKEN",
+			YAML:        "proxySessionToken",
+			Default:     "",
+			Value:       &proxySessionToken,
+			Group:       &externalProxyOptionGroup,
+			Hidden:      false,
+		},
+
+		clibase.Option{
+			Name:        "Coderd (Primary) Access URL",
+			Description: "URL to communicate with coderd. This should match the access URL of the Coder deployment.",
+			Flag:        "primary-access-url",
+			Env:         "CODER_PRIMARY_ACCESS_URL",
+			YAML:        "primaryAccessURL",
+			Default:     "",
+			Value:       &primaryAccessURL,
+			Group:       &externalProxyOptionGroup,
+			Hidden:      false,
+		},
+	)
+
+	cmd := &clibase.Cmd{
+		Use:     "server",
+		Short:   "Start a workspace proxy server",
+		Options: opts,
+		Middleware: clibase.Chain(
+			cli.WriteConfigMW(cfg),
+			cli.PrintDeprecatedOptions(),
+			clibase.RequireNArgs(0),
+		),
+		Handler: func(inv *clibase.Invocation) error {
+			if !(primaryAccessURL.Scheme == "http" || primaryAccessURL.Scheme == "https") {
+				return xerrors.Errorf("primary access URL must be http or https: url=%s", primaryAccessURL.String())
+			}
+
+			var closers closers
+			// Main command context for managing cancellation of running
+			// services.
+			ctx, topCancel := context.WithCancel(inv.Context())
+			defer topCancel()
+			closers.Add(topCancel)
+
+			go cli.DumpHandler(ctx)
+
+			cli.PrintLogo(inv)
+			logger, logCloser, err := cli.BuildLogger(inv, cfg)
+			if err != nil {
+				return xerrors.Errorf("make logger: %w", err)
+			}
+			defer logCloser()
+			closers.Add(logCloser)
+
+			logger.Debug(ctx, "started debug logging")
+			logger.Sync()
+
+			// Register signals early on so that graceful shutdown can't
+			// be interrupted by additional signals. Note that we avoid
+			// shadowing cancel() (from above) here because notifyStop()
+			// restores default behavior for the signals. This protects
+			// the shutdown sequence from abruptly terminating things
+			// like: database migrations, provisioner work, workspace
+			// cleanup in dev-mode, etc.
+			//
+			// To get out of a graceful shutdown, the user can send
+			// SIGQUIT with ctrl+\ or SIGKILL with `kill -9`.
+			notifyCtx, notifyStop := signal.NotifyContext(ctx, cli.InterruptSignals...)
+			defer notifyStop()
+
+			// Clean up idle connections at the end, e.g.
+			// embedded-postgres can leave an idle connection
+			// which is caught by goleaks.
+			defer http.DefaultClient.CloseIdleConnections()
+			closers.Add(http.DefaultClient.CloseIdleConnections)
+
+			tracer, _ := cli.ConfigureTraceProvider(ctx, logger, inv, cfg)
+
+			httpServers, err := cli.ConfigureHTTPServers(inv, cfg)
+			if err != nil {
+				return xerrors.Errorf("configure http(s): %w", err)
+			}
+			defer httpServers.Close()
+			closers.Add(httpServers.Close)
+
+			// If no access url given, use the local address.
+			if cfg.AccessURL.String() == "" {
+				// Prefer TLS
+				if httpServers.TLSUrl != nil {
+					cfg.AccessURL = clibase.URL(*httpServers.TLSUrl)
+				} else if httpServers.HTTPUrl != nil {
+					cfg.AccessURL = clibase.URL(*httpServers.HTTPUrl)
+				}
+			}
+
+			// TODO: @emyrk I find this strange that we add this to the context
+			// at the root here.
+			ctx, httpClient, err := cli.ConfigureHTTPClient(
+				ctx,
+				cfg.TLS.ClientCertFile.String(),
+				cfg.TLS.ClientKeyFile.String(),
+				cfg.TLS.ClientCAFile.String(),
+			)
+			if err != nil {
+				return xerrors.Errorf("configure http client: %w", err)
+			}
+			defer httpClient.CloseIdleConnections()
+			closers.Add(httpClient.CloseIdleConnections)
+
+			// Warn the user if the access URL appears to be a loopback address.
+			isLocal, err := cli.IsLocalURL(ctx, cfg.AccessURL.Value())
+			if isLocal || err != nil {
+				reason := "could not be resolved"
+				if isLocal {
+					reason = "isn't externally reachable"
+				}
+				cliui.Warnf(
+					inv.Stderr,
+					"The access URL %s %s, this may cause unexpected problems when creating workspaces. Generate a unique *.try.coder.app URL by not specifying an access URL.\n",
+					cliui.Styles.Field.Render(cfg.AccessURL.String()), reason,
+				)
+			}
+
+			// A newline is added before for visibility in terminal output.
+			cliui.Infof(inv.Stdout, "\nView the Web UI: %s", cfg.AccessURL.String())
+
+			var appHostnameRegex *regexp.Regexp
+			appHostname := cfg.WildcardAccessURL.String()
+			if appHostname != "" {
+				appHostnameRegex, err = httpapi.CompileHostnamePattern(appHostname)
+				if err != nil {
+					return xerrors.Errorf("parse wildcard access URL %q: %w", appHostname, err)
+				}
+			}
+
+			realIPConfig, err := httpmw.ParseRealIPConfig(cfg.ProxyTrustedHeaders, cfg.ProxyTrustedOrigins)
+			if err != nil {
+				return xerrors.Errorf("parse real ip config: %w", err)
+			}
+
+			if cfg.Pprof.Enable {
+				// This prevents the pprof import from being accidentally deleted.
+				// pprof has an init function that attaches itself to the default handler.
+				// By passing a nil handler to 'serverHandler', it will automatically use
+				// the default, which has pprof attached.
+				_ = pprof.Handler
+				//nolint:revive
+				closeFunc := cli.ServeHandler(ctx, logger, nil, cfg.Pprof.Address.String(), "pprof")
+				defer closeFunc()
+				closers.Add(closeFunc)
+			}
+
+			prometheusRegistry := prometheus.NewRegistry()
+			if cfg.Prometheus.Enable {
+				prometheusRegistry.MustRegister(collectors.NewGoCollector())
+				prometheusRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+
+				//nolint:revive
+				closeFunc := cli.ServeHandler(ctx, logger, promhttp.InstrumentMetricHandler(
+					prometheusRegistry, promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{}),
+				), cfg.Prometheus.Address.String(), "prometheus")
+				defer closeFunc()
+				closers.Add(closeFunc)
+			}
+
+			proxy, err := wsproxy.New(ctx, &wsproxy.Options{
+				Logger:             logger,
+				DashboardURL:       primaryAccessURL.Value(),
+				AccessURL:          cfg.AccessURL.Value(),
+				AppHostname:        appHostname,
+				AppHostnameRegex:   appHostnameRegex,
+				RealIPConfig:       realIPConfig,
+				Tracing:            tracer,
+				PrometheusRegistry: prometheusRegistry,
+				APIRateLimit:       int(cfg.RateLimit.API.Value()),
+				SecureAuthCookie:   cfg.SecureAuthCookie.Value(),
+				DisablePathApps:    cfg.DisablePathApps.Value(),
+				ProxySessionToken:  proxySessionToken.Value(),
+			})
+			if err != nil {
+				return xerrors.Errorf("create workspace proxy: %w", err)
+			}
+
+			shutdownConnsCtx, shutdownConns := context.WithCancel(ctx)
+			defer shutdownConns()
+			closers.Add(shutdownConns)
+			// ReadHeaderTimeout is purposefully not enabled. It caused some
+			// issues with websockets over the dev tunnel.
+			// See: https://github.com/coder/coder/pull/3730
+			//nolint:gosec
+			httpServer := &http.Server{
+				// These errors are typically noise like "TLS: EOF". Vault does
+				// similar:
+				// https://github.com/hashicorp/vault/blob/e2490059d0711635e529a4efcbaa1b26998d6e1c/command/server.go#L2714
+				ErrorLog: log.New(io.Discard, "", 0),
+				Handler:  proxy.Handler,
+				BaseContext: func(_ net.Listener) context.Context {
+					return shutdownConnsCtx
+				},
+			}
+			defer func() {
+				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+				defer cancel()
+				_ = httpServer.Shutdown(ctx)
+			}()
+
+			// TODO: So this obviously is not going to work well.
+			errCh := make(chan error, 1)
+			go rpprof.Do(ctx, rpprof.Labels("service", "workspace-proxy"), func(ctx context.Context) {
+				errCh <- httpServers.Serve(httpServer)
+			})
+
+			cliui.Infof(inv.Stdout, "\n==> Logs will stream in below (press ctrl+c to gracefully exit):")
+
+			// Updates the systemd status from activating to activated.
+			_, err = daemon.SdNotify(false, daemon.SdNotifyReady)
+			if err != nil {
+				return xerrors.Errorf("notify systemd: %w", err)
+			}
+
+			// Currently there is no way to ask the server to shut
+			// itself down, so any exit signal will result in a non-zero
+			// exit of the server.
+			var exitErr error
+			select {
+			case exitErr = <-errCh:
+			case <-notifyCtx.Done():
+				exitErr = notifyCtx.Err()
+				_, _ = fmt.Fprintln(inv.Stdout, cliui.Styles.Bold.Render(
+					"Interrupt caught, gracefully exiting. Use ctrl+\\ to force quit",
+				))
+			}
+
+			if exitErr != nil && !xerrors.Is(exitErr, context.Canceled) {
+				cliui.Errorf(inv.Stderr, "Unexpected error, shutting down server: %s\n", exitErr)
+			}
+
+			// Begin clean shut down stage, we try to shut down services
+			// gracefully in an order that gives the best experience.
+			// This procedure should not differ greatly from the order
+			// of `defer`s in this function, but allows us to inform
+			// the user about what's going on and handle errors more
+			// explicitly.
+
+			_, err = daemon.SdNotify(false, daemon.SdNotifyStopping)
+			if err != nil {
+				cliui.Errorf(inv.Stderr, "Notify systemd failed: %s", err)
+			}
+
+			// Stop accepting new connections without interrupting
+			// in-flight requests, give in-flight requests 5 seconds to
+			// complete.
+			cliui.Info(inv.Stdout, "Shutting down API server..."+"\n")
+			shutdownCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+			defer cancel()
+			err = httpServer.Shutdown(shutdownCtx)
+			if err != nil {
+				cliui.Errorf(inv.Stderr, "API server shutdown took longer than 3s: %s\n", err)
+			} else {
+				cliui.Info(inv.Stdout, "Gracefully shut down API server\n")
+			}
+			// Cancel any remaining in-flight requests.
+			shutdownConns()
+
+			// Trigger context cancellation for any remaining services.
+			closers.Close()
+
+			switch {
+			case xerrors.Is(exitErr, context.DeadlineExceeded):
+				cliui.Warnf(inv.Stderr, "Graceful shutdown timed out")
+				// Errors here cause a significant number of benign CI failures.
+				return nil
+			case xerrors.Is(exitErr, context.Canceled):
+				return nil
+			case exitErr != nil:
+				return xerrors.Errorf("graceful shutdown: %w", exitErr)
+			default:
+				return nil
+			}
+		},
+	}
+
+	return cmd
+}

enterprise/cli/proxyserver_slim.go

@@ -0,0 +1,37 @@
+//go:build slim
+
+package cli
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/coder/coder/cli/clibase"
+	"github.com/coder/coder/cli/cliui"
+)
+
+func (r *RootCmd) proxyServer() *clibase.Cmd {
+	root := &clibase.Cmd{
+		Use:     "server",
+		Short:   "Start a workspace proxy server",
+		Aliases: []string{},
+		// We accept RawArgs so all commands and flags are accepted.
+		RawArgs: true,
+		Hidden:  true,
+		Handler: func(inv *clibase.Invocation) error {
+			serverUnsupported(inv.Stderr)
+			return nil
+		},
+	}
+
+	return root
+}
+
+func serverUnsupported(w io.Writer) {
+	_, _ = fmt.Fprintf(w, "You are using a 'slim' build of Coder, which does not support the %s subcommand.\n", cliui.Styles.Code.Render("server"))
+	_, _ = fmt.Fprintln(w, "")
+	_, _ = fmt.Fprintln(w, "Please use a build of Coder from GitHub releases:")
+	_, _ = fmt.Fprintln(w, "  https://github.com/coder/coder/releases")
+	os.Exit(1)
+}

enterprise/cli/root.go

@@ -12,6 +12,7 @@ type RootCmd struct {
 func (r *RootCmd) enterpriseOnly() []*clibase.Cmd {
 	return []*clibase.Cmd{
 		r.server(),
+		r.workspaceProxy(),
 		r.features(),
 		r.licenses(),
 		r.groups(),

enterprise/cli/workspaceproxy.go

@@ -0,0 +1,156 @@
+package cli
+
+import (
+	"fmt"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/cli/clibase"
+	"github.com/coder/coder/cli/cliui"
+	"github.com/coder/coder/codersdk"
+)
+
+func (r *RootCmd) workspaceProxy() *clibase.Cmd {
+	cmd := &clibase.Cmd{
+		Use:     "workspace-proxy",
+		Short:   "Manage workspace proxies",
+		Aliases: []string{"proxy"},
+		Hidden:  true,
+		Handler: func(inv *clibase.Invocation) error {
+			return inv.Command.HelpHandler(inv)
+		},
+		Children: []*clibase.Cmd{
+			r.proxyServer(),
+			r.createProxy(),
+			r.deleteProxy(),
+		},
+	}
+
+	return cmd
+}
+
+func (r *RootCmd) deleteProxy() *clibase.Cmd {
+	client := new(codersdk.Client)
+	cmd := &clibase.Cmd{
+		Use:   "delete <name|id>",
+		Short: "Delete a workspace proxy",
+		Middleware: clibase.Chain(
+			clibase.RequireNArgs(1),
+			r.InitClient(client),
+		),
+		Handler: func(inv *clibase.Invocation) error {
+			ctx := inv.Context()
+			err := client.DeleteWorkspaceProxyByName(ctx, inv.Args[0])
+			if err != nil {
+				return xerrors.Errorf("delete workspace proxy %q: %w", inv.Args[0], err)
+			}
+
+			_, _ = fmt.Fprintf(inv.Stdout, "Workspace proxy %q deleted successfully\n", inv.Args[0])
+			return nil
+		},
+	}
+
+	return cmd
+}
+
+func (r *RootCmd) createProxy() *clibase.Cmd {
+	var (
+		proxyName             string
+		displayName           string
+		proxyIcon             string
+		proxyURL              string
+		proxyWildcardHostname string
+		onlyToken             bool
+		formatter             = cliui.NewOutputFormatter(
+			// Text formatter should be human readable.
+			cliui.ChangeFormatterData(cliui.TextFormat(), func(data any) (any, error) {
+				response, ok := data.(codersdk.CreateWorkspaceProxyResponse)
+				if !ok {
+					return nil, xerrors.Errorf("unexpected type %T", data)
+				}
+				return fmt.Sprintf("Workspace Proxy %q registered successfully\nToken: %s", response.Proxy.Name, response.ProxyToken), nil
+			}),
+			cliui.JSONFormat(),
+			// Table formatter expects a slice, make a slice of one.
+			cliui.ChangeFormatterData(cliui.TableFormat([]codersdk.CreateWorkspaceProxyResponse{}, []string{"proxy name", "proxy url", "proxy token"}),
+				func(data any) (any, error) {
+					response, ok := data.(codersdk.CreateWorkspaceProxyResponse)
+					if !ok {
+						return nil, xerrors.Errorf("unexpected type %T", data)
+					}
+					return []codersdk.CreateWorkspaceProxyResponse{response}, nil
+				}),
+		)
+	)
+
+	client := new(codersdk.Client)
+	cmd := &clibase.Cmd{
+		Use:   "create",
+		Short: "Create a workspace proxy",
+		Middleware: clibase.Chain(
+			clibase.RequireNArgs(0),
+			r.InitClient(client),
+		),
+		Handler: func(inv *clibase.Invocation) error {
+			ctx := inv.Context()
+			resp, err := client.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
+				Name:             proxyName,
+				DisplayName:      displayName,
+				Icon:             proxyIcon,
+				URL:              proxyURL,
+				WildcardHostname: proxyWildcardHostname,
+			})
+			if err != nil {
+				return xerrors.Errorf("create workspace proxy: %w", err)
+			}
+
+			var output string
+			if onlyToken {
+				output = resp.ProxyToken
+			} else {
+				output, err = formatter.Format(ctx, resp)
+				if err != nil {
+					return err
+				}
+			}
+
+			_, err = fmt.Fprintln(inv.Stdout, output)
+			return err
+		},
+	}
+
+	formatter.AttachOptions(&cmd.Options)
+	cmd.Options.Add(
+		clibase.Option{
+			Flag:        "name",
+			Description: "Name of the proxy. This is used to identify the proxy.",
+			Value:       clibase.StringOf(&proxyName),
+		},
+		clibase.Option{
+			Flag:        "display-name",
+			Description: "Display of the proxy. If omitted, the name is reused as the display name.",
+			Value:       clibase.StringOf(&displayName),
+		},
+		clibase.Option{
+			Flag:        "icon",
+			Description: "Display icon of the proxy.",
+			Value:       clibase.StringOf(&proxyIcon),
+		},
+		clibase.Option{
+			Flag:        "access-url",
+			Description: "Access URL of the proxy.",
+			Value:       clibase.StringOf(&proxyURL),
+		},
+		clibase.Option{
+			Flag:        "wildcard-access-url",
+			Description: "(Optional) Access url of the proxy for subdomain based apps.",
+			Value:       clibase.StringOf(&proxyWildcardHostname),
+		},
+		clibase.Option{
+			Flag:        "only-token",
+			Description: "Only print the token. This is useful for scripting.",
+			Value:       clibase.BoolOf(&onlyToken),
+		},
+	)
+	return cmd
+}

enterprise/cli/workspaceproxy_test.go

@@ -0,0 +1,122 @@
+package cli_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/cli/clitest"
+	"github.com/coder/coder/coderd/coderdtest"
+	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/enterprise/coderd/coderdenttest"
+	"github.com/coder/coder/enterprise/coderd/license"
+	"github.com/coder/coder/pty/ptytest"
+	"github.com/coder/coder/testutil"
+)
+
+func Test_ProxyCRUD(t *testing.T) {
+	t.Parallel()
+
+	t.Run("Create", func(t *testing.T) {
+		t.Parallel()
+
+		dv := coderdtest.DeploymentValues(t)
+		dv.Experiments = []string{
+			string(codersdk.ExperimentMoons),
+			"*",
+		}
+
+		client := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				DeploymentValues: dv,
+			},
+		})
+		_ = coderdtest.CreateFirstUser(t, client)
+		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspaceProxy: 1,
+			},
+		})
+
+		expectedName := "test-proxy"
+		ctx := testutil.Context(t, testutil.WaitLong)
+		inv, conf := newCLI(
+			t,
+			"proxy", "create",
+			"--name", expectedName,
+			"--display-name", "Test Proxy",
+			"--icon", "/emojis/1f4bb.png",
+			"--access-url", "http://localhost:3010",
+			"--only-token",
+		)
+
+		pty := ptytest.New(t)
+		inv.Stdout = pty.Output()
+		clitest.SetupConfig(t, client, conf)
+
+		err := inv.WithContext(ctx).Run()
+		require.NoError(t, err)
+
+		line := pty.ReadLine(ctx)
+		parts := strings.Split(line, ":")
+		require.Len(t, parts, 2, "expected 2 parts")
+		_, err = uuid.Parse(parts[0])
+		require.NoError(t, err, "expected token to be a uuid")
+
+		proxies, err := client.WorkspaceProxies(ctx)
+		require.NoError(t, err, "failed to get workspace proxies")
+		require.Len(t, proxies, 1, "expected 1 proxy")
+		require.Equal(t, expectedName, proxies[0].Name, "expected proxy name to match")
+	})
+
+	t.Run("Delete", func(t *testing.T) {
+		t.Parallel()
+
+		dv := coderdtest.DeploymentValues(t)
+		dv.Experiments = []string{
+			string(codersdk.ExperimentMoons),
+			"*",
+		}
+
+		client := coderdenttest.New(t, &coderdenttest.Options{
+			Options: &coderdtest.Options{
+				DeploymentValues: dv,
+			},
+		})
+		_ = coderdtest.CreateFirstUser(t, client)
+		_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspaceProxy: 1,
+			},
+		})
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+		expectedName := "test-proxy"
+		_, err := client.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
+			Name:        expectedName,
+			DisplayName: "Test Proxy",
+			Icon:        "/emojis/us.png",
+			URL:         "http://localhost:3010",
+		})
+		require.NoError(t, err, "failed to create workspace proxy")
+
+		inv, conf := newCLI(
+			t,
+			"proxy", "delete", expectedName,
+		)
+
+		pty := ptytest.New(t)
+		inv.Stdout = pty.Output()
+		clitest.SetupConfig(t, client, conf)
+
+		err = inv.WithContext(ctx).Run()
+		require.NoError(t, err)
+
+		proxies, err := client.WorkspaceProxies(ctx)
+		require.NoError(t, err, "failed to get workspace proxies")
+		require.Len(t, proxies, 0, "expected no proxies")
+	})
+}