feat: add PSK for external provisionerd auth (#8877)

Signed-off-by: Spike Curtis <spike@coder.com>

enterprise/cli/provisionerdaemons.go

@@ -44,13 +44,14 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 		rawTags      []string
 		pollInterval time.Duration
 		pollJitter   time.Duration
+		preSharedKey string
 	)
 	client := new(codersdk.Client)
 	cmd := &clibase.Cmd{
 		Use:   "start",
 		Short: "Run a provisioner daemon",
 		Middleware: clibase.Chain(
-			r.InitClient(client),
+			r.InitClientMissingTokenOK(client),
 		),
 		Handler: func(inv *clibase.Invocation) error {
 			ctx, cancel := context.WithCancel(inv.Context())
@@ -59,11 +60,6 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 			notifyCtx, notifyStop := signal.NotifyContext(ctx, agpl.InterruptSignals...)
 			defer notifyStop()
 
-			org, err := agpl.CurrentOrganization(inv, client)
-			if err != nil {
-				return xerrors.Errorf("get current organization: %w", err)
-			}
-
 			tags, err := agpl.ParseProvisionerTags(rawTags)
 			if err != nil {
 				return err
@@ -112,9 +108,13 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 				string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(terraformClient),
 			}
 			srv := provisionerd.New(func(ctx context.Context) (provisionerdproto.DRPCProvisionerDaemonClient, error) {
-				return client.ServeProvisionerDaemon(ctx, org.ID, []codersdk.ProvisionerType{
-					codersdk.ProvisionerTypeTerraform,
-				}, tags)
+				return client.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+					Provisioners: []codersdk.ProvisionerType{
+						codersdk.ProvisionerTypeTerraform,
+					},
+					Tags:         tags,
+					PreSharedKey: preSharedKey,
+				})
 			}, &provisionerd.Options{
 				Logger:          logger,
 				JobPollInterval: pollInterval,
@@ -182,6 +182,12 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 			Default:     (100 * time.Millisecond).String(),
 			Value:       clibase.DurationOf(&pollJitter),
 		},
+		{
+			Flag:        "psk",
+			Env:         "CODER_PROVISIONER_DAEMON_PSK",
+			Description: "Pre-shared key to authenticate with Coder server.",
+			Value:       clibase.StringOf(&preSharedKey),
+		},
 	}
 
 	return cmd

enterprise/cli/provisionerdaemons_test.go

@@ -0,0 +1,56 @@
+package cli_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/cli/clitest"
+	"github.com/coder/coder/codersdk"
+	"github.com/coder/coder/enterprise/coderd/coderdenttest"
+	"github.com/coder/coder/enterprise/coderd/license"
+	"github.com/coder/coder/pty/ptytest"
+	"github.com/coder/coder/testutil"
+)
+
+func TestProvisionerDaemon_PSK(t *testing.T) {
+	t.Parallel()
+
+	client, _ := coderdenttest.New(t, &coderdenttest.Options{
+		ProvisionerDaemonPSK: "provisionersftw",
+		LicenseOptions: &coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureExternalProvisionerDaemons: 1,
+			},
+		},
+	})
+	inv, conf := newCLI(t, "provisionerd", "start", "--psk=provisionersftw")
+	err := conf.URL().Write(client.URL.String())
+	require.NoError(t, err)
+	pty := ptytest.New(t).Attach(inv)
+	ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
+	defer cancel()
+	clitest.Start(t, inv)
+	pty.ExpectMatchContext(ctx, "starting provisioner daemon")
+}
+
+func TestProvisionerDaemon_SessionToken(t *testing.T) {
+	t.Parallel()
+
+	client, _ := coderdenttest.New(t, &coderdenttest.Options{
+		ProvisionerDaemonPSK: "provisionersftw",
+		LicenseOptions: &coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureExternalProvisionerDaemons: 1,
+			},
+		},
+	})
+	inv, conf := newCLI(t, "provisionerd", "start")
+	clitest.SetupConfig(t, client, conf)
+	pty := ptytest.New(t).Attach(inv)
+	ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
+	defer cancel()
+	clitest.Start(t, inv)
+	pty.ExpectMatchContext(ctx, "starting provisioner daemon")
+}

enterprise/cli/server.go

@@ -66,6 +66,7 @@ func (r *RootCmd) server() *clibase.Cmd {
 			DERPServerRegionID:        int(options.DeploymentValues.DERP.Server.RegionID.Value()),
 			ProxyHealthInterval:       options.DeploymentValues.ProxyHealthStatusInterval.Value(),
 			DefaultQuietHoursSchedule: options.DeploymentValues.UserQuietHoursSchedule.DefaultSchedule.Value(),
+			ProvisionerDaemonPSK:      options.DeploymentValues.Provisioner.DaemonPSK.Value(),
 		}
 
 		api, err := coderd.New(ctx, o)

enterprise/cli/testdata/coder_provisionerd_start_--help.golden

@@ -12,6 +12,9 @@ Run a provisioner daemon
       --poll-jitter duration, $CODER_PROVISIONERD_POLL_JITTER (default: 100ms)
           How much to jitter the poll interval by.
 
+      --psk string, $CODER_PROVISIONER_DAEMON_PSK
+          Pre-shared key to authenticate with Coder server.
+
   -t, --tag string-array, $CODER_PROVISIONERD_TAGS
           Tags to filter provisioner jobs by.
 

enterprise/cli/testdata/coder_server_--help.golden

@@ -373,6 +373,10 @@ updating, and deleting workspace resources.
       --provisioner-daemon-poll-jitter duration, $CODER_PROVISIONER_DAEMON_POLL_JITTER (default: 100ms)
           Random jitter added to the poll interval.
 
+      --provisioner-daemon-psk string, $CODER_PROVISIONER_DAEMON_PSK
+          Pre-shared key to authenticate external provisioner daemons to Coder
+          server.
+
       --provisioner-daemons int, $CODER_PROVISIONER_DAEMONS (default: 3)
           Number of provisioner daemons to create on start. If builds are stuck
           in queued state for a long time, consider increasing this.

enterprise/coderd/coderd.go

@@ -67,6 +67,10 @@ func New(ctx context.Context, options *Options) (_ *API, err error) {
 
 		AGPL:    coderd.New(options.Options),
 		Options: options,
+		provisionerDaemonAuth: &provisionerDaemonAuth{
+			psk:        options.ProvisionerDaemonPSK,
+			authorizer: options.Authorizer,
+		},
 	}
 	defer func() {
 		if err != nil {
@@ -193,14 +197,21 @@ func New(ctx context.Context, options *Options) (_ *API, err error) {
 				r.Get("/", api.groupByOrganization)
 			})
 		})
+		// TODO: provisioner daemons are not scoped to organizations in the database, so placing them
+		// under an organization route doesn't make sense.  In order to allow the /serve endpoint to
+		// work with a pre-shared key (PSK) without an API key, these routes will simply ignore the
+		// value of {organization}.  That is, the route will work with any organization ID, whether or
+		// not it exits.  This doesn't leak any information about the existence of organizations, so is
+		// fine from a security perspective, but might be a little surprising.
+		//
+		// We may in future decide to scope provisioner daemons to organizations, so we'll keep the API
+		// route as is.
 		r.Route("/organizations/{organization}/provisionerdaemons", func(r chi.Router) {
 			r.Use(
 				api.provisionerDaemonsEnabledMW,
-				apiKeyMiddleware,
-				httpmw.ExtractOrganizationParam(api.Database),
 			)
-			r.Get("/", api.provisionerDaemons)
-			r.Get("/serve", api.provisionerDaemonServe)
+			r.With(apiKeyMiddleware).Get("/", api.provisionerDaemons)
+			r.With(apiKeyMiddlewareOptional).Get("/serve", api.provisionerDaemonServe)
 		})
 		r.Route("/templates/{template}/acl", func(r chi.Router) {
 			r.Use(
@@ -362,6 +373,9 @@ type Options struct {
 	EntitlementsUpdateInterval time.Duration
 	ProxyHealthInterval        time.Duration
 	Keys                       map[string]ed25519.PublicKey
+
+	// optional pre-shared key for authentication of external provisioner daemons
+	ProvisionerDaemonPSK string
 }
 
 type API struct {
@@ -383,6 +397,8 @@ type API struct {
 	entitlementsUpdateMu sync.Mutex
 	entitlementsMu       sync.RWMutex
 	entitlements         codersdk.Entitlements
+
+	provisionerDaemonAuth *provisionerDaemonAuth
 }
 
 func (api *API) Close() error {

enterprise/coderd/coderdenttest/coderdenttest.go

@@ -56,6 +56,7 @@ type Options struct {
 	DontAddLicense              bool
 	DontAddFirstUser            bool
 	ReplicaSyncUpdateInterval   time.Duration
+	ProvisionerDaemonPSK        string
 }
 
 // New constructs a codersdk client connected to an in-memory Enterprise API instance.
@@ -94,6 +95,7 @@ func NewWithAPI(t *testing.T, options *Options) (
 		Keys:                       Keys,
 		ProxyHealthInterval:        options.ProxyHealthInterval,
 		DefaultQuietHoursSchedule:  oop.DeploymentValues.UserQuietHoursSchedule.DefaultSchedule.Value(),
+		ProvisionerDaemonPSK:       options.ProvisionerDaemonPSK,
 	})
 	require.NoError(t, err)
 	setHandler(coderAPI.AGPL.RootHandler)

enterprise/coderd/provisionerdaemons.go

@@ -2,6 +2,7 @@ package coderd
 
 import (
 	"context"
+	"crypto/subtle"
 	"database/sql"
 	"encoding/json"
 	"errors"
@@ -87,6 +88,40 @@ func (api *API) provisionerDaemons(rw http.ResponseWriter, r *http.Request) {
 	httpapi.Write(ctx, rw, http.StatusOK, apiDaemons)
 }
 
+type provisionerDaemonAuth struct {
+	psk        string
+	authorizer rbac.Authorizer
+}
+
+// authorize returns mutated tags and true if the given HTTP request is authorized to access the provisioner daemon
+// protobuf API, and returns nil, false otherwise.
+func (p *provisionerDaemonAuth) authorize(r *http.Request, tags map[string]string) (map[string]string, bool) {
+	ctx := r.Context()
+	apiKey, ok := httpmw.APIKeyOptional(r)
+	if ok {
+		tags = provisionerdserver.MutateTags(apiKey.UserID, tags)
+		if tags[provisionerdserver.TagScope] == provisionerdserver.ScopeUser {
+			// Any authenticated user can create provisioner daemons scoped
+			// for jobs that they own,
+			return tags, true
+		}
+		ua := httpmw.UserAuthorization(r)
+		if err := p.authorizer.Authorize(ctx, ua.Actor, rbac.ActionCreate, rbac.ResourceProvisionerDaemon); err == nil {
+			// User is allowed to create provisioner daemons
+			return tags, true
+		}
+	}
+
+	// Check for PSK
+	if p.psk != "" {
+		psk := r.Header.Get(codersdk.ProvisionerDaemonPSK)
+		if subtle.ConstantTimeCompare([]byte(p.psk), []byte(psk)) == 1 {
+			return tags, true
+		}
+	}
+	return nil, false
+}
+
 // Serves the provisioner daemon protobuf API over a WebSocket.
 //
 // @Summary Serve provisioner daemon
@@ -134,19 +169,11 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	// Any authenticated user can create provisioner daemons scoped
-	// for jobs that they own, but only authorized users can create
-	// globally scoped provisioners that attach to all jobs.
-	apiKey := httpmw.APIKey(r)
-	tags = provisionerdserver.MutateTags(apiKey.UserID, tags)
-
-	if tags[provisionerdserver.TagScope] == provisionerdserver.ScopeOrganization {
-		if !api.AGPL.Authorize(r, rbac.ActionCreate, rbac.ResourceProvisionerDaemon) {
-			httpapi.Write(ctx, rw, http.StatusForbidden, codersdk.Response{
-				Message: "You aren't allowed to create provisioner daemons for the organization.",
-			})
-			return
-		}
+	tags, authorized := api.provisionerDaemonAuth.authorize(r, tags)
+	if !authorized {
+		httpapi.Write(ctx, rw, http.StatusForbidden,
+			codersdk.Response{Message: "You aren't allowed to create provisioner daemons"})
+		return
 	}
 
 	provisioners := make([]database.ProvisionerType, 0)

enterprise/coderd/provisionerdaemons_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/coder/coder/enterprise/coderd/license"
 	"github.com/coder/coder/provisioner/echo"
 	"github.com/coder/coder/provisionersdk/proto"
+	"github.com/coder/coder/testutil"
 )
 
 func TestProvisionerDaemonServe(t *testing.T) {
@@ -28,23 +29,43 @@ func TestProvisionerDaemonServe(t *testing.T) {
 				codersdk.FeatureExternalProvisionerDaemons: 1,
 			},
 		}})
-		srv, err := client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{
-			codersdk.ProvisionerTypeEcho,
-		}, map[string]string{})
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		srv, err := client.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{},
+		})
 		require.NoError(t, err)
 		srv.DRPCConn().Close()
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 1)
 	})
 
 	t.Run("NoLicense", func(t *testing.T) {
 		t.Parallel()
 		client, user := coderdenttest.New(t, &coderdenttest.Options{DontAddLicense: true})
-		_, err := client.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{
-			codersdk.ProvisionerTypeEcho,
-		}, map[string]string{})
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		_, err := client.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{},
+		})
 		require.Error(t, err)
 		var apiError *codersdk.Error
 		require.ErrorAs(t, err, &apiError)
 		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
+
+		// querying provisioner daemons is forbidden without license
+		_, err = client.ProvisionerDaemons(ctx)
+		require.ErrorAs(t, err, &apiError)
+		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
 	})
 
 	t.Run("Organization", func(t *testing.T) {
@@ -55,15 +76,24 @@ func TestProvisionerDaemonServe(t *testing.T) {
 			},
 		}})
 		another, _ := coderdtest.CreateAnotherUser(t, client, user.OrganizationID, rbac.RoleOrgAdmin(user.OrganizationID))
-		_, err := another.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{
-			codersdk.ProvisionerTypeEcho,
-		}, map[string]string{
-			provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		_, err := another.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{
+				provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+			},
 		})
 		require.Error(t, err)
 		var apiError *codersdk.Error
 		require.ErrorAs(t, err, &apiError)
 		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 0)
 	})
 
 	t.Run("OrganizationNoPerms", func(t *testing.T) {
@@ -74,15 +104,24 @@ func TestProvisionerDaemonServe(t *testing.T) {
 			},
 		}})
 		another, _ := coderdtest.CreateAnotherUser(t, client, user.OrganizationID)
-		_, err := another.ServeProvisionerDaemon(context.Background(), user.OrganizationID, []codersdk.ProvisionerType{
-			codersdk.ProvisionerTypeEcho,
-		}, map[string]string{
-			provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		_, err := another.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{
+				provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+			},
 		})
 		require.Error(t, err)
 		var apiError *codersdk.Error
 		require.ErrorAs(t, err, &apiError)
 		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 0)
 	})
 
 	t.Run("UserLocal", func(t *testing.T) {
@@ -141,4 +180,129 @@ func TestProvisionerDaemonServe(t *testing.T) {
 		workspace := coderdtest.CreateWorkspace(t, another, user.OrganizationID, template.ID)
 		coderdtest.AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
 	})
+
+	t.Run("PSK", func(t *testing.T) {
+		t.Parallel()
+		client, user := coderdenttest.New(t, &coderdenttest.Options{
+			LicenseOptions: &coderdenttest.LicenseOptions{
+				Features: license.Features{
+					codersdk.FeatureExternalProvisionerDaemons: 1,
+				},
+			},
+			ProvisionerDaemonPSK: "provisionersftw",
+		})
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		another := codersdk.New(client.URL)
+		srv, err := another.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{
+				provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+			},
+			PreSharedKey: "provisionersftw",
+		})
+		require.NoError(t, err)
+		err = srv.DRPCConn().Close()
+		require.NoError(t, err)
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 1)
+	})
+
+	t.Run("BadPSK", func(t *testing.T) {
+		t.Parallel()
+		client, user := coderdenttest.New(t, &coderdenttest.Options{
+			LicenseOptions: &coderdenttest.LicenseOptions{
+				Features: license.Features{
+					codersdk.FeatureExternalProvisionerDaemons: 1,
+				},
+			},
+			ProvisionerDaemonPSK: "provisionersftw",
+		})
+		another := codersdk.New(client.URL)
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		_, err := another.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{
+				provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+			},
+			PreSharedKey: "the wrong key",
+		})
+		require.Error(t, err)
+		var apiError *codersdk.Error
+		require.ErrorAs(t, err, &apiError)
+		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 0)
+	})
+
+	t.Run("NoAuth", func(t *testing.T) {
+		t.Parallel()
+		client, user := coderdenttest.New(t, &coderdenttest.Options{
+			LicenseOptions: &coderdenttest.LicenseOptions{
+				Features: license.Features{
+					codersdk.FeatureExternalProvisionerDaemons: 1,
+				},
+			},
+			ProvisionerDaemonPSK: "provisionersftw",
+		})
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		another := codersdk.New(client.URL)
+		_, err := another.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{
+				provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+			},
+		})
+		require.Error(t, err)
+		var apiError *codersdk.Error
+		require.ErrorAs(t, err, &apiError)
+		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 0)
+	})
+
+	t.Run("NoPSK", func(t *testing.T) {
+		t.Parallel()
+		client, user := coderdenttest.New(t, &coderdenttest.Options{
+			LicenseOptions: &coderdenttest.LicenseOptions{
+				Features: license.Features{
+					codersdk.FeatureExternalProvisionerDaemons: 1,
+				},
+			},
+		})
+		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+		defer cancel()
+		another := codersdk.New(client.URL)
+		_, err := another.ServeProvisionerDaemon(ctx, codersdk.ServeProvisionerDaemonRequest{
+			Organization: user.OrganizationID,
+			Provisioners: []codersdk.ProvisionerType{
+				codersdk.ProvisionerTypeEcho,
+			},
+			Tags: map[string]string{
+				provisionerdserver.TagScope: provisionerdserver.ScopeOrganization,
+			},
+			PreSharedKey: "provisionersftw",
+		})
+		require.Error(t, err)
+		var apiError *codersdk.Error
+		require.ErrorAs(t, err, &apiError)
+		require.Equal(t, http.StatusForbidden, apiError.StatusCode())
+		daemons, err := client.ProvisionerDaemons(ctx)
+		require.NoError(t, err)
+		require.Len(t, daemons, 0)
+	})
 }