feat(coderd): add support for external agents to API's and provisioner (#19286)

This pull request introduces support for external workspace management, allowing users to register and manage workspaces that are provisioned and managed outside of the Coder.

Depends on: https://github.com/coder/terraform-provider-coder/pull/424

* GET /api/v2/init-script - Gets the agent initialization script
  * By default, it returns a script for Linux (amd64), but with query parameters (os and arch) you can get the init script for different platforms
* GET /api/v2/workspaces/{workspace}/external-agent/{agent}/credentials - Gets credentials for an external agent **(enterprise)**
* Updated queries to filter workspaces/templates by the has_external_agent field

enterprise/coderd/coderd.go

@@ -506,6 +506,15 @@ func New(ctx context.Context, options *Options) (_ *API, err error) {
 			apiKeyMiddleware,
 			httpmw.ExtractNotificationTemplateParam(options.Database),
 		).Put("/notifications/templates/{notification_template}/method", api.updateNotificationTemplateMethod)
+
+		r.Route("/workspaces/{workspace}/external-agent", func(r chi.Router) {
+			r.Use(
+				apiKeyMiddleware,
+				httpmw.ExtractWorkspaceParam(options.Database),
+				api.RequireFeatureMW(codersdk.FeatureWorkspaceExternalAgent),
+			)
+			r.Get("/{agent}/credentials", api.workspaceExternalAgentCredentials)
+		})
 	})
 
 	if len(options.SCIMAPIKey) != 0 {
@@ -920,17 +929,9 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 		}
 		reloadedEntitlements.Features[codersdk.FeatureExternalTokenEncryption] = featureExternalTokenEncryption
 
-		// If there's a license installed, we will use the enterprise build
-		// limit checker.
-		// This checker currently only enforces the managed agent limit.
-		if reloadedEntitlements.HasLicense {
-			var checker wsbuilder.UsageChecker = api
-			api.AGPL.BuildUsageChecker.Store(&checker)
-		} else {
-			// Don't check any usage, just like AGPL.
-			var checker wsbuilder.UsageChecker = wsbuilder.NoopUsageChecker{}
-			api.AGPL.BuildUsageChecker.Store(&checker)
-		}
+		// Always use the enterprise usage checker
+		var checker wsbuilder.UsageChecker = api
+		api.AGPL.BuildUsageChecker.Store(&checker)
 
 		return reloadedEntitlements, nil
 	})
@@ -939,9 +940,17 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 var _ wsbuilder.UsageChecker = &API{}
 
 func (api *API) CheckBuildUsage(ctx context.Context, store database.Store, templateVersion *database.TemplateVersion) (wsbuilder.UsageCheckResponse, error) {
-	// We assume that if this function is called, a valid license is installed.
-	// When there are no licenses installed, a noop usage checker is used
-	// instead.
+	// If the template version has an external agent, we need to check that the
+	// license is entitled to this feature.
+	if templateVersion.HasExternalAgent.Valid && templateVersion.HasExternalAgent.Bool {
+		feature, ok := api.Entitlements.Feature(codersdk.FeatureWorkspaceExternalAgent)
+		if !ok || !feature.Enabled {
+			return wsbuilder.UsageCheckResponse{
+				Permitted: false,
+				Message:   "You have a template which uses external agents but your license is not entitled to this feature. You will be unable to create new workspaces from these templates.",
+			}, nil
+		}
+	}
 
 	// If the template version doesn't have an AI task, we don't need to check
 	// usage.
@@ -951,32 +960,35 @@ func (api *API) CheckBuildUsage(ctx context.Context, store database.Store, templ
 		}, nil
 	}
 
-	// Otherwise, we need to check that we haven't breached the managed agent
+	// When unlicensed, we need to check that we haven't breached the managed agent
 	// limit.
-	managedAgentLimit, ok := api.Entitlements.Feature(codersdk.FeatureManagedAgentLimit)
-	if !ok || !managedAgentLimit.Enabled || managedAgentLimit.Limit == nil || managedAgentLimit.UsagePeriod == nil {
-		return wsbuilder.UsageCheckResponse{
-			Permitted: false,
-			Message:   "Your license is not entitled to managed agents. Please contact sales to continue using managed agents.",
-		}, nil
-	}
+	// Unlicensed deployments are allowed to use unlimited managed agents.
+	if api.Entitlements.HasLicense() {
+		managedAgentLimit, ok := api.Entitlements.Feature(codersdk.FeatureManagedAgentLimit)
+		if !ok || !managedAgentLimit.Enabled || managedAgentLimit.Limit == nil || managedAgentLimit.UsagePeriod == nil {
+			return wsbuilder.UsageCheckResponse{
+				Permitted: false,
+				Message:   "Your license is not entitled to managed agents. Please contact sales to continue using managed agents.",
+			}, nil
+		}
 
-	// This check is intentionally not committed to the database. It's fine if
-	// it's not 100% accurate or allows for minor breaches due to build races.
-	// nolint:gocritic // Requires permission to read all workspaces to read managed agent count.
-	managedAgentCount, err := store.GetManagedAgentCount(agpldbauthz.AsSystemRestricted(ctx), database.GetManagedAgentCountParams{
-		StartTime: managedAgentLimit.UsagePeriod.Start,
-		EndTime:   managedAgentLimit.UsagePeriod.End,
-	})
-	if err != nil {
-		return wsbuilder.UsageCheckResponse{}, xerrors.Errorf("get managed agent count: %w", err)
-	}
+		// This check is intentionally not committed to the database. It's fine if
+		// it's not 100% accurate or allows for minor breaches due to build races.
+		// nolint:gocritic // Requires permission to read all workspaces to read managed agent count.
+		managedAgentCount, err := store.GetManagedAgentCount(agpldbauthz.AsSystemRestricted(ctx), database.GetManagedAgentCountParams{
+			StartTime: managedAgentLimit.UsagePeriod.Start,
+			EndTime:   managedAgentLimit.UsagePeriod.End,
+		})
+		if err != nil {
+			return wsbuilder.UsageCheckResponse{}, xerrors.Errorf("get managed agent count: %w", err)
+		}
 
-	if managedAgentCount >= *managedAgentLimit.Limit {
-		return wsbuilder.UsageCheckResponse{
-			Permitted: false,
-			Message:   "You have breached the managed agent limit in your license. Please contact sales to continue using managed agents.",
-		}, nil
+		if managedAgentCount >= *managedAgentLimit.Limit {
+			return wsbuilder.UsageCheckResponse{
+				Permitted: false,
+				Message:   "You have breached the managed agent limit in your license. Please contact sales to continue using managed agents.",
+			}, nil
+		}
 	}
 
 	return wsbuilder.UsageCheckResponse{

enterprise/coderd/license/license.go

@@ -3,6 +3,7 @@ package license
 import (
 	"context"
 	"crypto/ed25519"
+	"database/sql"
 	"fmt"
 	"math"
 	"time"
@@ -94,10 +95,34 @@ func Entitlements(
 		return codersdk.Entitlements{}, xerrors.Errorf("query active user count: %w", err)
 	}
 
+	// nolint:gocritic // Getting external workspaces is a system function.
+	externalWorkspaces, err := db.GetWorkspaces(dbauthz.AsSystemRestricted(ctx), database.GetWorkspacesParams{
+		HasExternalAgent: sql.NullBool{
+			Bool:  true,
+			Valid: true,
+		},
+	})
+	if err != nil {
+		return codersdk.Entitlements{}, xerrors.Errorf("query external workspaces: %w", err)
+	}
+
+	// nolint:gocritic // Getting external templates is a system function.
+	externalTemplates, err := db.GetTemplatesWithFilter(dbauthz.AsSystemRestricted(ctx), database.GetTemplatesWithFilterParams{
+		HasExternalAgent: sql.NullBool{
+			Bool:  true,
+			Valid: true,
+		},
+	})
+	if err != nil {
+		return codersdk.Entitlements{}, xerrors.Errorf("query external templates: %w", err)
+	}
+
 	entitlements, err := LicensesEntitlements(ctx, now, licenses, enablements, keys, FeatureArguments{
-		ActiveUserCount:   activeUserCount,
-		ReplicaCount:      replicaCount,
-		ExternalAuthCount: externalAuthCount,
+		ActiveUserCount:        activeUserCount,
+		ReplicaCount:           replicaCount,
+		ExternalAuthCount:      externalAuthCount,
+		ExternalWorkspaceCount: int64(len(externalWorkspaces)),
+		ExternalTemplateCount:  int64(len(externalTemplates)),
 		ManagedAgentCountFn: func(ctx context.Context, startTime time.Time, endTime time.Time) (int64, error) {
 			// nolint:gocritic // Requires permission to read all workspaces to read managed agent count.
 			return db.GetManagedAgentCount(dbauthz.AsSystemRestricted(ctx), database.GetManagedAgentCountParams{
@@ -114,9 +139,11 @@ func Entitlements(
 }
 
 type FeatureArguments struct {
-	ActiveUserCount   int64
-	ReplicaCount      int
-	ExternalAuthCount int
+	ActiveUserCount        int64
+	ReplicaCount           int
+	ExternalAuthCount      int
+	ExternalWorkspaceCount int64
+	ExternalTemplateCount  int64
 	// Unfortunately, managed agent count is not a simple count of the current
 	// state of the world, but a count between two points in time determined by
 	// the licenses.
@@ -418,6 +445,30 @@ func LicensesEntitlements(
 		}
 	}
 
+	if featureArguments.ExternalWorkspaceCount > 0 {
+		feature := entitlements.Features[codersdk.FeatureWorkspaceExternalAgent]
+		switch feature.Entitlement {
+		case codersdk.EntitlementNotEntitled:
+			entitlements.Errors = append(entitlements.Errors,
+				"You have external workspaces but your license is not entitled to this feature.")
+		case codersdk.EntitlementGracePeriod:
+			entitlements.Warnings = append(entitlements.Warnings,
+				"You have external workspaces but your license is expired.")
+		}
+	}
+
+	if featureArguments.ExternalTemplateCount > 0 {
+		feature := entitlements.Features[codersdk.FeatureWorkspaceExternalAgent]
+		switch feature.Entitlement {
+		case codersdk.EntitlementNotEntitled:
+			entitlements.Errors = append(entitlements.Errors,
+				"You have templates which use external agents but your license is not entitled to this feature.")
+		case codersdk.EntitlementGracePeriod:
+			entitlements.Warnings = append(entitlements.Warnings,
+				"You have templates which use external agents but your license is expired.")
+		}
+	}
+
 	// Managed agent warnings are applied based on usage period. We only
 	// generate a warning if the license actually has managed agents.
 	// Note that agents are free when unlicensed.

enterprise/coderd/license/license_test.go

@@ -723,6 +723,12 @@ func TestEntitlements(t *testing.T) {
 				return true
 			})).
 			Return(int64(175), nil)
+		mDB.EXPECT().
+			GetWorkspaces(gomock.Any(), gomock.Any()).
+			Return([]database.GetWorkspacesRow{}, nil)
+		mDB.EXPECT().
+			GetTemplatesWithFilter(gomock.Any(), gomock.Any()).
+			Return([]database.Template{}, nil)
 
 		entitlements, err := license.Entitlements(context.Background(), mDB, 1, 0, coderdenttest.Keys, all)
 		require.NoError(t, err)
@@ -766,6 +772,7 @@ func TestLicenseEntitlements(t *testing.T) {
 		codersdk.FeatureUserRoleManagement:         true,
 		codersdk.FeatureAccessControl:              true,
 		codersdk.FeatureControlSharedPorts:         true,
+		codersdk.FeatureWorkspaceExternalAgent:     true,
 	}
 
 	legacyLicense := func() *coderdenttest.LicenseOptions {
@@ -1109,6 +1116,32 @@ func TestLicenseEntitlements(t *testing.T) {
 				assert.Equal(t, int64(200), *feature.Actual)
 			},
 		},
+		{
+			Name: "ExternalWorkspace",
+			Licenses: []*coderdenttest.LicenseOptions{
+				enterpriseLicense().UserLimit(100),
+			},
+			Arguments: license.FeatureArguments{
+				ExternalWorkspaceCount: 1,
+			},
+			AssertEntitlements: func(t *testing.T, entitlements codersdk.Entitlements) {
+				assert.Equal(t, codersdk.EntitlementEntitled, entitlements.Features[codersdk.FeatureWorkspaceExternalAgent].Entitlement)
+				assert.True(t, entitlements.Features[codersdk.FeatureWorkspaceExternalAgent].Enabled)
+			},
+		},
+		{
+			Name: "ExternalTemplate",
+			Licenses: []*coderdenttest.LicenseOptions{
+				enterpriseLicense().UserLimit(100),
+			},
+			Arguments: license.FeatureArguments{
+				ExternalTemplateCount: 1,
+			},
+			AssertEntitlements: func(t *testing.T, entitlements codersdk.Entitlements) {
+				assert.Equal(t, codersdk.EntitlementEntitled, entitlements.Features[codersdk.FeatureWorkspaceExternalAgent].Entitlement)
+				assert.True(t, entitlements.Features[codersdk.FeatureWorkspaceExternalAgent].Enabled)
+			},
+		},
 	}
 
 	for _, tc := range testCases {

enterprise/coderd/workspaceagents.go

@@ -2,9 +2,14 @@ package coderd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 
+	"github.com/go-chi/chi/v5"
+
+	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/httpmw"
 	"github.com/coder/coder/v2/codersdk"
 )
 
@@ -17,3 +22,77 @@ func (api *API) shouldBlockNonBrowserConnections(rw http.ResponseWriter) bool {
 	}
 	return false
 }
+
+// @Summary Get workspace external agent credentials
+// @ID get-workspace-external-agent-credentials
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Enterprise
+// @Param workspace path string true "Workspace ID" format(uuid)
+// @Param agent path string true "Agent name"
+// @Success 200 {object} codersdk.ExternalAgentCredentials
+// @Router /workspaces/{workspace}/external-agent/{agent}/credentials [get]
+func (api *API) workspaceExternalAgentCredentials(rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	workspace := httpmw.WorkspaceParam(r)
+	agentName := chi.URLParam(r, "agent")
+
+	build, err := api.Database.GetLatestWorkspaceBuildByWorkspaceID(ctx, workspace.ID)
+	if err != nil {
+		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to get latest workspace build.",
+			Detail:  err.Error(),
+		})
+		return
+	}
+	if !build.HasExternalAgent.Bool {
+		httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
+			Message: "Workspace does not have an external agent.",
+		})
+		return
+	}
+
+	agents, err := api.Database.GetWorkspaceAgentsByWorkspaceAndBuildNumber(ctx, database.GetWorkspaceAgentsByWorkspaceAndBuildNumberParams{
+		WorkspaceID: workspace.ID,
+		BuildNumber: build.BuildNumber,
+	})
+	if err != nil {
+		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to get workspace agents.",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	var agent *database.WorkspaceAgent
+	for i := range agents {
+		if agents[i].Name == agentName {
+			agent = &agents[i]
+			break
+		}
+	}
+	if agent == nil {
+		httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
+			Message: fmt.Sprintf("External agent '%s' not found in workspace.", agentName),
+		})
+		return
+	}
+
+	if agent.AuthInstanceID.Valid {
+		httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
+			Message: "External agent is authenticated with an instance ID.",
+		})
+		return
+	}
+
+	initScriptURL := fmt.Sprintf("%s/api/v2/init-script/%s/%s", api.AccessURL.String(), agent.OperatingSystem, agent.Architecture)
+	command := fmt.Sprintf("CODER_AGENT_TOKEN=%q curl -fsSL %q | sh", agent.AuthToken.String(), initScriptURL)
+	if agent.OperatingSystem == "windows" {
+		command = fmt.Sprintf("$env:CODER_AGENT_TOKEN=%q; iwr -useb %q | iex", agent.AuthToken.String(), initScriptURL)
+	}
+
+	httpapi.Write(ctx, rw, http.StatusOK, codersdk.ExternalAgentCredentials{
+		AgentToken: agent.AuthToken.String(),
+		Command:    command,
+	})
+}

enterprise/coderd/workspaceagents_test.go

@@ -3,6 +3,7 @@ package coderd_test
 import (
 	"context"
 	"crypto/tls"
+	"database/sql"
 	"fmt"
 	"net/http"
 	"os"
@@ -12,6 +13,7 @@ import (
 	"time"
 
 	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbfake"
 	"github.com/coder/coder/v2/coderd/database/dbgen"
 	"github.com/coder/coder/v2/coderd/database/dbtestutil"
 	"github.com/coder/coder/v2/provisionersdk"
@@ -344,3 +346,123 @@ func setupWorkspaceAgent(t *testing.T, client *codersdk.Client, user codersdk.Cr
 
 	return setupResp{workspace, sdkAgent, agnt}
 }
+
+func TestWorkspaceExternalAgentCredentials(t *testing.T) {
+	t.Parallel()
+
+	client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{
+		LicenseOptions: &coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspaceExternalAgent: 1,
+			},
+		},
+	})
+
+	t.Run("Success - linux", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).Seed(database.WorkspaceBuild{
+			HasExternalAgent: sql.NullBool{
+				Bool:  true,
+				Valid: true,
+			},
+		}).Resource(&proto.Resource{
+			Name: "test-agent",
+			Type: "coder_external_agent",
+		}).WithAgent(func(a []*proto.Agent) []*proto.Agent {
+			a[0].Name = "test-agent"
+			a[0].OperatingSystem = "linux"
+			a[0].Architecture = "amd64"
+			return a
+		}).Do()
+
+		credentials, err := client.WorkspaceExternalAgentCredentials(
+			ctx, r.Workspace.ID, "test-agent")
+		require.NoError(t, err)
+
+		require.Equal(t, r.AgentToken, credentials.AgentToken)
+		expectedCommand := fmt.Sprintf("CODER_AGENT_TOKEN=%q curl -fsSL \"%s/api/v2/init-script/linux/amd64\" | sh", r.AgentToken, client.URL)
+		require.Equal(t, expectedCommand, credentials.Command)
+	})
+
+	t.Run("Success - windows", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).Resource(&proto.Resource{
+			Name: "test-agent",
+			Type: "coder_external_agent",
+		}).Seed(database.WorkspaceBuild{
+			HasExternalAgent: sql.NullBool{
+				Bool:  true,
+				Valid: true,
+			},
+		}).WithAgent(func(a []*proto.Agent) []*proto.Agent {
+			a[0].Name = "test-agent"
+			a[0].OperatingSystem = "windows"
+			a[0].Architecture = "amd64"
+			return a
+		}).Do()
+
+		credentials, err := client.WorkspaceExternalAgentCredentials(
+			ctx, r.Workspace.ID, "test-agent")
+		require.NoError(t, err)
+
+		require.Equal(t, r.AgentToken, credentials.AgentToken)
+		expectedCommand := fmt.Sprintf("$env:CODER_AGENT_TOKEN=%q; iwr -useb \"%s/api/v2/init-script/windows/amd64\" | iex", r.AgentToken, client.URL)
+		require.Equal(t, expectedCommand, credentials.Command)
+	})
+
+	t.Run("WithInstanceID - should return 404", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).Seed(database.WorkspaceBuild{
+			HasExternalAgent: sql.NullBool{
+				Bool:  true,
+				Valid: true,
+			},
+		}).Resource(&proto.Resource{
+			Name: "test-agent",
+			Type: "coder_external_agent",
+		}).WithAgent(func(a []*proto.Agent) []*proto.Agent {
+			a[0].Name = "test-agent"
+			a[0].Auth = &proto.Agent_InstanceId{
+				InstanceId: uuid.New().String(),
+			}
+			return a
+		}).Do()
+
+		_, err := client.WorkspaceExternalAgentCredentials(ctx, r.Workspace.ID, "test-agent")
+		require.Error(t, err)
+		var apiErr *codersdk.Error
+		require.ErrorAs(t, err, &apiErr)
+		require.Equal(t, "External agent is authenticated with an instance ID.", apiErr.Message)
+	})
+
+	t.Run("No external agent - should return 404", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		r := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+			OrganizationID: user.OrganizationID,
+			OwnerID:        user.UserID,
+		}).Do()
+
+		_, err := client.WorkspaceExternalAgentCredentials(ctx, r.Workspace.ID, "test-agent")
+		require.Error(t, err)
+		var apiErr *codersdk.Error
+		require.ErrorAs(t, err, &apiErr)
+		require.Equal(t, "Workspace does not have an external agent.", apiErr.Message)
+	})
+}