opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(coderd): insert provisioner daemons (#11207)

Browse Source 
* Adds UpdateProvisionerDaemonLastSeenAt
* Adds heartbeat to provisioner daemons
* Inserts provisioner daemons to database upon start
* Ensures TagOwner is an empty string and not nil
* Adds COALESCE() in idx_provisioner_daemons_name_owner_key
This commit is contained in:
Cian Johnston
2023-12-18 16:44:52 +00:00
committed by GitHub
parent a6901ae2c5
commit 213b768785
26 changed files with 548 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
enterprise/cli/provisionerdaemons_test.go
+77 -4
View File
@@ -4,13 +4,16 @@ import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/cli/clitest"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/enterprise/coderd/coderdenttest"
"github.com/coder/coder/v2/enterprise/coderd/license"
"github.com/coder/coder/v2/provisionersdk"
"github.com/coder/coder/v2/pty/ptytest"
"github.com/coder/coder/v2/testutil"
)
@@ -35,6 +38,17 @@ func TestProvisionerDaemon_PSK(t *testing.T) {
clitest.Start(t, inv)
pty.ExpectMatchContext(ctx, "starting provisioner daemon")
pty.ExpectMatchContext(ctx, "matt-daemon")
var daemons []codersdk.ProvisionerDaemon
require.Eventually(t, func() bool {
daemons, err = client.ProvisionerDaemons(ctx)
if err != nil {
return false
}
return len(daemons) == 1
}, testutil.WaitShort, testutil.IntervalSlow)
require.Equal(t, "matt-daemon", daemons[0].Name)
require.Equal(t, provisionersdk.ScopeOrganization, daemons[0].Tags[provisionersdk.TagScope])
}
func TestProvisionerDaemon_SessionToken(t *testing.T) {
@@ -49,14 +63,61 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
},
},
})
anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
inv, conf := newCLI(t, "provisionerd", "start", "--tag", "scope=user")
anotherClient, anotherUser := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
inv, conf := newCLI(t, "provisionerd", "start", "--tag", "scope=user", "--name", "my-daemon")
clitest.SetupConfig(t, anotherClient, conf)
pty := ptytest.New(t).Attach(inv)
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
defer cancel()
clitest.Start(t, inv)
pty.ExpectMatchContext(ctx, "starting provisioner daemon")
var daemons []codersdk.ProvisionerDaemon
var err error
require.Eventually(t, func() bool {
daemons, err = client.ProvisionerDaemons(ctx)
if err != nil {
return false
}
return len(daemons) == 1
}, testutil.WaitShort, testutil.IntervalSlow)
assert.Equal(t, "my-daemon", daemons[0].Name)
assert.Equal(t, provisionersdk.ScopeUser, daemons[0].Tags[provisionersdk.TagScope])
assert.Equal(t, anotherUser.ID.String(), daemons[0].Tags[provisionersdk.TagOwner])
})
t.Run("ScopeAnotherUser", func(t *testing.T) {
t.Parallel()
client, admin := coderdenttest.New(t, &coderdenttest.Options{
ProvisionerDaemonPSK: "provisionersftw",
LicenseOptions: &coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureExternalProvisionerDaemons: 1,
},
},
})
anotherClient, anotherUser := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
inv, conf := newCLI(t, "provisionerd", "start", "--tag", "scope=user", "--tag", "owner="+admin.UserID.String(), "--name", "my-daemon")
clitest.SetupConfig(t, anotherClient, conf)
pty := ptytest.New(t).Attach(inv)
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
defer cancel()
clitest.Start(t, inv)
pty.ExpectMatchContext(ctx, "starting provisioner daemon")
var daemons []codersdk.ProvisionerDaemon
var err error
require.Eventually(t, func() bool {
daemons, err = client.ProvisionerDaemons(ctx)
if err != nil {
return false
}
return len(daemons) == 1
}, testutil.WaitShort, testutil.IntervalSlow)
assert.Equal(t, "my-daemon", daemons[0].Name)
assert.Equal(t, provisionersdk.ScopeUser, daemons[0].Tags[provisionersdk.TagScope])
// This should get clobbered to the user who started the daemon.
assert.Equal(t, anotherUser.ID.String(), daemons[0].Tags[provisionersdk.TagOwner])
})
t.Run("ScopeOrg", func(t *testing.T) {
@@ -69,13 +130,25 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
},
},
})
anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID)
inv, conf := newCLI(t, "provisionerd", "start", "--tag", "scope=organization")
anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID, rbac.RoleTemplateAdmin())
inv, conf := newCLI(t, "provisionerd", "start", "--tag", "scope=organization", "--name", "org-daemon")
clitest.SetupConfig(t, anotherClient, conf)
pty := ptytest.New(t).Attach(inv)
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
defer cancel()
clitest.Start(t, inv)
pty.ExpectMatchContext(ctx, "starting provisioner daemon")
var daemons []codersdk.ProvisionerDaemon
var err error
require.Eventually(t, func() bool {
daemons, err = client.ProvisionerDaemons(ctx)
if err != nil {
return false
}
return len(daemons) == 1
}, testutil.WaitShort, testutil.IntervalSlow)
assert.Equal(t, "org-daemon", daemons[0].Name)
assert.Equal(t, provisionersdk.ScopeOrganization, daemons[0].Tags[provisionersdk.TagScope])
})
}
enterprise/coderd/provisionerdaemons.go
+41 -4
View File
@@ -26,6 +26,8 @@ import (
"cdr.dev/slog"
"github.com/coder/coder/v2/coderd"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/httpapi"
"github.com/coder/coder/v2/coderd/httpmw"
"github.com/coder/coder/v2/coderd/provisionerdserver"
@@ -121,7 +123,7 @@ func (p *provisionerDaemonAuth) authorize(r *http.Request, tags map[string]strin
psk := r.Header.Get(codersdk.ProvisionerDaemonPSK)
if subtle.ConstantTimeCompare([]byte(p.psk), []byte(psk)) == 1 {
// If using PSK auth, the daemon is, by definition, scoped to the organization.
tags[provisionersdk.TagScope] = provisionersdk.ScopeOrganization
tags = provisionersdk.MutateTags(uuid.Nil, tags)
return tags, true
}
}
@@ -191,7 +193,10 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
if !authorized {
api.Logger.Warn(ctx, "unauthorized provisioner daemon serve request", slog.F("tags", tags))
httpapi.Write(ctx, rw, http.StatusForbidden,
codersdk.Response{Message: "You aren't allowed to create provisioner daemons"})
codersdk.Response{
Message: fmt.Sprintf("You aren't allowed to create provisioner daemons with scope %q", tags[provisionersdk.TagScope]),
},
)
return
}
api.Logger.Debug(ctx, "provisioner authorized", slog.F("tags", tags))
@@ -221,6 +226,35 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
slog.F("tags", tags),
)
authCtx := ctx
if r.Header.Get(codersdk.ProvisionerDaemonPSK) != "" {
//nolint:gocritic // PSK auth means no actor in request,
// so use system restricted.
authCtx = dbauthz.AsSystemRestricted(ctx)
}
// Create the daemon in the database.
now := dbtime.Now()
daemon, err := api.Database.UpsertProvisionerDaemon(authCtx, database.UpsertProvisionerDaemonParams{
Name: name,
Provisioners: provisioners,
Tags: tags,
CreatedAt: now,
LastSeenAt: sql.NullTime{Time: now, Valid: true},
Version: "", // TODO: provisionerd needs to send version
APIVersion: "1.0",
})
if err != nil {
if !xerrors.Is(err, context.Canceled) {
log.Error(ctx, "create provisioner daemon", slog.Error(err))
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error creating provisioner daemon.",
Detail: err.Error(),
})
}
return
}
api.AGPL.WebsocketWaitMutex.Lock()
api.AGPL.WebsocketWaitGroup.Add(1)
api.AGPL.WebsocketWaitMutex.Unlock()
@@ -264,11 +298,13 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
}
mux := drpcmux.New()
logger := api.Logger.Named(fmt.Sprintf("ext-provisionerd-%s", name))
srvCtx, srvCancel := context.WithCancel(ctx)
defer srvCancel()
logger.Info(ctx, "starting external provisioner daemon")
srv, err := provisionerdserver.NewServer(
api.ctx,
srvCtx,
api.AccessURL,
id,
daemon.ID,
logger,
provisioners,
tags,
@@ -308,6 +344,7 @@ func (api *API) provisionerDaemonServe(rw http.ResponseWriter, r *http.Request)
},
})
err = server.Serve(ctx, session)
srvCancel()
logger.Info(ctx, "provisioner daemon disconnected", slog.Error(err))
if err != nil && !xerrors.Is(err, io.EOF) {
_ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("serve: %s", err))
enterprise/coderd/provisionerdaemons_test.go
+32
View File
@@ -7,6 +7,7 @@ import (
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"cdr.dev/slog"
@@ -50,6 +51,10 @@ func TestProvisionerDaemonServe(t *testing.T) {
})
require.NoError(t, err)
srv.DRPCConn().Close()
daemons, err := client.ProvisionerDaemons(ctx) //nolint:gocritic // Test assertion.
require.NoError(t, err)
require.Len(t, daemons, 1)
})
t.Run("NoLicense", func(t *testing.T) {
@@ -163,6 +168,15 @@ func TestProvisionerDaemonServe(t *testing.T) {
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
require.NoError(t, err)
require.Eventually(t, func() bool {
daemons, err := client.ProvisionerDaemons(context.Background())
assert.NoError(t, err, "failed to get provisioner daemons")
return len(daemons) > 0 &&
assert.Equal(t, t.Name(), daemons[0].Name) &&
assert.Equal(t, provisionersdk.ScopeUser, daemons[0].Tags[provisionersdk.TagScope]) &&
assert.Equal(t, user.UserID.String(), daemons[0].Tags[provisionersdk.TagOwner])
}, testutil.WaitShort, testutil.IntervalMedium)
version, err := client.CreateTemplateVersion(context.Background(), user.OrganizationID, codersdk.CreateTemplateVersionRequest{
Name: "example",
StorageMethod: codersdk.ProvisionerStorageMethodFile,
@@ -211,6 +225,12 @@ func TestProvisionerDaemonServe(t *testing.T) {
require.NoError(t, err)
err = srv.DRPCConn().Close()
require.NoError(t, err)
daemons, err := client.ProvisionerDaemons(ctx) //nolint:gocritic // Test assertion.
require.NoError(t, err)
if assert.Len(t, daemons, 1) {
assert.Equal(t, provisionersdk.ScopeOrganization, daemons[0].Tags[provisionersdk.TagScope])
}
})
t.Run("PSK_daily_cost", func(t *testing.T) {
@@ -346,6 +366,10 @@ func TestProvisionerDaemonServe(t *testing.T) {
var apiError *codersdk.Error
require.ErrorAs(t, err, &apiError)
require.Equal(t, http.StatusForbidden, apiError.StatusCode())
daemons, err := client.ProvisionerDaemons(ctx) //nolint:gocritic // Test assertion.
require.NoError(t, err)
require.Len(t, daemons, 0)
})
t.Run("NoAuth", func(t *testing.T) {
@@ -376,6 +400,10 @@ func TestProvisionerDaemonServe(t *testing.T) {
var apiError *codersdk.Error
require.ErrorAs(t, err, &apiError)
require.Equal(t, http.StatusForbidden, apiError.StatusCode())
daemons, err := client.ProvisionerDaemons(ctx) //nolint:gocritic // Test assertion.
require.NoError(t, err)
require.Len(t, daemons, 0)
})
t.Run("NoPSK", func(t *testing.T) {
@@ -406,5 +434,9 @@ func TestProvisionerDaemonServe(t *testing.T) {
var apiError *codersdk.Error
require.ErrorAs(t, err, &apiError)
require.Equal(t, http.StatusForbidden, apiError.StatusCode())
daemons, err := client.ProvisionerDaemons(ctx) //nolint:gocritic // Test assertion.
require.NoError(t, err)
require.Len(t, daemons, 0)
})
}