opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: track resource replacements when claiming a prebuilt workspace (#17571)

Browse Source 
Closes https://github.com/coder/internal/issues/369

We can't know whether a replacement (i.e. drift of terraform state
leading to a resource needing to be deleted/recreated) will take place
apriori; we can only detect it at `plan` time, because the provider
decides whether a resource must be replaced and it cannot be inferred
through static analysis of the template.

**This is likely to be the most common gotcha with using prebuilds,
since it requires a slight template modification to use prebuilds
effectively**, so let's head this off before it's an issue for
customers.

Drift details will now be logged in the workspace build logs:


![image](https://github.com/user-attachments/assets/da1988b6-2cbe-4a79-a3c5-ea29891f3d6f)

Plus a notification will be sent to template admins when this situation
arises:


![image](https://github.com/user-attachments/assets/39d555b1-a262-4a3e-b529-03b9f23bf66a)

A new metric - `coderd_prebuilt_workspaces_resource_replacements_total`
- will also increment each time a workspace encounters replacements.

We only track _that_ a resource replacement occurred, not how many. Just
one is enough to ruin a prebuild, but we can't know apriori which
replacement would cause this.
For example, say we have 2 replacements: a `docker_container` and a
`null_resource`; we don't know which one might
cause an issue (or indeed if either would), so we just track the
replacement.

---------

Signed-off-by: Danny Kopping <dannykopping@gmail.com>
This commit is contained in:
Danny Kopping
2025-05-14 14:52:22 +02:00
committed by GitHub
parent e75d1c1ce5
commit 6e967780c9
33 changed files with 2048 additions and 969 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/provisionerdserver/provisionerdserver.go
+14 -2
View File
@@ -28,6 +28,7 @@ import (
protobuf "google.golang.org/protobuf/proto"
"cdr.dev/slog"
"github.com/coder/coder/v2/codersdk/drpcsdk"
"github.com/coder/quartz"
@@ -116,6 +117,7 @@ type server struct {
UserQuietHoursScheduleStore *atomic.Pointer[schedule.UserQuietHoursScheduleStore]
DeploymentValues *codersdk.DeploymentValues
NotificationsEnqueuer notifications.Enqueuer
PrebuildsOrchestrator *atomic.Pointer[prebuilds.ReconciliationOrchestrator]
OIDCConfig promoauth.OAuth2Config
@@ -151,8 +153,7 @@ func (t Tags) Valid() error {
return nil
}
func NewServer(
lifecycleCtx context.Context,
func NewServer(lifecycleCtx context.Context,
accessURL *url.URL,
id uuid.UUID,
organizationID uuid.UUID,
@@ -171,6 +172,7 @@ func NewServer(
deploymentValues *codersdk.DeploymentValues,
options Options,
enqueuer notifications.Enqueuer,
prebuildsOrchestrator *atomic.Pointer[prebuilds.ReconciliationOrchestrator],
) (proto.DRPCProvisionerDaemonServer, error) {
// Fail-fast if pointers are nil
if lifecycleCtx == nil {
@@ -235,6 +237,7 @@ func NewServer(
acquireJobLongPollDur: options.AcquireJobLongPollDur,
heartbeatInterval: options.HeartbeatInterval,
heartbeatFn: options.HeartbeatFn,
PrebuildsOrchestrator: prebuildsOrchestrator,
}
if s.heartbeatFn == nil {
@@ -1828,6 +1831,15 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
})
}
if s.PrebuildsOrchestrator != nil {
// Track resource replacements, if there are any.
orchestrator := s.PrebuildsOrchestrator.Load()
if resourceReplacements := completed.GetWorkspaceBuild().GetResourceReplacements(); orchestrator != nil && len(resourceReplacements) > 0 {
// Fire and forget. Bind to the lifecycle of the server so shutdowns are handled gracefully.
go (*orchestrator).TrackResourceReplacement(s.lifecycleCtx, workspace.ID, workspaceBuild.ID, resourceReplacements)
}
}
msg, err := json.Marshal(wspubsub.WorkspaceEvent{
Kind: wspubsub.WorkspaceEventKindStateChange,
WorkspaceID: workspace.ID,
coderd/provisionerdserver/provisionerdserver_test.go
+114 -6
View File
@@ -26,11 +26,6 @@ import (
"github.com/coder/quartz"
"github.com/coder/serpent"
"github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/provisionerdserver"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/codersdk/agentsdk"
"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/coderd/audit"
"github.com/coder/coder/v2/coderd/database"
@@ -42,11 +37,15 @@ import (
"github.com/coder/coder/v2/coderd/externalauth"
"github.com/coder/coder/v2/coderd/notifications"
"github.com/coder/coder/v2/coderd/notifications/notificationstest"
agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/provisionerdserver"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/coderd/schedule"
"github.com/coder/coder/v2/coderd/schedule/cron"
"github.com/coder/coder/v2/coderd/telemetry"
"github.com/coder/coder/v2/coderd/wspubsub"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/agentsdk"
"github.com/coder/coder/v2/provisionerd/proto"
"github.com/coder/coder/v2/provisionersdk"
sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
@@ -1889,7 +1888,7 @@ func TestCompleteJob(t *testing.T) {
// GIVEN something is listening to process workspace reinitialization:
reinitChan := make(chan agentsdk.ReinitializationEvent, 1) // Buffered to simplify test structure
cancel, err := prebuilds.NewPubsubWorkspaceClaimListener(ps, testutil.Logger(t)).ListenForWorkspaceClaims(ctx, workspace.ID, reinitChan)
cancel, err := agplprebuilds.NewPubsubWorkspaceClaimListener(ps, testutil.Logger(t)).ListenForWorkspaceClaims(ctx, workspace.ID, reinitChan)
require.NoError(t, err)
defer cancel()
@@ -1917,6 +1916,106 @@ func TestCompleteJob(t *testing.T) {
})
}
})
t.Run("PrebuiltWorkspaceClaimWithResourceReplacements", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
// Given: a mock prebuild orchestrator which stores calls to TrackResourceReplacement.
done := make(chan struct{})
orchestrator := &mockPrebuildsOrchestrator{
ReconciliationOrchestrator: agplprebuilds.DefaultReconciler,
done: done,
}
srv, db, ps, pd := setup(t, false, &overrides{
prebuildsOrchestrator: orchestrator,
})
// Given: a workspace build which simulates claiming a prebuild.
user := dbgen.User(t, db, database.User{})
template := dbgen.Template(t, db, database.Template{
Name: "template",
Provisioner: database.ProvisionerTypeEcho,
OrganizationID: pd.OrganizationID,
})
file := dbgen.File(t, db, database.File{CreatedBy: user.ID})
workspaceTable := dbgen.Workspace(t, db, database.WorkspaceTable{
TemplateID: template.ID,
OwnerID: user.ID,
OrganizationID: pd.OrganizationID,
})
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: pd.OrganizationID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
JobID: uuid.New(),
})
build := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
WorkspaceID: workspaceTable.ID,
InitiatorID: user.ID,
TemplateVersionID: version.ID,
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
})
job := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
FileID: file.ID,
InitiatorID: user.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: build.ID,
PrebuiltWorkspaceBuildStage: sdkproto.PrebuiltWorkspaceBuildStage_CLAIM,
})),
OrganizationID: pd.OrganizationID,
})
_, err := db.AcquireProvisionerJob(ctx, database.AcquireProvisionerJobParams{
OrganizationID: pd.OrganizationID,
WorkerID: uuid.NullUUID{
UUID: pd.ID,
Valid: true,
},
Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
})
require.NoError(t, err)
// When: a replacement is encountered.
replacements := []*sdkproto.ResourceReplacement{
{
Resource: "docker_container[0]",
Paths: []string{"env"},
},
}
// Then: CompleteJob makes a call to TrackResourceReplacement.
_, err = srv.CompleteJob(ctx, &proto.CompletedJob{
JobId: job.ID.String(),
Type: &proto.CompletedJob_WorkspaceBuild_{
WorkspaceBuild: &proto.CompletedJob_WorkspaceBuild{
State: []byte{},
ResourceReplacements: replacements,
},
},
})
require.NoError(t, err)
// Then: the replacements are as we expected.
testutil.RequireReceive(ctx, t, done)
require.Equal(t, replacements, orchestrator.replacements)
})
}
type mockPrebuildsOrchestrator struct {
agplprebuilds.ReconciliationOrchestrator
replacements []*sdkproto.ResourceReplacement
done chan struct{}
}
func (m *mockPrebuildsOrchestrator) TrackResourceReplacement(_ context.Context, _, _ uuid.UUID, replacements []*sdkproto.ResourceReplacement) {
m.replacements = replacements
m.done <- struct{}{}
}
func TestInsertWorkspacePresetsAndParameters(t *testing.T) {
@@ -2803,6 +2902,7 @@ type overrides struct {
heartbeatInterval time.Duration
auditor audit.Auditor
notificationEnqueuer notifications.Enqueuer
prebuildsOrchestrator agplprebuilds.ReconciliationOrchestrator
}
func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisionerDaemonServer, database.Store, pubsub.Pubsub, database.ProvisionerDaemon) {
@@ -2884,6 +2984,13 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
})
require.NoError(t, err)
prebuildsOrchestrator := ov.prebuildsOrchestrator
if prebuildsOrchestrator == nil {
prebuildsOrchestrator = agplprebuilds.DefaultReconciler
}
var op atomic.Pointer[agplprebuilds.ReconciliationOrchestrator]
op.Store(&prebuildsOrchestrator)
srv, err := provisionerdserver.NewServer(
ov.ctx,
&url.URL{},
@@ -2911,6 +3018,7 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
HeartbeatFn: ov.heartbeatFn,
},
notifEnq,
&op,
)
require.NoError(t, err)
return srv, db, ps, daemon