feat: plumb user secrets through provisioner chain to terraform (#24542)

This change passes user secrets from coderd to the Terraform process at
workspace build time so the `data.coder_secret` data source in
terraform-provider-coder can resolve values at plan time.

Secrets traverse two proto hops: `provisionerdserver` fetches them
via`ListUserSecretsWithValues`, attaches them to
`AcquiredJob.WorkspaceBuild.user_secrets` on `provisionerd.proto`;
`runner.go` forwards into `PlanRequest.user_secrets` on
`provisioner.proto`; the Terraform provisioner encodes each as
`CODER_SECRET_ENV_<name>` or `CODER_SECRET_FILE_<hex(path)>` before
invoking `terraform plan`. Only plan requests carry secrets; apply runs
with `nil` because values are baked into plan state.

Fetch is gated on a workspace transitioning to start. stop and delete
transitions never carry secrets, so revoking or deleting a stored secret
cannot make a workspace unstoppable. DB errors on the fetch fail the job
outright rather than silently continuing with an empty secret set.

Note that user secrets will be stored in the workspace_builds table in
provisioner_state with other Terraform state (including other sensitive data).
This commit is contained in:
Zach
2026-04-27 08:26:07 -06:00
committed by GitHub
parent 66abd8a271
commit 79735f2d45
15 changed files with 1471 additions and 686 deletions
+2 -1
View File
@@ -245,6 +245,7 @@ var (
rbac.ResourceWorkspaceAgentDevcontainers.Type: {policy.ActionCreate},
// Provisionerd creates usage events
rbac.ResourceUsageEvent.Type: {policy.ActionCreate},
rbac.ResourceUserSecret.Type: {policy.ActionRead},
}),
User: []rbac.Permission{},
ByOrgID: map[string]rbac.OrgPermissions{},
@@ -5821,7 +5822,7 @@ func (q *querier) ListUserSecretsWithValues(ctx context.Context, userID uuid.UUI
// This query returns decrypted secret values and must only be called
// from system contexts (provisioner, agent manifest). REST API
// handlers should use ListUserSecrets (metadata only).
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceUserSecret); err != nil {
return nil, err
}
return q.db.ListUserSecretsWithValues(ctx, userID)
+1 -1
View File
@@ -5667,7 +5667,7 @@ func (s *MethodTestSuite) TestUserSecrets() {
secret := testutil.Fake(s.T(), faker, database.UserSecret{UserID: user.ID})
dbm.EXPECT().ListUserSecretsWithValues(gomock.Any(), user.ID).Return([]database.UserSecret{secret}, nil).AnyTimes()
check.Args(user.ID).
Asserts(rbac.ResourceSystem, policy.ActionRead).
Asserts(rbac.ResourceUserSecret, policy.ActionRead).
Returns([]database.UserSecret{secret})
}))
s.Run("CreateUserSecret", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+7 -3
View File
@@ -1590,8 +1590,8 @@ func PresetParameter(t testing.TB, db database.Store, seed database.InsertPreset
return parameters
}
func UserSecret(t testing.TB, db database.Store, seed database.UserSecret) database.UserSecret {
userSecret, err := db.CreateUserSecret(genCtx, database.CreateUserSecretParams{
func UserSecret(t testing.TB, db database.Store, seed database.UserSecret, mutators ...func(params *database.CreateUserSecretParams)) database.UserSecret {
params := database.CreateUserSecretParams{
ID: takeFirst(seed.ID, uuid.New()),
UserID: takeFirst(seed.UserID, uuid.New()),
Name: takeFirst(seed.Name, "secret-name"),
@@ -1600,7 +1600,11 @@ func UserSecret(t testing.TB, db database.Store, seed database.UserSecret) datab
ValueKeyID: seed.ValueKeyID,
EnvName: takeFirst(seed.EnvName, "SECRET_ENV_NAME"),
FilePath: takeFirst(seed.FilePath, "~/secret/file/path"),
})
}
for _, mut := range mutators {
mut(&params)
}
userSecret, err := db.CreateUserSecret(genCtx, params)
require.NoError(t, err, "failed to insert user secret")
return userSecret
}
@@ -591,6 +591,26 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
}
}
// Fetch user secrets for build-time injection, but only on start
// transitions where the workspace actually needs them.
var userSecrets []*sdkproto.UserSecretValue
if workspaceBuild.Transition == database.WorkspaceTransitionStart {
dbSecrets, err := s.Database.ListUserSecretsWithValues(ctx, owner.ID)
if err != nil {
return nil, failJob(fmt.Sprintf("get user secrets: %s", err))
}
for _, secret := range dbSecrets {
if secret.EnvName == "" && secret.FilePath == "" {
continue
}
userSecrets = append(userSecrets, &sdkproto.UserSecretValue{
EnvName: secret.EnvName,
FilePath: secret.FilePath,
Value: []byte(secret.Value),
})
}
}
transition, err := convertWorkspaceTransition(workspaceBuild.Transition)
if err != nil {
return nil, failJob(fmt.Sprintf("convert workspace transition: %s", err))
@@ -773,7 +793,8 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
TaskPrompt: task.Prompt,
TemplateVersionModulesFile: versionModulesFile,
},
LogLevel: input.LogLevel,
LogLevel: input.LogLevel,
UserSecrets: userSecrets,
},
}
case database.ProvisionerJobTypeTemplateVersionDryRun:
@@ -856,6 +856,368 @@ func TestAcquireJob(t *testing.T) {
require.NoError(t, err)
require.JSONEq(t, string(want), string(got))
})
t.Run(tc.name+"_UserSecrets", func(t *testing.T) {
t.Parallel()
srv, db, ps, pd := setup(t, false, nil)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancel()
user := dbgen.User(t, db, database.User{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: user.ID,
OrganizationID: pd.OrganizationID,
})
dbgen.GitSSHKey(t, db, database.GitSSHKey{UserID: user.ID})
// Create secrets: 4 valid + 1 that should be filtered out.
insert1 := database.UserSecret{ID: uuid.New(), UserID: user.ID, Name: "github-token", EnvName: "GITHUB_TOKEN", Value: "ghp_xxxx"}
secret1 := dbgen.UserSecret(t, db, insert1, func(p *database.CreateUserSecretParams) { p.FilePath = "" })
insert2 := database.UserSecret{ID: uuid.New(), UserID: user.ID, Name: "ssh-key", FilePath: "~/.ssh/id_rsa", Value: "private-key"}
secret2 := dbgen.UserSecret(t, db, insert2, func(p *database.CreateUserSecretParams) { p.EnvName = "" })
insert3 := database.UserSecret{ID: uuid.New(), UserID: user.ID, Name: "both", EnvName: "BOTH", FilePath: "/etc/both", Value: "both-val"}
secret3 := dbgen.UserSecret(t, db, insert3)
insert4 := database.UserSecret{ID: uuid.New(), UserID: user.ID, Name: "empty-value", Value: "", EnvName: "EMPTY_VALUE", FilePath: "/etc/empty-value"}
secret4 := dbgen.UserSecret(t, db, insert4, func(p *database.CreateUserSecretParams) { p.Value = "" })
insert5 := database.UserSecret{ID: uuid.New(), UserID: user.ID, Name: "no-injection", Value: "no-injection"}
_ = dbgen.UserSecret(t, db, insert5, func(p *database.CreateUserSecretParams) { p.EnvName = ""; p.FilePath = "" })
template := dbgen.Template(t, db, database.Template{
Name: "template",
Provisioner: database.ProvisionerTypeEcho,
OrganizationID: pd.OrganizationID,
CreatedBy: user.ID,
})
file := dbgen.File(t, db, database.File{CreatedBy: user.ID})
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
CreatedBy: user.ID,
OrganizationID: pd.OrganizationID,
TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true},
JobID: uuid.New(),
})
// Import version job
_ = dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
OrganizationID: pd.OrganizationID,
ID: version.JobID,
InitiatorID: user.ID,
FileID: file.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
Type: database.ProvisionerJobTypeTemplateVersionImport,
Input: must(json.Marshal(provisionerdserver.TemplateVersionImportJob{
TemplateVersionID: version.ID,
})),
})
workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
TemplateID: template.ID,
OwnerID: user.ID,
OrganizationID: pd.OrganizationID,
})
buildID := uuid.New()
dbJob := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
OrganizationID: pd.OrganizationID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: buildID,
})),
Tags: pd.Tags,
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
ID: buildID,
WorkspaceID: workspace.ID,
BuildNumber: 1,
JobID: dbJob.ID,
TemplateVersionID: version.ID,
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
})
startPublished := make(chan struct{})
var closed bool
closeStartSubscribe, err := ps.SubscribeWithErr(wspubsub.WorkspaceEventChannel(workspace.OwnerID),
wspubsub.HandleWorkspaceEvent(
func(_ context.Context, e wspubsub.WorkspaceEvent, err error) {
if err != nil {
return
}
if e.Kind == wspubsub.WorkspaceEventKindStateChange && e.WorkspaceID == workspace.ID {
if !closed {
close(startPublished)
closed = true
}
}
}))
require.NoError(t, err)
defer closeStartSubscribe()
// Grab jobs until we find the workspace build job.
var job *proto.AcquiredJob
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
job, err = tc.acquire(ctx, srv)
require.NoError(t, err)
_, ok := job.Type.(*proto.AcquiredJob_WorkspaceBuild_)
return ok
}, testutil.IntervalMedium)
select {
case <-startPublished:
case <-time.After(testutil.WaitShort):
t.Fatalf("timed out waiting for workspace build job to start")
}
wb := job.Type.(*proto.AcquiredJob_WorkspaceBuild_).WorkspaceBuild
require.Len(t, wb.UserSecrets, 4, "expected 4 secrets (the one with empty env_name and file_path should be filtered)")
// Re-sort by (env_name+file_path) before asserting field values.
// The terraform-provider-coder contract does not require a
// specific secret order, so this test intentionally does not
// assert the order produced by ListUserSecretsWithValues.
slices.SortFunc(wb.UserSecrets, func(a, b *sdkproto.UserSecretValue) int {
return strings.Compare(a.EnvName+a.FilePath, b.EnvName+b.FilePath)
})
// After sorting: []{secret3, secret4, secret1, secret2}
require.Equal(t, secret3.EnvName, wb.UserSecrets[0].EnvName)
require.Equal(t, secret3.FilePath, wb.UserSecrets[0].FilePath)
require.Equal(t, []byte(secret3.Value), wb.UserSecrets[0].Value)
require.Equal(t, secret4.EnvName, wb.UserSecrets[1].EnvName)
require.Equal(t, secret4.FilePath, wb.UserSecrets[1].FilePath)
require.Equal(t, []byte(secret4.Value), wb.UserSecrets[1].Value)
require.Equal(t, secret1.EnvName, wb.UserSecrets[2].EnvName)
require.Equal(t, secret1.FilePath, wb.UserSecrets[2].FilePath)
require.Equal(t, []byte(secret1.Value), wb.UserSecrets[2].Value)
require.Equal(t, secret2.EnvName, wb.UserSecrets[3].EnvName)
require.Equal(t, secret2.FilePath, wb.UserSecrets[3].FilePath)
require.Equal(t, []byte(secret2.Value), wb.UserSecrets[3].Value)
})
for _, transitionCase := range []struct {
name string
transition database.WorkspaceTransition
}{
{
name: "Stop",
transition: database.WorkspaceTransitionStop,
},
{
name: "Delete",
transition: database.WorkspaceTransitionDelete,
},
} {
t.Run(tc.name+"_UserSecrets"+transitionCase.name+"Transition", func(t *testing.T) {
// Secrets must never be populated on non-start transitions. The
// terraform-provider-coder data source intentionally returns empty
// values on stop/delete so that workspaces with revoked or deleted
// secrets can still be torn down.
t.Parallel()
srv, db, ps, pd := setup(t, false, nil)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancel()
user := dbgen.User(t, db, database.User{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: user.ID,
OrganizationID: pd.OrganizationID,
})
dbgen.GitSSHKey(t, db, database.GitSSHKey{UserID: user.ID})
// Give the owner a secret so we can prove it is not forwarded on a
// transition.
authCtx := dbauthz.AsSystemRestricted(ctx)
_, err := db.CreateUserSecret(authCtx, database.CreateUserSecretParams{
ID: uuid.New(),
UserID: user.ID,
Name: "github-token",
EnvName: "GITHUB_TOKEN",
Value: "must-not-leak",
})
require.NoError(t, err)
template := dbgen.Template(t, db, database.Template{
Name: "template",
Provisioner: database.ProvisionerTypeEcho,
OrganizationID: pd.OrganizationID,
CreatedBy: user.ID,
})
file := dbgen.File(t, db, database.File{CreatedBy: user.ID})
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
CreatedBy: user.ID,
OrganizationID: pd.OrganizationID,
TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true},
JobID: uuid.New(),
})
_ = dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
OrganizationID: pd.OrganizationID,
ID: version.JobID,
InitiatorID: user.ID,
FileID: file.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
Type: database.ProvisionerJobTypeTemplateVersionImport,
Input: must(json.Marshal(provisionerdserver.TemplateVersionImportJob{
TemplateVersionID: version.ID,
})),
})
workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
TemplateID: template.ID,
OwnerID: user.ID,
OrganizationID: pd.OrganizationID,
})
buildID := uuid.New()
dbJob := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
OrganizationID: pd.OrganizationID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: buildID,
})),
Tags: pd.Tags,
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
ID: buildID,
WorkspaceID: workspace.ID,
BuildNumber: 1,
JobID: dbJob.ID,
TemplateVersionID: version.ID,
Transition: transitionCase.transition,
Reason: database.BuildReasonInitiator,
})
var job *proto.AcquiredJob
for {
job, err = tc.acquire(ctx, srv)
require.NoError(t, err)
if _, ok := job.Type.(*proto.AcquiredJob_WorkspaceBuild_); ok {
break
}
}
wb := job.Type.(*proto.AcquiredJob_WorkspaceBuild_).WorkspaceBuild
require.Empty(t, wb.UserSecrets)
})
}
t.Run(tc.name+"_UserSecretsDBError", func(t *testing.T) {
// A DB failure fetching user secrets must surface as a provisioner
// job failure rather than being silently treated as "no secrets".
// Silent treatment would let a transient DB error cause a
// workspace to build without the secrets it needs, producing a
// confusing downstream terraform error about missing secrets that
// the user actually owns.
t.Parallel()
srv, db, ps, pd := setup(t, true, &overrides{
wrapDB: func(inner database.Store) database.Store {
return &errOnListUserSecretsWithValues{Store: inner}
},
})
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
defer cancel()
user := dbgen.User(t, db, database.User{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: user.ID,
OrganizationID: pd.OrganizationID,
})
dbgen.GitSSHKey(t, db, database.GitSSHKey{UserID: user.ID})
template := dbgen.Template(t, db, database.Template{
Name: "template",
Provisioner: database.ProvisionerTypeEcho,
OrganizationID: pd.OrganizationID,
CreatedBy: user.ID,
})
file := dbgen.File(t, db, database.File{CreatedBy: user.ID})
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
CreatedBy: user.ID,
OrganizationID: pd.OrganizationID,
TemplateID: uuid.NullUUID{UUID: template.ID, Valid: true},
JobID: uuid.New(),
})
_ = dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
OrganizationID: pd.OrganizationID,
ID: version.JobID,
InitiatorID: user.ID,
FileID: file.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
Type: database.ProvisionerJobTypeTemplateVersionImport,
Input: must(json.Marshal(provisionerdserver.TemplateVersionImportJob{
TemplateVersionID: version.ID,
})),
})
workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
TemplateID: template.ID,
OwnerID: user.ID,
OrganizationID: pd.OrganizationID,
})
buildID := uuid.New()
dbJob := dbgen.ProvisionerJob(t, db, ps, database.ProvisionerJob{
OrganizationID: pd.OrganizationID,
InitiatorID: user.ID,
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
FileID: file.ID,
Type: database.ProvisionerJobTypeWorkspaceBuild,
Input: must(json.Marshal(provisionerdserver.WorkspaceProvisionJob{
WorkspaceBuildID: buildID,
})),
Tags: pd.Tags,
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
ID: buildID,
WorkspaceID: workspace.ID,
BuildNumber: 1,
JobID: dbJob.ID,
TemplateVersionID: version.ID,
// Only start transitions fetch secrets.
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
})
var acquireErr error
for {
// Keep acquiring until we either get our build back (possible
// for the Deprecated path to return an empty AcquiredJob once
// its long-poll window elapses on unrelated jobs) or propagate
// an error.
job, err := tc.acquire(ctx, srv)
if err != nil {
acquireErr = err
break
}
if job != nil && job.JobId != "" {
t.Fatalf("expected acquire to error, got job %s", job.JobId)
}
}
require.ErrorContains(t, acquireErr, "request job was invalidated",
"DB error should surface as a job invalidation")
require.ErrorContains(t, acquireErr, "get user secrets",
"error should identify the failing operation")
require.ErrorContains(t, acquireErr, "ListUserSecretsWithValues query failed",
"underlying DB error message should be preserved")
// Confirm the provisioner job itself was marked as failed so the
// workspace build does not remain stuck in-progress.
authCtx := dbauthz.AsSystemRestricted(ctx)
gotJob, err := db.GetProvisionerJobByID(authCtx, dbJob.ID)
require.NoError(t, err)
require.True(t, gotJob.Error.Valid, "job should be marked with an error")
require.Contains(t, gotJob.Error.String, "get user secrets")
require.True(t, gotJob.CompletedAt.Valid, "job should be marked complete")
})
}
}
@@ -4795,6 +5157,9 @@ type overrides struct {
auditor audit.Auditor
notificationEnqueuer notifications.Enqueuer
prebuildsOrchestrator agplprebuilds.ReconciliationOrchestrator
// wrapDB wraps the raw DB before dbauthz.New. Use this to inject
// errors or observe calls on specific queries for a single test.
wrapDB func(database.Store) database.Store
}
func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisionerDaemonServer, database.Store, pubsub.Pubsub, database.ProvisionerDaemon) {
@@ -4895,6 +5260,9 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
// Use an authz wrapped database for the server to ensure permission checks
// work.
authorizer := rbac.NewStrictCachingAuthorizer(prometheus.NewRegistry())
if ov.wrapDB != nil {
db = ov.wrapDB(db)
}
serverDB := dbauthz.New(db, authorizer, logger, coderdtest.AccessControlStorePointer())
srv, err := provisionerdserver.NewServer(
ov.ctx,
@@ -5039,3 +5407,32 @@ func newFakeUsageInserter() (*coderdtest.UsageInserter, *atomic.Pointer[usage.In
poitr.Store(&inserter)
return fake, poitr
}
// errListUserSecretsWithValues is the sentinel returned by the test wrapper
// below. Its message is matched by assertions that verify the underlying DB
// error propagated through failJob's formatting. The chain is not preserved
// via errors.Is because failJob uses fmt.Sprintf, not %w.
var errListUserSecretsWithValues = xerrors.New("ListUserSecretsWithValues query failed")
// errOnListUserSecretsWithValues is a database.Store wrapper that errors only
// on ListUserSecretsWithValues. All other methods pass through to the
// underlying store. Used to simulate a transient DB failure on the secret
// fetch without breaking the rest of the acquire flow (user lookup, job
// update, etc.).
type errOnListUserSecretsWithValues struct {
database.Store
}
func (*errOnListUserSecretsWithValues) ListUserSecretsWithValues(context.Context, uuid.UUID) ([]database.UserSecret, error) {
return nil, errListUserSecretsWithValues
}
// InTx must be overridden to keep the wrapped store visible inside a
// transaction. Without this override, InTx would pass the raw inner store to
// its closure and tests would see the unwrapped behavior from anywhere that
// runs inside a transaction.
func (e *errOnListUserSecretsWithValues) InTx(fn func(database.Store) error, opts *database.TxOptions) error {
return e.Store.InTx(func(tx database.Store) error {
return fn(&errOnListUserSecretsWithValues{Store: tx})
}, opts)
}