feat: order lifecycle scripts by their dependencies (#30765)

feat(agent): enforce lifecycle script ordering

Honor `coder_script_order` dependencies for startup and shutdown scripts
using the existing outcome-aware unit manager. Cron, legacy scripts,
and scripts without dependencies retain their existing behavior.

- Wait for all prerequisites before running. `success` requires a
  successful prerequisite; `completion` also accepts failure, timeout,
  or skip. Skip dependents as soon as a required success becomes
  impossible, preserving the original failure as the lifecycle error.
- Reject invalid dependency graphs before any scripts start and write
  the reason to every selected script's log.
- Log unmet dependencies every 30 seconds and explain skips. Populate
  display names in the manifest so skip messages use readable names,
  falling back to Terraform addresses.
- Exclude dependency waits from script timeouts, record an outcome on
  every exit, and use fresh state per execution. Cancellation stops
  waiters without starting dependents or emitting skip messages.

Add regression coverage for ordering, skip propagation, timeouts,
cancellation, invalid graphs, and unchanged unordered execution.

Refs https://linear.app/codercom/issue/PLAT-549/phase-3-enforce-lifecycle-script-ordering

agent/agent.go

@@ -422,6 +422,7 @@ func (a *agent) init() {
 		LogDir:      a.logDir,
 		DataDirBase: a.scriptDataDir,
 		Logger:      a.logger,
+		Clock:       a.clock,
 		SSHServer:   sshSrv,
 		Filesystem:  a.filesystem,
 		GetScriptLogger: func(logSourceID uuid.UUID) agentscripts.ScriptLogger {

agent/agentscripts/agentscripts.go

@@ -26,6 +26,7 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/quartz"
 )
 
 var (
@@ -56,10 +57,17 @@ type Options struct {
 	SSHServer       *agentssh.Server
 	Filesystem      afero.Fs
 	GetScriptLogger func(logSourceID uuid.UUID) ScriptLogger
+	// Clock drives the periodic "Waiting for dependencies" line for
+	// ordered scripts, so a test can advance it. Script timeouts keep
+	// using the context deadline.
+	Clock quartz.Clock
 }
 
 // New creates a runner for the provided scripts.
 func New(opts Options) *Runner {
+	if opts.Clock == nil {
+		opts.Clock = quartz.NewReal()
+	}
 	cronCtx, cronCtxCancel := context.WithCancel(context.Background())
 	return &Runner{
 		Options:       opts,
@@ -211,7 +219,7 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 		return initErr
 	}
 
-	var eg errgroup.Group
+	var scripts []codersdk.WorkspaceAgentScript
 	for _, script := range r.scripts {
 		runScript := (option == ExecuteStartScripts && script.RunOnStart) ||
 			(option == ExecuteStopScripts && script.RunOnStop) ||
@@ -221,18 +229,47 @@ func (r *Runner) Execute(ctx context.Context, option ExecuteOption) error {
 		if !runScript {
 			continue
 		}
+		scripts = append(scripts, script)
+	}
 
+	if (option == ExecuteStartScripts || option == ExecuteStopScripts) && anyOrdered(scripts) {
+		executor := lifecycleExecutor{
+			clock:           r.Clock,
+			logger:          r.Logger,
+			getScriptLogger: r.GetScriptLogger,
+			run:             r.runTracked,
+		}
+		return executor.execute(ctx, scripts, option)
+	}
+
+	var eg errgroup.Group
+	for _, script := range scripts {
 		eg.Go(func() error {
-			err := r.trackRun(ctx, script, option)
-			if err != nil {
-				return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
-			}
-			return nil
+			return r.runTracked(ctx, script, option)
 		})
 	}
 	return eg.Wait()
 }
 
+// runTracked runs one script and wraps its error the same way for both the
+// ordered and the unordered path.
+func (r *Runner) runTracked(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	err := r.trackRun(ctx, script, option)
+	if err != nil {
+		return xerrors.Errorf("run agent script %q: %w", script.LogSourceID, err)
+	}
+	return nil
+}
+
+func anyOrdered(scripts []codersdk.WorkspaceAgentScript) bool {
+	for _, script := range scripts {
+		if len(script.Dependencies) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 // trackRun wraps "run" with metrics.
 func (r *Runner) trackRun(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
 	err := r.run(ctx, script, option)

agent/agentscripts/agentscripts_test.go

@@ -341,6 +341,126 @@ func (*executeOptionTestLogger) Flush(context.Context) error {
 	return nil
 }
 
+func TestExecuteFastPath(t *testing.T) {
+	t.Parallel()
+
+	t.Run("AddressesWithoutDependencies", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+		logs := newRecordingScriptLogs()
+		runner := setup(t, logs.logger)
+		defer runner.Close()
+		aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+
+		scripts := []codersdk.WorkspaceAgentScript{
+			{ID: uuid.New(), LogSourceID: uuid.New(), ResourceAddress: "coder_script.a", Script: "echo a", RunOnStart: true},
+			{ID: uuid.New(), LogSourceID: uuid.New(), ResourceAddress: "coder_script.b", Script: "echo b", RunOnStart: true},
+		}
+		require.NoError(t, runner.Init(scripts, aAPI.ScriptCompleted))
+		require.NoError(t, runner.Execute(ctx, agentscripts.ExecuteStartScripts))
+		require.Equal(t, []string{"a"}, logs.outputs(scripts[0].LogSourceID))
+		require.Equal(t, []string{"b"}, logs.outputs(scripts[1].LogSourceID))
+	})
+
+	t.Run("DependencyOnlyAtStop", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+		logs := newRecordingScriptLogs()
+		runner := setup(t, logs.logger)
+		defer runner.Close()
+		aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+
+		// "exit 1" alone runs the same under sh and PowerShell.
+		save := codersdk.WorkspaceAgentScript{
+			ID: uuid.New(), LogSourceID: uuid.New(), DisplayName: "Save state",
+			ResourceAddress: "coder_script.save_state", Script: "exit 1",
+			RunOnStart: true, RunOnStop: true,
+		}
+		stopDB := codersdk.WorkspaceAgentScript{
+			ID: uuid.New(), LogSourceID: uuid.New(), DisplayName: "Stop database",
+			ResourceAddress: "coder_script.stop_db", Script: "echo stopping", RunOnStop: true,
+			Dependencies: []codersdk.WorkspaceAgentScriptDependency{{
+				PrerequisiteResourceAddress: "coder_script.save_state",
+				Requirement:                 codersdk.WorkspaceAgentScriptDependencyRequirementSuccess,
+			}},
+		}
+		require.NoError(t, runner.Init([]codersdk.WorkspaceAgentScript{save, stopDB}, aAPI.ScriptCompleted))
+
+		require.Error(t, runner.Execute(ctx, agentscripts.ExecuteStartScripts))
+		require.Empty(t, logs.outputs(stopDB.LogSourceID))
+
+		require.Error(t, runner.Execute(ctx, agentscripts.ExecuteStopScripts))
+		require.Equal(t, []string{`Skipping script: dependency "Save state" failed.`}, logs.outputs(stopDB.LogSourceID))
+	})
+}
+
+func TestExecuteOrderedTimeoutStartsAfterWaiting(t *testing.T) {
+	t.Parallel()
+	if runtime.GOOS == "windows" {
+		t.Skip("uses sleep")
+	}
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logs := newRecordingScriptLogs()
+	runner := setup(t, logs.logger)
+	defer runner.Close()
+	aAPI := agenttest.NewFakeAgentAPI(t, testutil.Logger(t), nil, nil)
+
+	clone := codersdk.WorkspaceAgentScript{
+		ID: uuid.New(), LogSourceID: uuid.New(), ResourceAddress: "coder_script.clone",
+		Script: "sleep 2", RunOnStart: true,
+	}
+	install := codersdk.WorkspaceAgentScript{
+		ID: uuid.New(), LogSourceID: uuid.New(), ResourceAddress: "coder_script.install",
+		Script: "echo installed", RunOnStart: true, Timeout: time.Second,
+		Dependencies: []codersdk.WorkspaceAgentScriptDependency{{
+			PrerequisiteResourceAddress: "coder_script.clone",
+			Requirement:                 codersdk.WorkspaceAgentScriptDependencyRequirementSuccess,
+		}},
+	}
+	require.NoError(t, runner.Init([]codersdk.WorkspaceAgentScript{clone, install}, aAPI.ScriptCompleted))
+	require.NoError(t, runner.Execute(ctx, agentscripts.ExecuteStartScripts))
+	require.Equal(t, []string{"installed"}, logs.outputs(install.LogSourceID))
+}
+
+// recordingScriptLogs keeps every line per log source so a test can read a
+// script's output and its dependency lines together.
+type recordingScriptLogs struct {
+	mu   sync.Mutex
+	byID map[uuid.UUID][]string
+}
+
+func newRecordingScriptLogs() *recordingScriptLogs {
+	return &recordingScriptLogs{byID: make(map[uuid.UUID][]string)}
+}
+
+func (l *recordingScriptLogs) logger(logSourceID uuid.UUID) agentscripts.ScriptLogger {
+	return recordingScriptLogger{logs: l, logSourceID: logSourceID}
+}
+
+func (l *recordingScriptLogs) outputs(logSourceID uuid.UUID) []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.byID[logSourceID]...)
+}
+
+type recordingScriptLogger struct {
+	logs        *recordingScriptLogs
+	logSourceID uuid.UUID
+}
+
+func (l recordingScriptLogger) Send(_ context.Context, logs ...agentsdk.Log) error {
+	l.logs.mu.Lock()
+	defer l.logs.mu.Unlock()
+	for _, log := range logs {
+		l.logs.byID[l.logSourceID] = append(l.logs.byID[l.logSourceID], log.Output)
+	}
+	return nil
+}
+
+func (recordingScriptLogger) Flush(context.Context) error {
+	return nil
+}
+
 func setup(t *testing.T, getScriptLogger func(logSourceID uuid.UUID) agentscripts.ScriptLogger) *agentscripts.Runner {
 	t.Helper()
 	if getScriptLogger == nil {

agent/agentscripts/script_order.go

@@ -0,0 +1,355 @@
+package agentscripts
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/agent/unit"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/quartz"
+)
+
+// Often enough to show the wait is alive, rare enough not to flood the log.
+const dependencyLogInterval = 30 * time.Second
+
+// lifecycleExecutor runs one start or stop execution whose scripts carry
+// dependencies.
+type lifecycleExecutor struct {
+	clock           quartz.Clock
+	logger          slog.Logger
+	getScriptLogger func(uuid.UUID) ScriptLogger
+	// run is Runner.runTracked; tests replace it with a fake.
+	run func(context.Context, codersdk.WorkspaceAgentScript, ExecuteOption) error
+}
+
+func (e lifecycleExecutor) execute(ctx context.Context, scripts []codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	manager, scriptsByAddress, err := lifecycleManager(scripts)
+	if err != nil {
+		// The agent log is not visible in the dashboard.
+		for _, script := range scripts {
+			e.sendScriptLog(ctx, script, "Lifecycle scripts did not run: "+err.Error())
+		}
+		return err
+	}
+	return e.executeWith(ctx, manager, scriptsByAddress, scripts, option)
+}
+
+// executeWith is separate from execute so a test can keep the manager and
+// read every unit's outcome after the execution ends.
+func (e lifecycleExecutor) executeWith(
+	ctx context.Context,
+	manager *unit.Manager,
+	scriptsByAddress map[string]codersdk.WorkspaceAgentScript,
+	scripts []codersdk.WorkspaceAgentScript,
+	option ExecuteOption,
+) error {
+	var group errgroup.Group
+	for _, script := range scripts {
+		group.Go(func() error {
+			// A script without an address is not a unit and nothing can
+			// wait for it, so it starts at once.
+			if script.ResourceAddress == "" {
+				return e.run(ctx, script, option)
+			}
+			return e.executeScript(ctx, manager, scriptsByAddress, script, option)
+		})
+	}
+	return group.Wait()
+}
+
+// lifecycleManager builds the manager for one execution and checks the
+// whole graph before any script starts, so an inconsistent dependency list
+// fails with nothing half done.
+func lifecycleManager(scripts []codersdk.WorkspaceAgentScript) (*unit.Manager, map[string]codersdk.WorkspaceAgentScript, error) {
+	manager := unit.NewManager()
+	scriptsByAddress := make(map[string]codersdk.WorkspaceAgentScript, len(scripts))
+
+	// Both ends of an edge must be registered before the edge is added, and
+	// a prerequisite can come after its dependent in the list.
+	for _, script := range scripts {
+		if script.ResourceAddress == "" {
+			if len(script.Dependencies) > 0 {
+				return nil, nil, xerrors.Errorf("script %q has dependencies but no Terraform resource address", script.DisplayName)
+			}
+			continue
+		}
+		if _, exists := scriptsByAddress[script.ResourceAddress]; exists {
+			return nil, nil, xerrors.Errorf("duplicate script resource address %q", script.ResourceAddress)
+		}
+		scriptsByAddress[script.ResourceAddress] = script
+		if err := manager.Register(unit.ID(script.ResourceAddress)); err != nil {
+			return nil, nil, xerrors.Errorf("register script %q: %w", script.ResourceAddress, err)
+		}
+	}
+
+	for _, script := range scripts {
+		for _, dependency := range script.Dependencies {
+			if _, exists := scriptsByAddress[dependency.PrerequisiteResourceAddress]; !exists {
+				return nil, nil, xerrors.Errorf(
+					"script %q depends on %q, which is not part of this lifecycle execution",
+					script.ResourceAddress, dependency.PrerequisiteResourceAddress,
+				)
+			}
+			requirement, err := lifecycleRequirement(dependency.Requirement)
+			if err != nil {
+				return nil, nil, xerrors.Errorf(
+					"script %q dependency on %q: %w",
+					script.ResourceAddress, dependency.PrerequisiteResourceAddress, err,
+				)
+			}
+			if err := manager.AddConditionalDependency(
+				unit.ID(script.ResourceAddress),
+				unit.ID(dependency.PrerequisiteResourceAddress),
+				requirement,
+			); err != nil {
+				return nil, nil, xerrors.Errorf(
+					"add script dependency %q after %q: %w",
+					script.ResourceAddress, dependency.PrerequisiteResourceAddress, err,
+				)
+			}
+		}
+	}
+	return manager, scriptsByAddress, nil
+}
+
+// lifecycleRequirement uses a switch rather than a cast so the codersdk and
+// unit requirement strings can change independently.
+func lifecycleRequirement(requirement codersdk.WorkspaceAgentScriptDependencyRequirement) (unit.Requirement, error) {
+	switch requirement {
+	case codersdk.WorkspaceAgentScriptDependencyRequirementSuccess:
+		return unit.RequirementSuccess, nil
+	case codersdk.WorkspaceAgentScriptDependencyRequirementCompletion:
+		return unit.RequirementCompletion, nil
+	default:
+		return "", xerrors.Errorf("unsupported requirement %q", requirement)
+	}
+}
+
+// executeScript waits for the manager's decision, then skips or runs the
+// script. Every exit records one terminal outcome, because dependents only
+// wake on one and the execution would otherwise stay open until the agent
+// shuts down. Only this goroutine writes its unit's outcome.
+func (e lifecycleExecutor) executeScript(
+	ctx context.Context,
+	manager *unit.Manager,
+	scriptsByAddress map[string]codersdk.WorkspaceAgentScript,
+	script codersdk.WorkspaceAgentScript,
+	option ExecuteOption,
+) (err error) {
+	id := unit.ID(script.ResourceAddress)
+
+	terminal := false
+	record := func(outcome unit.Outcome) error {
+		if err := recordOutcome(manager, id, outcome); err != nil {
+			return err
+		}
+		if outcome.Terminal() {
+			terminal = true
+		}
+		return nil
+	}
+	defer func() {
+		if !terminal {
+			err = errors.Join(err, recordOutcome(manager, id, unit.OutcomeFailed))
+		}
+	}()
+
+	evaluation, err := e.waitForDecision(ctx, manager, script)
+	if err == nil {
+		// Nothing starts after cancellation, even if the decision arrived
+		// in the same instant.
+		err = ctx.Err()
+	}
+	if err != nil {
+		if ctx.Err() != nil {
+			return errors.Join(err, record(unit.OutcomeCanceled))
+		}
+		return err
+	}
+
+	switch evaluation.Decision {
+	case unit.DecisionSkip:
+		e.logSkipped(ctx, script, scriptsByAddress, evaluation)
+		if err := record(unit.OutcomeSkipped); err != nil {
+			return err
+		}
+		// The prerequisite's own error already sets the lifecycle state.
+		return nil
+	case unit.DecisionRunnable:
+		if err := record(unit.OutcomeRunning); err != nil {
+			return err
+		}
+	default:
+		return xerrors.Errorf("script %q returned unexpected dependency decision %q", script.ResourceAddress, evaluation.Decision)
+	}
+
+	runErr := e.run(ctx, script, option)
+	if err := record(scriptOutcome(runErr, ctx.Err())); err != nil {
+		return errors.Join(runErr, err)
+	}
+	return runErr
+}
+
+func recordOutcome(manager *unit.Manager, id unit.ID, outcome unit.Outcome) error {
+	if err := manager.UpdateOutcome(id, outcome); err != nil {
+		return xerrors.Errorf("record script %q outcome %q: %w", id, outcome, err)
+	}
+	return nil
+}
+
+func (e lifecycleExecutor) waitForDecision(
+	ctx context.Context,
+	manager *unit.Manager,
+	script codersdk.WorkspaceAgentScript,
+) (unit.Evaluation, error) {
+	id := unit.ID(script.ResourceAddress)
+	evaluation, err := manager.Evaluate(id)
+	if err != nil || evaluation.Decision != unit.DecisionWaiting {
+		return evaluation, err
+	}
+
+	ticker := e.clock.NewTicker(dependencyLogInterval, "agent_scripts", "dependency_wait")
+	defer ticker.Stop("agent_scripts", "dependency_wait")
+
+	type result struct {
+		evaluation unit.Evaluation
+		err        error
+	}
+	// Buffered so the goroutine can deliver after this function has
+	// already returned on ctx.Done.
+	decision := make(chan result, 1)
+	go func() {
+		evaluation, err := manager.WaitForDecision(ctx, id)
+		decision <- result{evaluation: evaluation, err: err}
+	}()
+
+	var waited time.Duration
+	for {
+		select {
+		case <-ctx.Done():
+			return unit.Evaluation{}, ctx.Err()
+		case result := <-decision:
+			return result.evaluation, result.err
+		case <-ticker.C:
+			waited += dependencyLogInterval
+			// Re-evaluate so prerequisites that finished since the wait
+			// began drop out of the line.
+			evaluation, err := manager.Evaluate(id)
+			if err != nil {
+				return unit.Evaluation{}, err
+			}
+			if evaluation.Decision == unit.DecisionWaiting {
+				e.logWaiting(ctx, script, evaluation.UnmetDependencies, waited)
+			}
+		}
+	}
+}
+
+func (e lifecycleExecutor) logWaiting(
+	ctx context.Context,
+	script codersdk.WorkspaceAgentScript,
+	dependencies []unit.ConditionalDependency,
+	waited time.Duration,
+) {
+	// The manager already sorts the unmet list by prerequisite address.
+	quoted := make([]string, 0, len(dependencies))
+	for _, dependency := range dependencies {
+		quoted = append(quoted, strconv.Quote(string(dependency.DependsOn)))
+	}
+	e.sendScriptLog(ctx, script, fmt.Sprintf(
+		"Waiting for dependencies: %s (%ds)",
+		strings.Join(quoted, ", "), int(waited.Seconds()),
+	))
+}
+
+// logSkipped names every direct success prerequisite that can no longer
+// succeed at the moment of the decision.
+func (e lifecycleExecutor) logSkipped(
+	ctx context.Context,
+	script codersdk.WorkspaceAgentScript,
+	scriptsByAddress map[string]codersdk.WorkspaceAgentScript,
+	evaluation unit.Evaluation,
+) {
+	var causes []string
+	for _, dependency := range evaluation.UnmetDependencies {
+		if dependency.Requirement != unit.RequirementSuccess {
+			continue
+		}
+		how, impossible := skipCause(dependency.CurrentOutcome)
+		if !impossible {
+			continue
+		}
+		prerequisite := scriptsByAddress[string(dependency.DependsOn)]
+		name := prerequisite.DisplayName
+		if name == "" {
+			name = prerequisite.ResourceAddress
+		}
+		causes = append(causes, fmt.Sprintf("dependency %q %s", name, how))
+	}
+	if len(causes) == 0 {
+		// A skip decision always has at least one impossible edge, so this
+		// means the manager's rules and skipCause disagree.
+		e.logger.Warn(ctx, "skip decision with no impossible dependency", slog.F("resource_address", script.ResourceAddress))
+		return
+	}
+	e.sendScriptLog(ctx, script, "Skipping script: "+strings.Join(causes, ", ")+".")
+}
+
+// skipCause mirrors unit.Requirement.impossible, which is not exported.
+func skipCause(outcome unit.Outcome) (string, bool) {
+	switch outcome {
+	case unit.OutcomeFailed:
+		return "failed", true
+	case unit.OutcomeTimedOut:
+		return "timed out", true
+	case unit.OutcomeSkipped:
+		return "was skipped", true
+	default:
+		return "", false
+	}
+}
+
+// sendScriptLog flushes after sending so the line is visible while the
+// script is still waiting. A failed send never changes a decision.
+func (e lifecycleExecutor) sendScriptLog(ctx context.Context, script codersdk.WorkspaceAgentScript, output string) {
+	if e.getScriptLogger == nil {
+		return
+	}
+	logger := e.getScriptLogger(script.LogSourceID)
+	if err := logger.Send(ctx, agentsdk.Log{
+		CreatedAt: e.clock.Now("agent_scripts", "dependency_log").UTC(),
+		Output:    output,
+		Level:     codersdk.LogLevelInfo,
+	}); err != nil {
+		e.logger.Warn(ctx, "send script dependency log", slog.F("resource_address", script.ResourceAddress), slog.Error(err))
+		return
+	}
+	if err := logger.Flush(ctx); err != nil {
+		e.logger.Warn(ctx, "flush script dependency log", slog.F("resource_address", script.ResourceAddress), slog.Error(err))
+	}
+}
+
+// scriptOutcome keeps success and the script's own timeout even when the
+// context is canceled, because cancellation did not cause them.
+func scriptOutcome(runErr error, ctxErr error) unit.Outcome {
+	switch {
+	case runErr == nil:
+		return unit.OutcomeSucceeded
+	case errors.Is(runErr, ErrTimeout):
+		return unit.OutcomeTimedOut
+	case ctxErr != nil:
+		return unit.OutcomeCanceled
+	default:
+		return unit.OutcomeFailed
+	}
+}

agent/agentscripts/script_order_internal_test.go

@@ -0,0 +1,933 @@
+package agentscripts
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/agent/unit"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/codersdk/agentsdk"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+)
+
+// goleak runs from TestMain in agentscripts_test.go, which shares this
+// test binary.
+
+const (
+	requireSuccess    = codersdk.WorkspaceAgentScriptDependencyRequirementSuccess
+	requireCompletion = codersdk.WorkspaceAgentScriptDependencyRequirementCompletion
+)
+
+func TestLifecycleExecutorOrdersScripts(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.clone")
+	executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+
+	clone := scriptOrderTestScript("coder_script.clone")
+	install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+
+	done := runExecute(ctx, executor, clone, install)
+
+	runs.waitStarted(ctx, t, "coder_script.clone")
+	runs.requireNotStarted(t, "coder_script.install")
+
+	runs.release("coder_script.clone", nil)
+	runs.waitStarted(ctx, t, "coder_script.install")
+	require.NoError(t, testutil.RequireReceive(ctx, t, done))
+}
+
+func TestLifecycleExecutorWaitsForEveryPrerequisite(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.a")
+	runs.hold("coder_script.b")
+	executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+
+	a := scriptOrderTestScript("coder_script.a")
+	b := scriptOrderTestScript("coder_script.b")
+	c := scriptOrderTestScript("coder_script.c",
+		scriptOrderTestDependency("coder_script.a", requireSuccess),
+		scriptOrderTestDependency("coder_script.b", requireCompletion),
+	)
+
+	done := runExecute(ctx, executor, a, b, c)
+
+	runs.waitStarted(ctx, t, "coder_script.a")
+	runs.waitStarted(ctx, t, "coder_script.b")
+	runs.release("coder_script.a", nil)
+	runs.requireNotStarted(t, "coder_script.c")
+
+	runs.release("coder_script.b", nil)
+	runs.waitStarted(ctx, t, "coder_script.c")
+	require.NoError(t, testutil.RequireReceive(ctx, t, done))
+}
+
+// The executor hands run a context with no deadline of its own, so the
+// timeout context created inside run cannot have started during the wait.
+// TestExecuteOrderedTimeoutStartsAfterWaiting covers the same with a real
+// process.
+func TestLifecycleExecutorTimeoutStartsAfterWaiting(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	executeCtx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.clone")
+	executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+	deadlines := make(chan bool, 2)
+	executor.run = func(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+		_, hasDeadline := ctx.Deadline()
+		deadlines <- hasDeadline
+		return runs.run(ctx, script, option)
+	}
+
+	clone := scriptOrderTestScript("coder_script.clone")
+	install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+	install.Timeout = 10 * time.Millisecond
+
+	done := runExecute(executeCtx, executor, clone, install)
+	runs.waitStarted(ctx, t, "coder_script.clone")
+	require.False(t, testutil.RequireReceive(ctx, t, deadlines))
+
+	runs.release("coder_script.clone", nil)
+	runs.waitStarted(ctx, t, "coder_script.install")
+	require.False(t, testutil.RequireReceive(ctx, t, deadlines))
+	require.NoError(t, testutil.RequireReceive(ctx, t, done))
+}
+
+func TestScriptOutcome(t *testing.T) {
+	t.Parallel()
+
+	for _, test := range []struct {
+		name   string
+		runErr error
+		ctxErr error
+		want   unit.Outcome
+	}{
+		{name: "Succeeded", want: unit.OutcomeSucceeded},
+		{name: "TimedOut", runErr: ErrTimeout, want: unit.OutcomeTimedOut},
+		{name: "TimedOutDuringCancel", runErr: ErrTimeout, ctxErr: context.Canceled, want: unit.OutcomeTimedOut},
+		{name: "PipesLeftOpen", runErr: ErrOutputPipesOpen, want: unit.OutcomeFailed},
+		{name: "ExitCode", runErr: xerrors.New("exit status 1"), want: unit.OutcomeFailed},
+		{name: "Canceled", runErr: context.Canceled, ctxErr: context.Canceled, want: unit.OutcomeCanceled},
+		{name: "OtherErrorDuringCancel", runErr: xerrors.New("signal: killed"), ctxErr: context.Canceled, want: unit.OutcomeCanceled},
+		{name: "WrappedTimeout", runErr: xerrors.Errorf("run agent script: %w", ErrTimeout), want: unit.OutcomeTimedOut},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			require.Equal(t, test.want, scriptOutcome(test.runErr, test.ctxErr))
+		})
+	}
+}
+
+func TestLifecycleExecutorRequirements(t *testing.T) {
+	t.Parallel()
+
+	for _, test := range []struct {
+		name        string
+		requirement codersdk.WorkspaceAgentScriptDependencyRequirement
+		cloneErr    error
+		wantRun     bool
+		wantLogs    []string
+	}{
+		{name: "SuccessAfterFailure", requirement: requireSuccess, cloneErr: xerrors.New("exit status 128"), wantLogs: []string{`Skipping script: dependency "Clone" failed.`}},
+		{name: "SuccessAfterTimeout", requirement: requireSuccess, cloneErr: ErrTimeout, wantLogs: []string{`Skipping script: dependency "Clone" timed out.`}},
+		{name: "SuccessAfterSuccess", requirement: requireSuccess, wantRun: true},
+		{name: "CompletionAfterFailure", requirement: requireCompletion, cloneErr: xerrors.New("exit status 128"), wantRun: true},
+		{name: "CompletionAfterTimeout", requirement: requireCompletion, cloneErr: ErrTimeout, wantRun: true},
+		{name: "CompletionAfterSuccess", requirement: requireCompletion, wantRun: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			ctx := testutil.Context(t, testutil.WaitShort)
+
+			runs := newScriptOrderRuns()
+			runs.result("coder_script.clone", test.cloneErr)
+			logs := newScriptOrderTestLogs()
+			executor := newTestExecutor(t, runs, logs)
+
+			clone := scriptOrderTestScript("coder_script.clone")
+			clone.DisplayName = "Clone"
+			install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", test.requirement))
+
+			err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, install}, ExecuteStartScripts)
+			if test.cloneErr != nil {
+				require.ErrorIs(t, err, test.cloneErr)
+			} else {
+				require.NoError(t, err)
+			}
+			require.Equal(t, test.wantRun, runs.started("coder_script.install"))
+			require.Equal(t, test.wantLogs, logs.outputs(install.LogSourceID))
+		})
+	}
+}
+
+func TestLifecycleExecutorPropagatesSkippedOutcomes(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	runs := newScriptOrderRuns()
+	runs.result("coder_script.a", xerrors.New("exit status 1"))
+	logs := newScriptOrderTestLogs()
+	executor := newTestExecutor(t, runs, logs)
+
+	a := scriptOrderTestScript("coder_script.a")
+	b := scriptOrderTestScript("coder_script.b", scriptOrderTestDependency("coder_script.a", requireSuccess))
+	c := scriptOrderTestScript("coder_script.c", scriptOrderTestDependency("coder_script.b", requireCompletion))
+	d := scriptOrderTestScript("coder_script.d", scriptOrderTestDependency("coder_script.b", requireSuccess))
+
+	err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{a, b, c, d}, ExecuteStartScripts)
+	require.Error(t, err)
+
+	require.ElementsMatch(t, []string{"coder_script.a", "coder_script.c"}, runs.allStarted())
+	require.Equal(t, []string{`Skipping script: dependency "coder_script.a" failed.`}, logs.outputs(b.LogSourceID))
+	require.Empty(t, logs.outputs(c.LogSourceID))
+	require.Equal(t, []string{`Skipping script: dependency "coder_script.b" was skipped.`}, logs.outputs(d.LogSourceID))
+}
+
+func TestLifecycleExecutorMixedRequirements(t *testing.T) {
+	t.Parallel()
+
+	scripts := func() (clone, warm, build codersdk.WorkspaceAgentScript) {
+		clone = scriptOrderTestScript("coder_script.clone_repo")
+		clone.DisplayName = "Clone repo"
+		warm = scriptOrderTestScript("coder_script.warm_cache")
+		warm.DisplayName = "Warm cache"
+		build = scriptOrderTestScript("coder_script.build",
+			scriptOrderTestDependency("coder_script.clone_repo", requireSuccess),
+			scriptOrderTestDependency("coder_script.warm_cache", requireCompletion),
+		)
+		return clone, warm, build
+	}
+
+	t.Run("CompletionPrerequisiteFails", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		runs := newScriptOrderRuns()
+		runs.result("coder_script.warm_cache", xerrors.New("exit status 1"))
+		logs := newScriptOrderTestLogs()
+		executor := newTestExecutor(t, runs, logs)
+		clone, warm, build := scripts()
+
+		err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, warm, build}, ExecuteStartScripts)
+		require.Error(t, err)
+		require.True(t, runs.started("coder_script.build"))
+		require.Empty(t, logs.outputs(build.LogSourceID))
+	})
+
+	t.Run("SuccessPrerequisiteFails", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		runs := newScriptOrderRuns()
+		runs.result("coder_script.clone_repo", xerrors.New("exit status 128"))
+		logs := newScriptOrderTestLogs()
+		executor := newTestExecutor(t, runs, logs)
+		clone, warm, build := scripts()
+
+		err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, warm, build}, ExecuteStartScripts)
+		require.Error(t, err)
+		require.False(t, runs.started("coder_script.build"))
+		require.Equal(t, []string{`Skipping script: dependency "Clone repo" failed.`}, logs.outputs(build.LogSourceID))
+	})
+}
+
+func TestLifecycleExecutorSkipsOnFirstImpossibleEdge(t *testing.T) {
+	t.Parallel()
+
+	scripts := func() (clone, secrets, build codersdk.WorkspaceAgentScript) {
+		clone = scriptOrderTestScript("coder_script.clone_repo")
+		clone.DisplayName = "Clone repo"
+		secrets = scriptOrderTestScript("coder_script.fetch_secrets")
+		secrets.DisplayName = "Fetch secrets"
+		build = scriptOrderTestScript("coder_script.build",
+			scriptOrderTestDependency("coder_script.clone_repo", requireSuccess),
+			scriptOrderTestDependency("coder_script.fetch_secrets", requireSuccess),
+		)
+		return clone, secrets, build
+	}
+
+	t.Run("OnePrerequisiteStillRunning", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		runs := newScriptOrderRuns()
+		runs.hold("coder_script.clone_repo")
+		runs.result("coder_script.fetch_secrets", xerrors.New("exit status 1"))
+		logs := newScriptOrderTestLogs()
+		executor := newTestExecutor(t, runs, logs)
+		clone, secrets, build := scripts()
+
+		done := runExecute(ctx, executor, clone, secrets, build)
+		runs.waitStarted(ctx, t, "coder_script.clone_repo")
+
+		// The skip line arrives while clone_repo is still held.
+		require.Equal(t, `Skipping script: dependency "Fetch secrets" failed.`, logs.waitOutput(ctx, t, build.LogSourceID))
+
+		runs.release("coder_script.clone_repo", nil)
+		require.Error(t, testutil.RequireReceive(ctx, t, done))
+		require.False(t, runs.started("coder_script.build"))
+	})
+
+	t.Run("BothPrerequisitesFinishedBadly", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		runs := newScriptOrderRuns()
+		runs.result("coder_script.clone_repo", xerrors.New("exit status 128"))
+		runs.result("coder_script.fetch_secrets", ErrTimeout)
+		logs := newScriptOrderTestLogs()
+		executor := newTestExecutor(t, runs, logs)
+		clone, secrets, build := scripts()
+
+		// Both prerequisites finish before build is evaluated, so the
+		// manager lists both as impossible.
+		manager, byAddress, err := lifecycleManager([]codersdk.WorkspaceAgentScript{clone, secrets, build})
+		require.NoError(t, err)
+		require.Error(t, executor.executeWith(ctx, manager, byAddress, []codersdk.WorkspaceAgentScript{clone, secrets}, ExecuteStartScripts))
+		require.NoError(t, executor.executeWith(ctx, manager, byAddress, []codersdk.WorkspaceAgentScript{build}, ExecuteStartScripts))
+
+		require.False(t, runs.started("coder_script.build"))
+		require.Equal(t, []string{
+			`Skipping script: dependency "Clone repo" failed, dependency "Fetch secrets" timed out.`,
+		}, logs.outputs(build.LogSourceID))
+	})
+}
+
+func TestLifecycleExecutorWaitingLogsAreCombinedAndSorted(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	clock := quartz.NewMock(t)
+	tickerTrap := clock.Trap().NewTicker("agent_scripts", "dependency_wait")
+	defer tickerTrap.Close()
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.z_clone")
+	runs.hold("coder_script.a_auth")
+	logs := newScriptOrderTestLogs()
+	executor := newTestExecutor(t, runs, logs)
+	executor.clock = clock
+
+	zClone := scriptOrderTestScript("coder_script.z_clone")
+	aAuth := scriptOrderTestScript("coder_script.a_auth")
+	install := scriptOrderTestScript("coder_script.install",
+		scriptOrderTestDependency("coder_script.z_clone", requireSuccess),
+		scriptOrderTestDependency("coder_script.a_auth", requireCompletion),
+	)
+
+	scripts := []codersdk.WorkspaceAgentScript{zClone, aAuth, install}
+	manager, byAddress, err := lifecycleManager(scripts)
+	require.NoError(t, err)
+	done := make(chan error, 1)
+	go func() { done <- executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts) }()
+	tickerTrap.MustWait(ctx).MustRelease(ctx)
+
+	clock.Advance(dependencyLogInterval).MustWait(ctx)
+	require.Equal(t, `Waiting for dependencies: "coder_script.a_auth", "coder_script.z_clone" (30s)`, logs.waitOutput(ctx, t, install.LogSourceID))
+
+	clock.Advance(dependencyLogInterval).MustWait(ctx)
+	require.Equal(t, `Waiting for dependencies: "coder_script.a_auth", "coder_script.z_clone" (60s)`, logs.waitOutput(ctx, t, install.LogSourceID))
+
+	runs.release("coder_script.a_auth", nil)
+	// The outcome is recorded after run returns, so wait for the record
+	// itself rather than for run.
+	require.Eventually(t, func() bool {
+		u, err := manager.Unit("coder_script.a_auth")
+		return err == nil && u.Outcome() == unit.OutcomeSucceeded
+	}, testutil.WaitShort, testutil.IntervalFast)
+	clock.Advance(dependencyLogInterval).MustWait(ctx)
+	require.Equal(t, `Waiting for dependencies: "coder_script.z_clone" (90s)`, logs.waitOutput(ctx, t, install.LogSourceID))
+
+	runs.release("coder_script.z_clone", nil)
+	runs.waitStarted(ctx, t, "coder_script.install")
+	require.NoError(t, testutil.RequireReceive(ctx, t, done))
+	require.Len(t, logs.outputs(install.LogSourceID), 3)
+}
+
+func TestLifecycleExecutorNoWaitingLineWhenDecidedEarly(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	clock := quartz.NewMock(t)
+	tickerTrap := clock.Trap().NewTicker("agent_scripts", "dependency_wait")
+	defer tickerTrap.Close()
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.clone")
+	logs := newScriptOrderTestLogs()
+	executor := newTestExecutor(t, runs, logs)
+	executor.clock = clock
+
+	clone := scriptOrderTestScript("coder_script.clone")
+	install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+
+	done := runExecute(ctx, executor, clone, install)
+	// Only install waits, so exactly one ticker is created.
+	tickerTrap.MustWait(ctx).MustRelease(ctx)
+
+	runs.release("coder_script.clone", nil)
+	runs.waitStarted(ctx, t, "coder_script.install")
+	require.NoError(t, testutil.RequireReceive(ctx, t, done))
+
+	require.Empty(t, logs.outputs(clone.LogSourceID))
+	require.Empty(t, logs.outputs(install.LogSourceID))
+}
+
+func TestLifecycleExecutorCancellation(t *testing.T) {
+	t.Parallel()
+	ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
+	defer cancel()
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.a")
+	logs := newScriptOrderTestLogs()
+	executor := newTestExecutor(t, runs, logs)
+
+	a := scriptOrderTestScript("coder_script.a")
+	b := scriptOrderTestScript("coder_script.b", scriptOrderTestDependency("coder_script.a", requireSuccess))
+	c := scriptOrderTestScript("coder_script.c", scriptOrderTestDependency("coder_script.a", requireCompletion))
+	d := scriptOrderTestScript("coder_script.d")
+	scripts := []codersdk.WorkspaceAgentScript{a, b, c, d}
+
+	manager, byAddress, err := lifecycleManager(scripts)
+	require.NoError(t, err)
+
+	done := make(chan error, 1)
+	go func() { done <- executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts) }()
+
+	runs.waitStarted(ctx, t, "coder_script.a")
+	runs.waitFinished(ctx, t, "coder_script.d")
+
+	cancel()
+	waitCtx := testutil.Context(t, testutil.WaitShort)
+	require.ErrorIs(t, testutil.RequireReceive(waitCtx, t, done), context.Canceled)
+
+	require.ElementsMatch(t, []string{"coder_script.a", "coder_script.d"}, runs.allStarted())
+	require.Empty(t, logs.outputs(b.LogSourceID))
+	require.Empty(t, logs.outputs(c.LogSourceID))
+
+	for address, want := range map[string]unit.Outcome{
+		"coder_script.a": unit.OutcomeCanceled,
+		"coder_script.b": unit.OutcomeCanceled,
+		"coder_script.c": unit.OutcomeCanceled,
+		"coder_script.d": unit.OutcomeSucceeded,
+	} {
+		u, err := manager.Unit(unit.ID(address))
+		require.NoError(t, err)
+		require.Equal(t, want, u.Outcome(), address)
+	}
+}
+
+func TestLifecycleExecutorStopScripts(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	saveErr := xerrors.New("exit status 1")
+	runs := newScriptOrderRuns()
+	runs.result("coder_script.save_state", saveErr)
+	logs := newScriptOrderTestLogs()
+	executor := newTestExecutor(t, runs, logs)
+
+	save := scriptOrderTestScript("coder_script.save_state")
+	save.DisplayName = "Save state"
+	stopDB := scriptOrderTestScript("coder_script.stop_db", scriptOrderTestDependency("coder_script.save_state", requireSuccess))
+	report := scriptOrderTestScript("coder_script.report_stop", scriptOrderTestDependency("coder_script.save_state", requireCompletion))
+
+	err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{save, stopDB, report}, ExecuteStopScripts)
+	require.ErrorIs(t, err, saveErr)
+	require.ElementsMatch(t, []string{"coder_script.save_state", "coder_script.report_stop"}, runs.allStarted())
+	require.Equal(t, []string{`Skipping script: dependency "Save state" failed.`}, logs.outputs(stopDB.LogSourceID))
+	require.Equal(t, ExecuteStopScripts, runs.option("coder_script.report_stop"))
+}
+
+func TestLifecycleExecutorFreshManagerPerExecution(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	clone := scriptOrderTestScript("coder_script.clone")
+	install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+	scripts := []codersdk.WorkspaceAgentScript{clone, install}
+
+	first := newScriptOrderRuns()
+	first.result("coder_script.clone", xerrors.New("exit status 128"))
+	executor := newTestExecutor(t, first, newScriptOrderTestLogs())
+	require.Error(t, executor.execute(ctx, scripts, ExecuteStartScripts))
+	require.False(t, first.started("coder_script.install"))
+
+	second := newScriptOrderRuns()
+	executor.run = second.run
+	require.NoError(t, executor.execute(ctx, scripts, ExecuteStartScripts))
+	require.True(t, second.started("coder_script.install"))
+}
+
+func TestLifecycleExecutorRecordsFailureOnUnrecordedExit(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	runs := newScriptOrderRuns()
+	logs := newScriptOrderTestLogs()
+	executor := newTestExecutor(t, runs, logs)
+
+	clone := scriptOrderTestScript("coder_script.clone")
+	install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+	scripts := []codersdk.WorkspaceAgentScript{clone, install}
+	manager, byAddress, err := lifecycleManager(scripts)
+	require.NoError(t, err)
+	// Pre-setting the outcome makes the executor's own update fail.
+	require.NoError(t, manager.UpdateOutcome("coder_script.clone", unit.OutcomeRunning))
+
+	err = executor.executeWith(ctx, manager, byAddress, scripts, ExecuteStartScripts)
+	require.ErrorIs(t, err, unit.ErrSameOutcomeAlreadySet)
+	require.NoError(t, ctx.Err())
+
+	require.Empty(t, runs.allStarted())
+	u, err := manager.Unit("coder_script.clone")
+	require.NoError(t, err)
+	require.Equal(t, unit.OutcomeFailed, u.Outcome())
+	require.Equal(t, []string{`Skipping script: dependency "coder_script.clone" failed.`}, logs.outputs(install.LogSourceID))
+}
+
+func TestLifecycleExecutorAddresslessScriptsRunAtOnce(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	runs := newScriptOrderRuns()
+	runs.hold("coder_script.clone")
+	executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+
+	clone := scriptOrderTestScript("coder_script.clone")
+	install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+	legacy := scriptOrderTestScript("")
+	devcontainer := scriptOrderTestScript("")
+
+	done := runExecute(ctx, executor, clone, install, legacy, devcontainer)
+
+	runs.waitStarted(ctx, t, "coder_script.clone")
+	runs.waitFinished(ctx, t, legacy.LogSourceID.String())
+	runs.waitFinished(ctx, t, devcontainer.LogSourceID.String())
+	runs.requireNotStarted(t, "coder_script.install")
+
+	runs.release("coder_script.clone", nil)
+	require.NoError(t, testutil.RequireReceive(ctx, t, done))
+}
+
+func TestLifecycleExecutorSkipIsNotAnError(t *testing.T) {
+	t.Parallel()
+
+	for _, test := range []struct {
+		name     string
+		cloneErr error
+	}{
+		{name: "Failure", cloneErr: xerrors.New("exit status 128")},
+		{name: "Timeout", cloneErr: ErrTimeout},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			ctx := testutil.Context(t, testutil.WaitShort)
+
+			runs := newScriptOrderRuns()
+			runs.result("coder_script.clone", test.cloneErr)
+			executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+
+			clone := scriptOrderTestScript("coder_script.clone")
+			install := scriptOrderTestScript("coder_script.install", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+			post := scriptOrderTestScript("coder_script.post", scriptOrderTestDependency("coder_script.clone", requireCompletion))
+
+			err := executor.execute(ctx, []codersdk.WorkspaceAgentScript{clone, install, post}, ExecuteStartScripts)
+			require.ErrorIs(t, err, test.cloneErr)
+			// errgroup keeps the first error only, so a skip that returned
+			// an error could only show up as the whole result.
+			require.Equal(t, test.cloneErr.Error(), err.Error())
+			require.ElementsMatch(t, []string{"coder_script.clone", "coder_script.post"}, runs.allStarted())
+		})
+	}
+}
+
+func TestLifecycleExecutorRejectsInvalidDependenciesBeforeRunning(t *testing.T) {
+	t.Parallel()
+
+	noAddress := scriptOrderTestScript("", scriptOrderTestDependency("coder_script.clone", requireSuccess))
+	noAddress.DisplayName = "Install tools"
+
+	for _, test := range []struct {
+		name    string
+		scripts []codersdk.WorkspaceAgentScript
+		wantErr string
+	}{
+		{
+			name: "DependenciesWithoutAddress",
+			scripts: []codersdk.WorkspaceAgentScript{
+				scriptOrderTestScript("coder_script.clone"),
+				noAddress,
+			},
+			wantErr: `script "Install tools" has dependencies but no Terraform resource address`,
+		},
+		{
+			name: "DuplicateAddress",
+			scripts: []codersdk.WorkspaceAgentScript{
+				scriptOrderTestScript("coder_script.setup"),
+				scriptOrderTestScript("coder_script.setup", scriptOrderTestDependency("coder_script.setup", requireSuccess)),
+			},
+			wantErr: `duplicate script resource address "coder_script.setup"`,
+		},
+		{
+			name: "MissingPrerequisite",
+			scripts: []codersdk.WorkspaceAgentScript{
+				scriptOrderTestScript("coder_script.install_tools", scriptOrderTestDependency("coder_script.clone", requireSuccess)),
+			},
+			wantErr: `script "coder_script.install_tools" depends on "coder_script.clone", which is not part of this lifecycle execution`,
+		},
+		{
+			name: "UnsupportedRequirement",
+			scripts: []codersdk.WorkspaceAgentScript{
+				scriptOrderTestScript("coder_script.clone_repo"),
+				scriptOrderTestScript("coder_script.install_tools", scriptOrderTestDependency("coder_script.clone_repo", "started")),
+			},
+			wantErr: `script "coder_script.install_tools" dependency on "coder_script.clone_repo": unsupported requirement "started"`,
+		},
+		{
+			name: "Cycle",
+			scripts: []codersdk.WorkspaceAgentScript{
+				scriptOrderTestScript("coder_script.a", scriptOrderTestDependency("coder_script.b", requireSuccess)),
+				scriptOrderTestScript("coder_script.b", scriptOrderTestDependency("coder_script.a", requireSuccess)),
+			},
+			wantErr: `add script dependency "coder_script.b" after "coder_script.a"`,
+		},
+		{
+			name: "ConflictingRequirement",
+			scripts: []codersdk.WorkspaceAgentScript{
+				scriptOrderTestScript("coder_script.clone_repo"),
+				scriptOrderTestScript("coder_script.build",
+					scriptOrderTestDependency("coder_script.clone_repo", requireSuccess),
+					scriptOrderTestDependency("coder_script.clone_repo", requireCompletion),
+				),
+			},
+			wantErr: `add script dependency "coder_script.build" after "coder_script.clone_repo"`,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			ctx := testutil.Context(t, testutil.WaitShort)
+
+			runs := newScriptOrderRuns()
+			logs := newScriptOrderTestLogs()
+			executor := newTestExecutor(t, runs, logs)
+
+			err := executor.execute(ctx, test.scripts, ExecuteStartScripts)
+			require.ErrorContains(t, err, test.wantErr)
+			switch test.name {
+			case "Cycle":
+				require.ErrorIs(t, err, unit.ErrCycleDetected)
+			case "ConflictingRequirement":
+				require.ErrorIs(t, err, unit.ErrConflictingRequirement)
+			}
+			require.Empty(t, runs.allStarted())
+			for _, script := range test.scripts {
+				require.Equal(t, []string{"Lifecycle scripts did not run: " + err.Error()}, logs.outputs(script.LogSourceID))
+			}
+		})
+	}
+}
+
+func TestLifecycleExecutorManyWaiters(t *testing.T) {
+	t.Parallel()
+
+	const waiters = 50
+	scripts := func() []codersdk.WorkspaceAgentScript {
+		all := []codersdk.WorkspaceAgentScript{scriptOrderTestScript("coder_script.clone")}
+		for i := range waiters {
+			all = append(all, scriptOrderTestScript(
+				fmt.Sprintf("coder_script.dependent_%02d", i),
+				scriptOrderTestDependency("coder_script.clone", requireSuccess),
+			))
+		}
+		return all
+	}
+
+	t.Run("AllRun", func(t *testing.T) {
+		t.Parallel()
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		runs := newScriptOrderRuns()
+		runs.hold("coder_script.clone")
+		executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+
+		done := runExecute(ctx, executor, scripts()...)
+		runs.waitStarted(ctx, t, "coder_script.clone")
+		require.Len(t, runs.allStarted(), 1)
+
+		runs.release("coder_script.clone", nil)
+		require.NoError(t, testutil.RequireReceive(ctx, t, done))
+		require.Len(t, runs.allStarted(), waiters+1)
+	})
+
+	t.Run("AllCanceled", func(t *testing.T) {
+		t.Parallel()
+		ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
+		defer cancel()
+
+		runs := newScriptOrderRuns()
+		runs.hold("coder_script.clone")
+		executor := newTestExecutor(t, runs, newScriptOrderTestLogs())
+		all := scripts()
+
+		manager, byAddress, err := lifecycleManager(all)
+		require.NoError(t, err)
+		done := make(chan error, 1)
+		go func() { done <- executor.executeWith(ctx, manager, byAddress, all, ExecuteStartScripts) }()
+		runs.waitStarted(ctx, t, "coder_script.clone")
+
+		cancel()
+		waitCtx := testutil.Context(t, testutil.WaitShort)
+		require.ErrorIs(t, testutil.RequireReceive(waitCtx, t, done), context.Canceled)
+		require.Len(t, runs.allStarted(), 1)
+		for _, script := range all {
+			u, err := manager.Unit(unit.ID(script.ResourceAddress))
+			require.NoError(t, err)
+			require.Equal(t, unit.OutcomeCanceled, u.Outcome(), script.ResourceAddress)
+		}
+	})
+}
+
+func newTestExecutor(t *testing.T, runs *scriptOrderRuns, logs *scriptOrderTestLogs) lifecycleExecutor {
+	t.Helper()
+	return lifecycleExecutor{
+		clock:           quartz.NewReal(),
+		logger:          testutil.Logger(t),
+		getScriptLogger: logs.logger,
+		run:             runs.run,
+	}
+}
+
+func runExecute(ctx context.Context, executor lifecycleExecutor, scripts ...codersdk.WorkspaceAgentScript) <-chan error {
+	done := make(chan error, 1)
+	go func() { done <- executor.execute(ctx, scripts, ExecuteStartScripts) }()
+	return done
+}
+
+func scriptOrderTestScript(address string, dependencies ...codersdk.WorkspaceAgentScriptDependency) codersdk.WorkspaceAgentScript {
+	return codersdk.WorkspaceAgentScript{
+		ID:              uuid.New(),
+		LogSourceID:     uuid.New(),
+		ResourceAddress: address,
+		RunOnStart:      true,
+		Dependencies:    dependencies,
+	}
+}
+
+func scriptOrderTestDependency(address string, requirement codersdk.WorkspaceAgentScriptDependencyRequirement) codersdk.WorkspaceAgentScriptDependency {
+	return codersdk.WorkspaceAgentScriptDependency{
+		PrerequisiteResourceAddress: address,
+		Requirement:                 requirement,
+	}
+}
+
+// scriptOrderRuns is a fake run. A script's key is its resource address,
+// or its log source ID when it has no address. A held script blocks until
+// released; any other script returns its configured result at once.
+type scriptOrderRuns struct {
+	mu         sync.Mutex
+	held       map[string]chan error
+	results    map[string]error
+	startedCh  map[string]chan struct{}
+	finishedCh map[string]chan struct{}
+	options    map[string]ExecuteOption
+	order      []string
+}
+
+func newScriptOrderRuns() *scriptOrderRuns {
+	return &scriptOrderRuns{
+		held:       make(map[string]chan error),
+		results:    make(map[string]error),
+		startedCh:  make(map[string]chan struct{}),
+		finishedCh: make(map[string]chan struct{}),
+		options:    make(map[string]ExecuteOption),
+	}
+}
+
+func scriptOrderKey(script codersdk.WorkspaceAgentScript) string {
+	if script.ResourceAddress != "" {
+		return script.ResourceAddress
+	}
+	return script.LogSourceID.String()
+}
+
+func (r *scriptOrderRuns) hold(key string) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.held[key] = make(chan error, 1)
+}
+
+func (r *scriptOrderRuns) result(key string, err error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.results[key] = err
+}
+
+func (r *scriptOrderRuns) release(key string, err error) {
+	r.mu.Lock()
+	release := r.held[key]
+	r.mu.Unlock()
+	release <- err
+}
+
+func signal(m map[string]chan struct{}, key string) chan struct{} {
+	if _, ok := m[key]; !ok {
+		m[key] = make(chan struct{})
+	}
+	return m[key]
+}
+
+func (r *scriptOrderRuns) run(ctx context.Context, script codersdk.WorkspaceAgentScript, option ExecuteOption) error {
+	key := scriptOrderKey(script)
+	r.mu.Lock()
+	r.order = append(r.order, key)
+	r.options[key] = option
+	close(signal(r.startedCh, key))
+	release, held := r.held[key]
+	result := r.results[key]
+	finished := signal(r.finishedCh, key)
+	r.mu.Unlock()
+	defer close(finished)
+
+	if !held {
+		return result
+	}
+	select {
+	case err := <-release:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func (r *scriptOrderRuns) waitStarted(ctx context.Context, t *testing.T, key string) {
+	t.Helper()
+	r.mu.Lock()
+	started := signal(r.startedCh, key)
+	r.mu.Unlock()
+	select {
+	case <-started:
+	case <-ctx.Done():
+		t.Fatalf("script %q did not start", key)
+	}
+}
+
+func (r *scriptOrderRuns) waitFinished(ctx context.Context, t *testing.T, key string) {
+	t.Helper()
+	r.mu.Lock()
+	finished := signal(r.finishedCh, key)
+	r.mu.Unlock()
+	select {
+	case <-finished:
+	case <-ctx.Done():
+		t.Fatalf("script %q did not finish", key)
+	}
+}
+
+func (r *scriptOrderRuns) requireNotStarted(t *testing.T, key string) {
+	t.Helper()
+	require.False(t, r.started(key), "script %q started too early", key)
+}
+
+func (r *scriptOrderRuns) started(key string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	started, ok := r.startedCh[key]
+	if !ok {
+		return false
+	}
+	select {
+	case <-started:
+		return true
+	default:
+		return false
+	}
+}
+
+func (r *scriptOrderRuns) allStarted() []string {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return append([]string(nil), r.order...)
+}
+
+func (r *scriptOrderRuns) option(key string) ExecuteOption {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.options[key]
+}
+
+// scriptOrderTestLogs records every line sent to each script's log source.
+type scriptOrderTestLogs struct {
+	mu   sync.Mutex
+	byID map[uuid.UUID][]string
+	sent chan scriptOrderLog
+}
+
+type scriptOrderLog struct {
+	logSourceID uuid.UUID
+	output      string
+}
+
+func newScriptOrderTestLogs() *scriptOrderTestLogs {
+	return &scriptOrderTestLogs{
+		byID: make(map[uuid.UUID][]string),
+		sent: make(chan scriptOrderLog, 256),
+	}
+}
+
+func (l *scriptOrderTestLogs) logger(logSourceID uuid.UUID) ScriptLogger {
+	return scriptOrderTestLogger{logs: l, logSourceID: logSourceID}
+}
+
+func (l *scriptOrderTestLogs) outputs(logSourceID uuid.UUID) []string {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	return append([]string(nil), l.byID[logSourceID]...)
+}
+
+// waitOutput blocks until the next line for the given log source arrives.
+func (l *scriptOrderTestLogs) waitOutput(ctx context.Context, t *testing.T, logSourceID uuid.UUID) string {
+	t.Helper()
+	for {
+		log := testutil.RequireReceive(ctx, t, l.sent)
+		if log.logSourceID == logSourceID {
+			return log.output
+		}
+	}
+}
+
+type scriptOrderTestLogger struct {
+	logs        *scriptOrderTestLogs
+	logSourceID uuid.UUID
+}
+
+func (l scriptOrderTestLogger) Send(_ context.Context, logs ...agentsdk.Log) error {
+	l.logs.mu.Lock()
+	defer l.logs.mu.Unlock()
+	for _, log := range logs {
+		l.logs.byID[l.logSourceID] = append(l.logs.byID[l.logSourceID], log.Output)
+		l.logs.sent <- scriptOrderLog{logSourceID: l.logSourceID, output: log.Output}
+	}
+	return nil
+}
+
+func (scriptOrderTestLogger) Flush(context.Context) error {
+	return nil
+}

coderd/agentapi/manifest.go

@@ -247,6 +247,7 @@ func dbAgentScriptToProto(script database.GetWorkspaceAgentScriptsByAgentIDsRow)
 		RunOnStop:        script.RunOnStop,
 		StartBlocksLogin: script.StartBlocksLogin,
 		Timeout:          durationpb.New(time.Duration(script.TimeoutSeconds) * time.Second),
+		DisplayName:      script.DisplayName,
 		ResourceAddress:  sdkScript.ResourceAddress,
 		Dependencies:     dependencies,
 	}, nil

coderd/agentapi/manifest_test.go

@@ -126,6 +126,7 @@ func TestGetManifest(t *testing.T) {
 				WorkspaceAgentID: agent.ID,
 				LogSourceID:      uuid.New(),
 				LogPath:          "/cool/log/path/1",
+				DisplayName:      "Install tools",
 				Script:           "cool script 1",
 				Cron:             "30 2 * * *",
 				StartBlocksLogin: true,
@@ -140,6 +141,7 @@ func TestGetManifest(t *testing.T) {
 				WorkspaceAgentID: agent.ID,
 				LogSourceID:      uuid.New(),
 				LogPath:          "/cool/log/path/2",
+				DisplayName:      "Clone repo",
 				Script:           "cool script 2",
 				Cron:             "",
 				StartBlocksLogin: false,
@@ -212,6 +214,7 @@ func TestGetManifest(t *testing.T) {
 				RunOnStop:        scripts[0].RunOnStop,
 				StartBlocksLogin: scripts[0].StartBlocksLogin,
 				Timeout:          durationpb.New(time.Duration(scripts[0].TimeoutSeconds) * time.Second),
+				DisplayName:      scripts[0].DisplayName,
 				ResourceAddress:  "coder_script.install_tools",
 				Dependencies: []*agentproto.WorkspaceAgentScriptDependency{{
 					PrerequisiteResourceAddress: "coder_script.clone_repo",
@@ -228,6 +231,7 @@ func TestGetManifest(t *testing.T) {
 				RunOnStop:        scripts[1].RunOnStop,
 				StartBlocksLogin: scripts[1].StartBlocksLogin,
 				Timeout:          durationpb.New(time.Duration(scripts[1].TimeoutSeconds) * time.Second),
+				DisplayName:      scripts[1].DisplayName,
 				ResourceAddress:  "coder_script.clone_repo",
 				// An unordered script sends no entries, not placeholders.
 				Dependencies: nil,

tailnet/proto/version.go

@@ -101,9 +101,7 @@ import (
 //     WorkspaceAgentScriptDependency message. They carry each script's
 //     Terraform address and its resolved coder_script_order prerequisites
 //     in the manifest. Both are empty for unordered scripts.
-//   - A v2.14 agent decodes these fields but does not yet enforce the
-//     ordering, so a version check alone does not tell coderd that an agent
-//     orders its scripts.
+//   - A v2.14 agent runs start and stop scripts in dependency order.
 const (
 	CurrentMajor = 2
 	CurrentMinor = 14