feat: add workspace restart functionality to API (#25757)

This models restart as durable orchestration of existing stop and
start workspace builds instead of adding a new restart transition.
Keeping restart as two existing transitions preserves the current
build/provisioner model.

The child start build is created only after the parent stop build
succeeds, rather than being inserted immediately in a pending
state. That keeps `workspace_builds` aligned with actual
provisioner-ready work and avoids introducing a second
pending-build lifecycle that the provisioner and build acquisition
paths would need to understand.

Refs: https://linear.app/codercom/issue/PLAT-143

coderd/apidoc/docs.go

@@ -15721,6 +15721,11 @@ const docTemplate = `{
                 "workspace_agent_resource_monitor:create",
                 "workspace_agent_resource_monitor:read",
                 "workspace_agent_resource_monitor:update",
+                "workspace_build_orchestration:*",
+                "workspace_build_orchestration:create",
+                "workspace_build_orchestration:delete",
+                "workspace_build_orchestration:read",
+                "workspace_build_orchestration:update",
                 "workspace_dormant:*",
                 "workspace_dormant:application_connect",
                 "workspace_dormant:create",
@@ -15956,6 +15961,11 @@ const docTemplate = `{
                 "APIKeyScopeWorkspaceAgentResourceMonitorCreate",
                 "APIKeyScopeWorkspaceAgentResourceMonitorRead",
                 "APIKeyScopeWorkspaceAgentResourceMonitorUpdate",
+                "APIKeyScopeWorkspaceBuildOrchestrationAll",
+                "APIKeyScopeWorkspaceBuildOrchestrationCreate",
+                "APIKeyScopeWorkspaceBuildOrchestrationDelete",
+                "APIKeyScopeWorkspaceBuildOrchestrationRead",
+                "APIKeyScopeWorkspaceBuildOrchestrationUpdate",
                 "APIKeyScopeWorkspaceDormantAll",
                 "APIKeyScopeWorkspaceDormantApplicationConnect",
                 "APIKeyScopeWorkspaceDormantCreate",
@@ -18759,6 +18769,42 @@ const docTemplate = `{
                 }
             }
         },
+        "codersdk.CreateWorkspaceBuildOnSuccessRequest": {
+            "type": "object",
+            "required": [
+                "transition"
+            ],
+            "properties": {
+                "rich_parameter_values": {
+                    "description": "RichParameterValues are applied to the child build. Parameters\nnot listed here fall back to their values from the previous\nbuild, matching normal build behavior.",
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/codersdk.WorkspaceBuildParameter"
+                    }
+                },
+                "template_version_id": {
+                    "description": "TemplateVersionID pins the child build to a specific template\nversion. Pinning requires permission to update the template,\nsince the active version may change before the child build\nruns. When empty, the child build uses the template's active\nversion at the time it runs.",
+                    "type": "string",
+                    "format": "uuid"
+                },
+                "template_version_preset_id": {
+                    "description": "TemplateVersionPresetID selects a preset for the child build.\nIt requires TemplateVersionID to also be set.",
+                    "type": "string",
+                    "format": "uuid"
+                },
+                "transition": {
+                    "description": "Transition must be \"start\". The parent build's transition must\nbe \"stop\".",
+                    "enum": [
+                        "start"
+                    ],
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/codersdk.WorkspaceTransition"
+                        }
+                    ]
+                }
+            }
+        },
         "codersdk.CreateWorkspaceBuildReason": {
             "type": "string",
             "enum": [
@@ -18800,6 +18846,14 @@ const docTemplate = `{
                         }
                     ]
                 },
+                "on_success": {
+                    "description": "OnSuccess queues a follow-up workspace build after this build succeeds.\nIt currently supports restarting a workspace by starting it after a\nsuccessful stop build.",
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/codersdk.CreateWorkspaceBuildOnSuccessRequest"
+                        }
+                    ]
+                },
                 "orphan": {
                     "description": "Orphan may be set for the Destroy transition.",
                     "type": "boolean"
@@ -22759,6 +22813,7 @@ const docTemplate = `{
                 "workspace",
                 "workspace_agent_devcontainers",
                 "workspace_agent_resource_monitor",
+                "workspace_build_orchestration",
                 "workspace_dormant",
                 "workspace_proxy"
             ],
@@ -22811,6 +22866,7 @@ const docTemplate = `{
                 "ResourceWorkspace",
                 "ResourceWorkspaceAgentDevcontainers",
                 "ResourceWorkspaceAgentResourceMonitor",
+                "ResourceWorkspaceBuildOrchestration",
                 "ResourceWorkspaceDormant",
                 "ResourceWorkspaceProxy"
             ]

coderd/apidoc/swagger.json

@@ -14059,6 +14059,11 @@
 				"workspace_agent_resource_monitor:create",
 				"workspace_agent_resource_monitor:read",
 				"workspace_agent_resource_monitor:update",
+				"workspace_build_orchestration:*",
+				"workspace_build_orchestration:create",
+				"workspace_build_orchestration:delete",
+				"workspace_build_orchestration:read",
+				"workspace_build_orchestration:update",
 				"workspace_dormant:*",
 				"workspace_dormant:application_connect",
 				"workspace_dormant:create",
@@ -14294,6 +14299,11 @@
 				"APIKeyScopeWorkspaceAgentResourceMonitorCreate",
 				"APIKeyScopeWorkspaceAgentResourceMonitorRead",
 				"APIKeyScopeWorkspaceAgentResourceMonitorUpdate",
+				"APIKeyScopeWorkspaceBuildOrchestrationAll",
+				"APIKeyScopeWorkspaceBuildOrchestrationCreate",
+				"APIKeyScopeWorkspaceBuildOrchestrationDelete",
+				"APIKeyScopeWorkspaceBuildOrchestrationRead",
+				"APIKeyScopeWorkspaceBuildOrchestrationUpdate",
 				"APIKeyScopeWorkspaceDormantAll",
 				"APIKeyScopeWorkspaceDormantApplicationConnect",
 				"APIKeyScopeWorkspaceDormantCreate",
@@ -16971,6 +16981,38 @@
 				}
 			}
 		},
+		"codersdk.CreateWorkspaceBuildOnSuccessRequest": {
+			"type": "object",
+			"required": ["transition"],
+			"properties": {
+				"rich_parameter_values": {
+					"description": "RichParameterValues are applied to the child build. Parameters\nnot listed here fall back to their values from the previous\nbuild, matching normal build behavior.",
+					"type": "array",
+					"items": {
+						"$ref": "#/definitions/codersdk.WorkspaceBuildParameter"
+					}
+				},
+				"template_version_id": {
+					"description": "TemplateVersionID pins the child build to a specific template\nversion. Pinning requires permission to update the template,\nsince the active version may change before the child build\nruns. When empty, the child build uses the template's active\nversion at the time it runs.",
+					"type": "string",
+					"format": "uuid"
+				},
+				"template_version_preset_id": {
+					"description": "TemplateVersionPresetID selects a preset for the child build.\nIt requires TemplateVersionID to also be set.",
+					"type": "string",
+					"format": "uuid"
+				},
+				"transition": {
+					"description": "Transition must be \"start\". The parent build's transition must\nbe \"stop\".",
+					"enum": ["start"],
+					"allOf": [
+						{
+							"$ref": "#/definitions/codersdk.WorkspaceTransition"
+						}
+					]
+				}
+			}
+		},
 		"codersdk.CreateWorkspaceBuildReason": {
 			"type": "string",
 			"enum": [
@@ -17008,6 +17050,14 @@
 						}
 					]
 				},
+				"on_success": {
+					"description": "OnSuccess queues a follow-up workspace build after this build succeeds.\nIt currently supports restarting a workspace by starting it after a\nsuccessful stop build.",
+					"allOf": [
+						{
+							"$ref": "#/definitions/codersdk.CreateWorkspaceBuildOnSuccessRequest"
+						}
+					]
+				},
 				"orphan": {
 					"description": "Orphan may be set for the Destroy transition.",
 					"type": "boolean"
@@ -20831,6 +20881,7 @@
 				"workspace",
 				"workspace_agent_devcontainers",
 				"workspace_agent_resource_monitor",
+				"workspace_build_orchestration",
 				"workspace_dormant",
 				"workspace_proxy"
 			],
@@ -20883,6 +20934,7 @@
 				"ResourceWorkspace",
 				"ResourceWorkspaceAgentDevcontainers",
 				"ResourceWorkspaceAgentResourceMonitor",
+				"ResourceWorkspaceBuildOrchestration",
 				"ResourceWorkspaceDormant",
 				"ResourceWorkspaceProxy"
 			]

coderd/coderd.go

@@ -96,6 +96,7 @@ import (
 	"github.com/coder/coder/v2/coderd/workspaceconnwatcher"
 	"github.com/coder/coder/v2/coderd/workspacestats"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
+	"github.com/coder/coder/v2/coderd/wsbuildorchestrator"
 	"github.com/coder/coder/v2/coderd/x/chatd"
 	"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
 	"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
@@ -968,6 +969,19 @@ func New(options *Options) *API {
 
 	api.workspaceAgentConnWatcher = workspaceconnwatcher.New(api.ctx, options.Logger, options.Pubsub, options.Database)
 
+	api.workspaceBuildOrchestrator = wsbuildorchestrator.New(wsbuildorchestrator.Options{
+		Logger:            options.Logger,
+		Database:          options.Database,
+		Pubsub:            options.Pubsub,
+		FileCache:         api.FileCache,
+		BuildUsageChecker: api.BuildUsageChecker,
+		DeploymentValues:  options.DeploymentValues,
+		Experiments:       api.Experiments,
+		BuilderMetrics:    options.WorkspaceBuilderMetrics,
+		Clock:             quartz.NewReal(),
+	})
+	api.workspaceBuildOrchestrator.Start(api.ctx)
+
 	apiKeyMiddleware := httpmw.ExtractAPIKeyMW(httpmw.ExtractAPIKeyConfig{
 		DB:                            options.Database,
 		ActivateDormantUser:           ActivateDormantUser(options.Logger, &api.Auditor, options.Database),
@@ -2316,7 +2330,8 @@ type API struct {
 	// profiler is process-global, so concurrent collections would fail.
 	ProfileCollecting atomic.Bool
 
-	workspaceAgentConnWatcher *workspaceconnwatcher.Watcher
+	workspaceAgentConnWatcher  *workspaceconnwatcher.Watcher
+	workspaceBuildOrchestrator *wsbuildorchestrator.Orchestrator
 }
 
 // chatDaemonPublishDiffStatusChangeFunc returns chatDaemon's
@@ -2400,6 +2415,7 @@ func (api *API) Close() error {
 	_ = api.AppEncryptionKeyCache.Close()
 	_ = api.UpdatesProvider.Close()
 	api.workspaceAgentConnWatcher.Close()
+	api.workspaceBuildOrchestrator.Close()
 
 	if current := api.PrebuildsReconciler.Load(); current != nil {
 		ctx, giveUp := context.WithTimeoutCause(context.Background(), time.Second*30, xerrors.New("gave up waiting for reconciler to stop before shutdown"))

coderd/database/check_constraint.go

@@ -6,58 +6,65 @@ type CheckConstraint string
 
 // CheckConstraint enums.
 const (
-	CheckAIGatewayKeysHashedSecretCheck                CheckConstraint = "ai_gateway_keys_hashed_secret_check"                  // ai_gateway_keys
-	CheckAIGatewayKeysNameCheck                        CheckConstraint = "ai_gateway_keys_name_check"                           // ai_gateway_keys
-	CheckAIGatewayKeysSecretPrefixCheck                CheckConstraint = "ai_gateway_keys_secret_prefix_check"                  // ai_gateway_keys
-	CheckAIModelPricesCacheReadPriceCheck              CheckConstraint = "ai_model_prices_cache_read_price_check"               // ai_model_prices
-	CheckAIModelPricesCacheWritePriceCheck             CheckConstraint = "ai_model_prices_cache_write_price_check"              // ai_model_prices
-	CheckAIModelPricesInputPriceCheck                  CheckConstraint = "ai_model_prices_input_price_check"                    // ai_model_prices
-	CheckAIModelPricesOutputPriceCheck                 CheckConstraint = "ai_model_prices_output_price_check"                   // ai_model_prices
-	CheckAIProvidersNameCheck                          CheckConstraint = "ai_providers_name_check"                              // ai_providers
-	CheckAIUserDailySpendSpendMicrosCheck              CheckConstraint = "ai_user_daily_spend_spend_micros_check"               // ai_user_daily_spend
-	CheckAibridgeTokenUsagesCacheReadPriceMicrosCheck  CheckConstraint = "aibridge_token_usages_cache_read_price_micros_check"  // aibridge_token_usages
-	CheckAibridgeTokenUsagesCacheWritePriceMicrosCheck CheckConstraint = "aibridge_token_usages_cache_write_price_micros_check" // aibridge_token_usages
-	CheckAibridgeTokenUsagesCostMicrosCheck            CheckConstraint = "aibridge_token_usages_cost_micros_check"              // aibridge_token_usages
-	CheckAibridgeTokenUsagesInputPriceMicrosCheck      CheckConstraint = "aibridge_token_usages_input_price_micros_check"       // aibridge_token_usages
-	CheckAibridgeTokenUsagesOutputPriceMicrosCheck     CheckConstraint = "aibridge_token_usages_output_price_micros_check"      // aibridge_token_usages
-	CheckAPIKeysAllowListNotEmpty                      CheckConstraint = "api_keys_allow_list_not_empty"                        // api_keys
-	CheckBoundaryLogsSequenceNumberCheck               CheckConstraint = "boundary_logs_sequence_number_check"                  // boundary_logs
-	CheckChatModelConfigsAIProviderRequiredWhenActive  CheckConstraint = "chat_model_configs_ai_provider_required_when_active"  // chat_model_configs
-	CheckChatModelConfigsCompressionThresholdCheck     CheckConstraint = "chat_model_configs_compression_threshold_check"       // chat_model_configs
-	CheckChatModelConfigsContextLimitCheck             CheckConstraint = "chat_model_configs_context_limit_check"               // chat_model_configs
-	CheckChatUsageLimitConfigDefaultLimitMicrosCheck   CheckConstraint = "chat_usage_limit_config_default_limit_micros_check"   // chat_usage_limit_config
-	CheckChatUsageLimitConfigPeriodCheck               CheckConstraint = "chat_usage_limit_config_period_check"                 // chat_usage_limit_config
-	CheckChatUsageLimitConfigSingletonCheck            CheckConstraint = "chat_usage_limit_config_singleton_check"              // chat_usage_limit_config
-	CheckChatAclOnlyOnRootChats                        CheckConstraint = "chat_acl_only_on_root_chats"                          // chats
-	CheckChatGroupAclNotNullJsonb                      CheckConstraint = "chat_group_acl_not_null_jsonb"                        // chats
-	CheckChatUserAclNotNullJsonb                       CheckConstraint = "chat_user_acl_not_null_jsonb"                         // chats
-	CheckChatsPinOrderArchivedCheck                    CheckConstraint = "chats_pin_order_archived_check"                       // chats
-	CheckChatsPinOrderParentCheck                      CheckConstraint = "chats_pin_order_parent_check"                         // chats
-	CheckOneTimePasscodeSet                            CheckConstraint = "one_time_passcode_set"                                // users
-	CheckUsersChatSpendLimitMicrosCheck                CheckConstraint = "users_chat_spend_limit_micros_check"                  // users
-	CheckUsersEmailNotEmpty                            CheckConstraint = "users_email_not_empty"                                // users
-	CheckUsersServiceAccountLoginType                  CheckConstraint = "users_service_account_login_type"                     // users
-	CheckUsersUsernameMinLength                        CheckConstraint = "users_username_min_length"                            // users
-	CheckOrganizationIDNotZero                         CheckConstraint = "organization_id_not_zero"                             // custom_roles
-	CheckGroupAIBudgetsSpendLimitMicrosCheck           CheckConstraint = "group_ai_budgets_spend_limit_micros_check"            // group_ai_budgets
-	CheckGroupsChatSpendLimitMicrosCheck               CheckConstraint = "groups_chat_spend_limit_micros_check"                 // groups
-	CheckMcpServerConfigsAuthTypeCheck                 CheckConstraint = "mcp_server_configs_auth_type_check"                   // mcp_server_configs
-	CheckMcpServerConfigsAvailabilityCheck             CheckConstraint = "mcp_server_configs_availability_check"                // mcp_server_configs
-	CheckMcpServerConfigsTransportCheck                CheckConstraint = "mcp_server_configs_transport_check"                   // mcp_server_configs
-	CheckMaxProvisionerLogsLength                      CheckConstraint = "max_provisioner_logs_length"                          // provisioner_jobs
-	CheckNatsPortValidTcp                              CheckConstraint = "nats_port_valid_tcp"                                  // replicas
-	CheckMaxLogsLength                                 CheckConstraint = "max_logs_length"                                      // workspace_agents
-	CheckSubsystemsNotNone                             CheckConstraint = "subsystems_not_none"                                  // workspace_agents
-	CheckWorkspaceBuildsDeadlineBelowMaxDeadline       CheckConstraint = "workspace_builds_deadline_below_max_deadline"         // workspace_builds
-	CheckGroupAclIsObject                              CheckConstraint = "group_acl_is_object"                                  // workspaces
-	CheckUserAclIsObject                               CheckConstraint = "user_acl_is_object"                                   // workspaces
-	CheckTelemetryLockEventTypeConstraint              CheckConstraint = "telemetry_lock_event_type_constraint"                 // telemetry_locks
-	CheckValidationMonotonicOrder                      CheckConstraint = "validation_monotonic_order"                           // template_version_parameters
-	CheckUsageEventTypeCheck                           CheckConstraint = "usage_event_type_check"                               // usage_events
-	CheckUserAIBudgetOverridesSpendLimitMicrosCheck    CheckConstraint = "user_ai_budget_overrides_spend_limit_micros_check"    // user_ai_budget_overrides
-	CheckUserAIProviderKeysAPIKeyCheck                 CheckConstraint = "user_ai_provider_keys_api_key_check"                  // user_ai_provider_keys
-	CheckUserSkillsContentSize                         CheckConstraint = "user_skills_content_size"                             // user_skills
-	CheckUserSkillsDescriptionSize                     CheckConstraint = "user_skills_description_size"                         // user_skills
-	CheckUserSkillsNameFormat                          CheckConstraint = "user_skills_name_format"                              // user_skills
-	CheckUserSkillsNameSize                            CheckConstraint = "user_skills_name_size"                                // user_skills
+	CheckAIGatewayKeysHashedSecretCheck                      CheckConstraint = "ai_gateway_keys_hashed_secret_check"                       // ai_gateway_keys
+	CheckAIGatewayKeysNameCheck                              CheckConstraint = "ai_gateway_keys_name_check"                                // ai_gateway_keys
+	CheckAIGatewayKeysSecretPrefixCheck                      CheckConstraint = "ai_gateway_keys_secret_prefix_check"                       // ai_gateway_keys
+	CheckAIModelPricesCacheReadPriceCheck                    CheckConstraint = "ai_model_prices_cache_read_price_check"                    // ai_model_prices
+	CheckAIModelPricesCacheWritePriceCheck                   CheckConstraint = "ai_model_prices_cache_write_price_check"                   // ai_model_prices
+	CheckAIModelPricesInputPriceCheck                        CheckConstraint = "ai_model_prices_input_price_check"                         // ai_model_prices
+	CheckAIModelPricesOutputPriceCheck                       CheckConstraint = "ai_model_prices_output_price_check"                        // ai_model_prices
+	CheckAIProvidersNameCheck                                CheckConstraint = "ai_providers_name_check"                                   // ai_providers
+	CheckAIUserDailySpendSpendMicrosCheck                    CheckConstraint = "ai_user_daily_spend_spend_micros_check"                    // ai_user_daily_spend
+	CheckAibridgeTokenUsagesCacheReadPriceMicrosCheck        CheckConstraint = "aibridge_token_usages_cache_read_price_micros_check"       // aibridge_token_usages
+	CheckAibridgeTokenUsagesCacheWritePriceMicrosCheck       CheckConstraint = "aibridge_token_usages_cache_write_price_micros_check"      // aibridge_token_usages
+	CheckAibridgeTokenUsagesCostMicrosCheck                  CheckConstraint = "aibridge_token_usages_cost_micros_check"                   // aibridge_token_usages
+	CheckAibridgeTokenUsagesInputPriceMicrosCheck            CheckConstraint = "aibridge_token_usages_input_price_micros_check"            // aibridge_token_usages
+	CheckAibridgeTokenUsagesOutputPriceMicrosCheck           CheckConstraint = "aibridge_token_usages_output_price_micros_check"           // aibridge_token_usages
+	CheckAPIKeysAllowListNotEmpty                            CheckConstraint = "api_keys_allow_list_not_empty"                             // api_keys
+	CheckBoundaryLogsSequenceNumberCheck                     CheckConstraint = "boundary_logs_sequence_number_check"                       // boundary_logs
+	CheckChatModelConfigsAIProviderRequiredWhenActive        CheckConstraint = "chat_model_configs_ai_provider_required_when_active"       // chat_model_configs
+	CheckChatModelConfigsCompressionThresholdCheck           CheckConstraint = "chat_model_configs_compression_threshold_check"            // chat_model_configs
+	CheckChatModelConfigsContextLimitCheck                   CheckConstraint = "chat_model_configs_context_limit_check"                    // chat_model_configs
+	CheckChatUsageLimitConfigDefaultLimitMicrosCheck         CheckConstraint = "chat_usage_limit_config_default_limit_micros_check"        // chat_usage_limit_config
+	CheckChatUsageLimitConfigPeriodCheck                     CheckConstraint = "chat_usage_limit_config_period_check"                      // chat_usage_limit_config
+	CheckChatUsageLimitConfigSingletonCheck                  CheckConstraint = "chat_usage_limit_config_singleton_check"                   // chat_usage_limit_config
+	CheckChatAclOnlyOnRootChats                              CheckConstraint = "chat_acl_only_on_root_chats"                               // chats
+	CheckChatGroupAclNotNullJsonb                            CheckConstraint = "chat_group_acl_not_null_jsonb"                             // chats
+	CheckChatUserAclNotNullJsonb                             CheckConstraint = "chat_user_acl_not_null_jsonb"                              // chats
+	CheckChatsPinOrderArchivedCheck                          CheckConstraint = "chats_pin_order_archived_check"                            // chats
+	CheckChatsPinOrderParentCheck                            CheckConstraint = "chats_pin_order_parent_check"                              // chats
+	CheckOneTimePasscodeSet                                  CheckConstraint = "one_time_passcode_set"                                     // users
+	CheckUsersChatSpendLimitMicrosCheck                      CheckConstraint = "users_chat_spend_limit_micros_check"                       // users
+	CheckUsersEmailNotEmpty                                  CheckConstraint = "users_email_not_empty"                                     // users
+	CheckUsersServiceAccountLoginType                        CheckConstraint = "users_service_account_login_type"                          // users
+	CheckUsersUsernameMinLength                              CheckConstraint = "users_username_min_length"                                 // users
+	CheckOrganizationIDNotZero                               CheckConstraint = "organization_id_not_zero"                                  // custom_roles
+	CheckGroupAIBudgetsSpendLimitMicrosCheck                 CheckConstraint = "group_ai_budgets_spend_limit_micros_check"                 // group_ai_budgets
+	CheckGroupsChatSpendLimitMicrosCheck                     CheckConstraint = "groups_chat_spend_limit_micros_check"                      // groups
+	CheckMcpServerConfigsAuthTypeCheck                       CheckConstraint = "mcp_server_configs_auth_type_check"                        // mcp_server_configs
+	CheckMcpServerConfigsAvailabilityCheck                   CheckConstraint = "mcp_server_configs_availability_check"                     // mcp_server_configs
+	CheckMcpServerConfigsTransportCheck                      CheckConstraint = "mcp_server_configs_transport_check"                        // mcp_server_configs
+	CheckMaxProvisionerLogsLength                            CheckConstraint = "max_provisioner_logs_length"                               // provisioner_jobs
+	CheckNatsPortValidTcp                                    CheckConstraint = "nats_port_valid_tcp"                                       // replicas
+	CheckMaxLogsLength                                       CheckConstraint = "max_logs_length"                                           // workspace_agents
+	CheckSubsystemsNotNone                                   CheckConstraint = "subsystems_not_none"                                       // workspace_agents
+	CheckWorkspaceBuildsDeadlineBelowMaxDeadline             CheckConstraint = "workspace_builds_deadline_below_max_deadline"              // workspace_builds
+	CheckGroupAclIsObject                                    CheckConstraint = "group_acl_is_object"                                       // workspaces
+	CheckUserAclIsObject                                     CheckConstraint = "user_acl_is_object"                                        // workspaces
+	CheckTelemetryLockEventTypeConstraint                    CheckConstraint = "telemetry_lock_event_type_constraint"                      // telemetry_locks
+	CheckValidationMonotonicOrder                            CheckConstraint = "validation_monotonic_order"                                // template_version_parameters
+	CheckUsageEventTypeCheck                                 CheckConstraint = "usage_event_type_check"                                    // usage_events
+	CheckUserAIBudgetOverridesSpendLimitMicrosCheck          CheckConstraint = "user_ai_budget_overrides_spend_limit_micros_check"         // user_ai_budget_overrides
+	CheckUserAIProviderKeysAPIKeyCheck                       CheckConstraint = "user_ai_provider_keys_api_key_check"                       // user_ai_provider_keys
+	CheckUserSkillsContentSize                               CheckConstraint = "user_skills_content_size"                                  // user_skills
+	CheckUserSkillsDescriptionSize                           CheckConstraint = "user_skills_description_size"                              // user_skills
+	CheckUserSkillsNameFormat                                CheckConstraint = "user_skills_name_format"                                   // user_skills
+	CheckUserSkillsNameSize                                  CheckConstraint = "user_skills_name_size"                                     // user_skills
+	CheckWorkspaceBuildOrchestrationsAttemptCountCheck       CheckConstraint = "workspace_build_orchestrations_attempt_count_check"        // workspace_build_orchestrations
+	CheckWorkspaceBuildOrchestrationsChildLogLevelCheck      CheckConstraint = "workspace_build_orchestrations_child_log_level_check"      // workspace_build_orchestrations
+	CheckWorkspaceBuildOrchestrationsChildParametersCheck    CheckConstraint = "workspace_build_orchestrations_child_parameters_check"     // workspace_build_orchestrations
+	CheckWorkspaceBuildOrchestrationsChildPresetVersionCheck CheckConstraint = "workspace_build_orchestrations_child_preset_version_check" // workspace_build_orchestrations
+	CheckWorkspaceBuildOrchestrationsCompletedChildCheck     CheckConstraint = "workspace_build_orchestrations_completed_child_check"      // workspace_build_orchestrations
+	CheckWorkspaceBuildOrchestrationsNextRetryAfterCheck     CheckConstraint = "workspace_build_orchestrations_next_retry_after_check"     // workspace_build_orchestrations
+	CheckWorkspaceBuildOrchestrationsStatusCheck             CheckConstraint = "workspace_build_orchestrations_status_check"               // workspace_build_orchestrations
 )

coderd/database/dbauthz/dbauthz.go

@@ -482,31 +482,32 @@ var (
 				Identifier:  rbac.RoleIdentifier{Name: "system"},
 				DisplayName: "Coder",
 				Site: rbac.Permissions(map[string][]policy.Action{
-					rbac.ResourceWildcard.Type:               {policy.ActionRead},
-					rbac.ResourceApiKey.Type:                 rbac.ResourceApiKey.AvailableActions(),
-					rbac.ResourceGroup.Type:                  {policy.ActionCreate, policy.ActionUpdate},
-					rbac.ResourceAssignRole.Type:             rbac.ResourceAssignRole.AvailableActions(),
-					rbac.ResourceAssignOrgRole.Type:          rbac.ResourceAssignOrgRole.AvailableActions(),
-					rbac.ResourceSystem.Type:                 {policy.WildcardSymbol},
-					rbac.ResourceOrganization.Type:           {policy.ActionCreate, policy.ActionRead},
-					rbac.ResourceOrganizationMember.Type:     {policy.ActionCreate, policy.ActionDelete, policy.ActionRead},
-					rbac.ResourceProvisionerDaemon.Type:      {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate},
-					rbac.ResourceUser.Type:                   rbac.ResourceUser.AvailableActions(),
-					rbac.ResourceWorkspaceDormant.Type:       {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStop},
-					rbac.ResourceWorkspace.Type:              {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, policy.ActionSSH, policy.ActionCreateAgent, policy.ActionDeleteAgent, policy.ActionUpdateAgent},
-					rbac.ResourceWorkspaceProxy.Type:         {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceDeploymentConfig.Type:       {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceNotificationMessage.Type:    {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceNotificationPreference.Type: {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceNotificationTemplate.Type:   {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceCryptoKey.Type:              {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceFile.Type:                   {policy.ActionCreate, policy.ActionRead},
-					rbac.ResourceProvisionerJobs.Type:        {policy.ActionRead, policy.ActionUpdate, policy.ActionCreate},
-					rbac.ResourceOauth2App.Type:              {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceOauth2AppSecret.Type:        {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceChat.Type:                   {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceAIProvider.Type:             {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
-					rbac.ResourceAIGatewayKey.Type:           {policy.ActionRead, policy.ActionUpdate},
+					rbac.ResourceWildcard.Type:                    {policy.ActionRead},
+					rbac.ResourceApiKey.Type:                      rbac.ResourceApiKey.AvailableActions(),
+					rbac.ResourceGroup.Type:                       {policy.ActionCreate, policy.ActionUpdate},
+					rbac.ResourceAssignRole.Type:                  rbac.ResourceAssignRole.AvailableActions(),
+					rbac.ResourceAssignOrgRole.Type:               rbac.ResourceAssignOrgRole.AvailableActions(),
+					rbac.ResourceSystem.Type:                      {policy.WildcardSymbol},
+					rbac.ResourceOrganization.Type:                {policy.ActionCreate, policy.ActionRead},
+					rbac.ResourceOrganizationMember.Type:          {policy.ActionCreate, policy.ActionDelete, policy.ActionRead},
+					rbac.ResourceProvisionerDaemon.Type:           {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate},
+					rbac.ResourceUser.Type:                        rbac.ResourceUser.AvailableActions(),
+					rbac.ResourceWorkspaceDormant.Type:            {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStop},
+					rbac.ResourceWorkspace.Type:                   {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStart, policy.ActionWorkspaceStop, policy.ActionSSH, policy.ActionCreateAgent, policy.ActionDeleteAgent, policy.ActionUpdateAgent},
+					rbac.ResourceWorkspaceProxy.Type:              {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceWorkspaceBuildOrchestration.Type: {policy.ActionUpdate, policy.ActionRead},
+					rbac.ResourceDeploymentConfig.Type:            {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceNotificationMessage.Type:         {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceNotificationPreference.Type:      {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceNotificationTemplate.Type:        {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceCryptoKey.Type:                   {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceFile.Type:                        {policy.ActionCreate, policy.ActionRead},
+					rbac.ResourceProvisionerJobs.Type:             {policy.ActionRead, policy.ActionUpdate, policy.ActionCreate},
+					rbac.ResourceOauth2App.Type:                   {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceOauth2AppSecret.Type:             {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceChat.Type:                        {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceAIProvider.Type:                  {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate, policy.ActionDelete},
+					rbac.ResourceAIGatewayKey.Type:                {policy.ActionRead, policy.ActionUpdate},
 				}),
 				User:    []rbac.Permission{},
 				ByOrgID: map[string]rbac.OrgPermissions{},
@@ -691,10 +692,11 @@ var (
 				Identifier:  rbac.RoleIdentifier{Name: "dbpurge"},
 				DisplayName: "DB Purge Daemon",
 				Site: rbac.Permissions(map[string][]policy.Action{
-					rbac.ResourceSystem.Type:               {policy.ActionDelete},
-					rbac.ResourceNotificationMessage.Type:  {policy.ActionDelete},
-					rbac.ResourceApiKey.Type:               {policy.ActionDelete},
-					rbac.ResourceAibridgeInterception.Type: {policy.ActionDelete},
+					rbac.ResourceSystem.Type:                      {policy.ActionDelete},
+					rbac.ResourceNotificationMessage.Type:         {policy.ActionDelete},
+					rbac.ResourceApiKey.Type:                      {policy.ActionDelete},
+					rbac.ResourceAibridgeInterception.Type:        {policy.ActionDelete},
+					rbac.ResourceWorkspaceBuildOrchestration.Type: {policy.ActionDelete},
 					// Chat auto-archive sets archived=true on inactive chats.
 					rbac.ResourceChat.Type: {policy.ActionRead, policy.ActionUpdate},
 					// Purge old boundary logs past the retention period.
@@ -2390,6 +2392,13 @@ func (q *querier) DeleteOldWorkspaceAgentStats(ctx context.Context) error {
 	return q.db.DeleteOldWorkspaceAgentStats(ctx)
 }
 
+func (q *querier) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) {
+	if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil {
+		return 0, err
+	}
+	return q.db.DeleteOldWorkspaceBuildOrchestrations(ctx, arg)
+}
+
 func (q *querier) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error {
 	return deleteQ[database.OrganizationMember](q.log, q.auth, func(ctx context.Context, arg database.DeleteOrganizationMemberParams) (database.OrganizationMember, error) {
 		member, err := database.ExpectOne(q.OrganizationMembers(ctx, database.OrganizationMembersParams{
@@ -3996,6 +4005,13 @@ func (q *querier) GetMCPServerUserTokensByUserID(ctx context.Context, userID uui
 	return q.db.GetMCPServerUserTokensByUserID(ctx, userID)
 }
 
+func (q *querier) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) (database.WorkspaceBuildOrchestration, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	return q.db.GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx)
+}
+
 func (q *querier) GetNotificationMessagesByStatus(ctx context.Context, arg database.GetNotificationMessagesByStatusParams) ([]database.NotificationMessage, error) {
 	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceNotificationMessage); err != nil {
 		return nil, err
@@ -6466,6 +6482,66 @@ func (q *querier) InsertWorkspaceBuild(ctx context.Context, arg database.InsertW
 	return q.db.InsertWorkspaceBuild(ctx, arg)
 }
 
+func (q *querier) InsertWorkspaceBuildOrchestration(ctx context.Context, arg database.InsertWorkspaceBuildOrchestrationParams) (database.WorkspaceBuildOrchestration, error) {
+	// Read through the raw q.db to fetch the authz context; authorization
+	// happens via q.authorizeContext below, as in InsertWorkspaceBuild.
+	parentBuild, err := q.db.GetWorkspaceBuildByID(ctx, arg.ParentBuildID)
+	if err != nil {
+		return database.WorkspaceBuildOrchestration{}, xerrors.Errorf("get parent workspace build by id: %w", err)
+	}
+
+	workspace, err := q.db.GetWorkspaceByID(ctx, parentBuild.WorkspaceID)
+	if err != nil {
+		return database.WorkspaceBuildOrchestration{}, xerrors.Errorf("get workspace by id: %w", err)
+	}
+	if workspace.IsPrebuild() {
+		return database.WorkspaceBuildOrchestration{}, xerrors.New("cannot orchestrate prebuild workspace builds")
+	}
+
+	// The current API flow inserts this row immediately after
+	// creating the parent build, so the parent transition has already
+	// been authorized. Still, make sure future callers cannot attach
+	// the child intent to a parent build the actor could not initiate.
+	parentAction, err := workspaceTransitionAction(parentBuild.Transition)
+	if err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	if err := q.authorizeContext(ctx, parentAction, workspace); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+
+	// The orchestrator uses system authority to create the child
+	// build after the parent succeeds, so the initiating actor must
+	// be authorized now.
+	childAction, err := workspaceTransitionAction(arg.ChildTransition)
+	if err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	if err := q.authorizeContext(ctx, childAction, workspace); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+
+	if arg.ChildTransition == database.WorkspaceTransitionStart && arg.ChildTemplateVersionID.Valid {
+		// Only template admins may queue child builds with a durable
+		// template version pin, since the active version can change
+		// before the worker creates the child build.
+		template, err := q.db.GetTemplateByID(ctx, workspace.TemplateID)
+		if err != nil {
+			return database.WorkspaceBuildOrchestration{}, xerrors.Errorf("get template by id: %w", err)
+		}
+
+		err = q.authorizeContext(ctx, policy.ActionUpdate, template)
+		var notAuthorized NotAuthorizedError
+		if xerrors.As(err, &notAuthorized) {
+			return database.WorkspaceBuildOrchestration{}, err
+		} else if err != nil {
+			return database.WorkspaceBuildOrchestration{}, xerrors.Errorf("cannot pin template version for child build: %w", err)
+		}
+	}
+
+	return q.db.InsertWorkspaceBuildOrchestration(ctx, arg)
+}
+
 func (q *querier) InsertWorkspaceBuildParameters(ctx context.Context, arg database.InsertWorkspaceBuildParametersParams) error {
 	// TODO: Optimize this. We always have the workspace and build already fetched.
 	build, err := q.db.GetWorkspaceBuildByID(ctx, arg.WorkspaceBuildID)
@@ -8457,6 +8533,34 @@ func (q *querier) UpdateWorkspaceBuildNotifiedAutostopDeadline(ctx context.Conte
 	return q.db.UpdateWorkspaceBuildNotifiedAutostopDeadline(ctx, arg)
 }
 
+func (q *querier) UpdateWorkspaceBuildOrchestrationCanceledByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	return q.db.UpdateWorkspaceBuildOrchestrationCanceledByID(ctx, arg)
+}
+
+func (q *querier) UpdateWorkspaceBuildOrchestrationCompletedByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	return q.db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, arg)
+}
+
+func (q *querier) UpdateWorkspaceBuildOrchestrationFailedByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationFailedByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	return q.db.UpdateWorkspaceBuildOrchestrationFailedByID(ctx, arg)
+}
+
+func (q *querier) UpdateWorkspaceBuildOrchestrationRetryByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationRetryByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization()); err != nil {
+		return database.WorkspaceBuildOrchestration{}, err
+	}
+	return q.db.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, arg)
+}
+
 func (q *querier) UpdateWorkspaceBuildProvisionerStateByID(ctx context.Context, arg database.UpdateWorkspaceBuildProvisionerStateByIDParams) error {
 	if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceSystem); err != nil {
 		return err

coderd/database/dbauthz/dbauthz_test.go

@@ -4024,6 +4024,162 @@ func (s *MethodTestSuite) TestWorkspace() {
 		dbm.EXPECT().InsertWorkspaceBuild(gomock.Any(), arg).Return(nil).AnyTimes()
 		check.Args(arg).Asserts(w, policy.ActionDelete)
 	}))
+	s.Run("Start/PinnedVersion/InsertWorkspaceBuildOrchestration", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		activeVersionID := uuid.New()
+		childVersionID := uuid.New()
+		t := testutil.Fake(s.T(), faker, database.Template{ActiveVersionID: activeVersionID})
+		w := testutil.Fake(s.T(), faker, database.Workspace{TemplateID: t.ID})
+		parentBuild := testutil.Fake(s.T(), faker, database.WorkspaceBuild{
+			WorkspaceID:       w.ID,
+			TemplateVersionID: activeVersionID,
+			Transition:        database.WorkspaceTransitionStop,
+		})
+		arg := database.InsertWorkspaceBuildOrchestrationParams{
+			ParentBuildID:   parentBuild.ID,
+			ChildTransition: database.WorkspaceTransitionStart,
+			ChildTemplateVersionID: uuid.NullUUID{
+				UUID:  childVersionID,
+				Valid: true,
+			},
+		}
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		dbm.EXPECT().GetWorkspaceBuildByID(gomock.Any(), parentBuild.ID).Return(parentBuild, nil).AnyTimes()
+		dbm.EXPECT().GetWorkspaceByID(gomock.Any(), w.ID).Return(w, nil).AnyTimes()
+		dbm.EXPECT().GetTemplateByID(gomock.Any(), t.ID).Return(t, nil).AnyTimes()
+		// Ensure template admins may queue child builds with a durable
+		// template version pin.
+		dbm.EXPECT().InsertWorkspaceBuildOrchestration(gomock.Any(), arg).Return(orchestration, nil).AnyTimes()
+		check.Args(arg).
+			Asserts(
+				w, policy.ActionWorkspaceStop,
+				w, policy.ActionWorkspaceStart,
+				t, policy.ActionUpdate,
+			).
+			Returns(orchestration)
+	}))
+	s.Run("Start/PinnedVersionWithoutTemplateUpdate/InsertWorkspaceBuildOrchestration", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		activeVersionID := uuid.New()
+		t := testutil.Fake(s.T(), faker, database.Template{ActiveVersionID: activeVersionID})
+		w := testutil.Fake(s.T(), faker, database.Workspace{TemplateID: t.ID})
+		parentBuild := testutil.Fake(s.T(), faker, database.WorkspaceBuild{
+			WorkspaceID:       w.ID,
+			TemplateVersionID: activeVersionID,
+			Transition:        database.WorkspaceTransitionStop,
+		})
+		arg := database.InsertWorkspaceBuildOrchestrationParams{
+			ParentBuildID:   parentBuild.ID,
+			ChildTransition: database.WorkspaceTransitionStart,
+			ChildTemplateVersionID: uuid.NullUUID{
+				UUID:  activeVersionID,
+				Valid: true,
+			},
+		}
+		dbm.EXPECT().GetWorkspaceBuildByID(gomock.Any(), parentBuild.ID).Return(parentBuild, nil).AnyTimes()
+		dbm.EXPECT().GetWorkspaceByID(gomock.Any(), w.ID).Return(w, nil).AnyTimes()
+		dbm.EXPECT().GetTemplateByID(gomock.Any(), t.ID).Return(t, nil).AnyTimes()
+		// Ensure non-template admins cannot queue a durable template
+		// version pin for the child build.
+		check.Args(arg).
+			Asserts(
+				w, policy.ActionWorkspaceStop,
+				w, policy.ActionWorkspaceStart,
+				t, policy.ActionUpdate,
+			).
+			Errors(errMatchAny).
+			WithSuccessAuthorizer(func(_ context.Context, _ rbac.Subject, action policy.Action, obj rbac.Object) error {
+				if action == policy.ActionUpdate && obj.Type == rbac.ResourceTemplate.Type {
+					return xerrors.New("not authorized to update template")
+				}
+				return nil
+			})
+	}))
+	s.Run("Start/UnpinnedVersion/InsertWorkspaceBuildOrchestration", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		t := testutil.Fake(s.T(), faker, database.Template{})
+		w := testutil.Fake(s.T(), faker, database.Workspace{TemplateID: t.ID})
+		parentBuild := testutil.Fake(s.T(), faker, database.WorkspaceBuild{
+			WorkspaceID: w.ID,
+			Transition:  database.WorkspaceTransitionStop,
+		})
+		arg := database.InsertWorkspaceBuildOrchestrationParams{
+			ParentBuildID:   parentBuild.ID,
+			ChildTransition: database.WorkspaceTransitionStart,
+		}
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		dbm.EXPECT().GetWorkspaceBuildByID(gomock.Any(), parentBuild.ID).Return(parentBuild, nil).AnyTimes()
+		dbm.EXPECT().GetWorkspaceByID(gomock.Any(), w.ID).Return(w, nil).AnyTimes()
+		// Ensure an unpinned child build does not require template update permission.
+		dbm.EXPECT().InsertWorkspaceBuildOrchestration(gomock.Any(), arg).Return(orchestration, nil).AnyTimes()
+		check.Args(arg).
+			Asserts(
+				w, policy.ActionWorkspaceStop,
+				w, policy.ActionWorkspaceStart,
+			).
+			Returns(orchestration)
+	}))
+	s.Run("GetNextPendingWorkspaceBuildOrchestrationForUpdate", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		dbm.EXPECT().GetNextPendingWorkspaceBuildOrchestrationForUpdate(gomock.Any()).Return(orchestration, nil).AnyTimes()
+		check.Args().
+			Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionRead).
+			Returns(orchestration)
+	}))
+	s.Run("UpdateWorkspaceBuildOrchestrationCanceledByID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		arg := database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams{
+			UpdatedAt: dbtime.Now(),
+			ID:        orchestration.ID,
+		}
+		dbm.EXPECT().UpdateWorkspaceBuildOrchestrationCanceledByID(gomock.Any(), arg).Return(orchestration, nil).AnyTimes()
+		check.Args(arg).
+			Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionUpdate).
+			Returns(orchestration)
+	}))
+	s.Run("UpdateWorkspaceBuildOrchestrationCompletedByID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		arg := database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{
+			ChildBuildID: uuid.NullUUID{UUID: uuid.New(), Valid: true},
+			UpdatedAt:    dbtime.Now(),
+			ID:           orchestration.ID,
+		}
+		dbm.EXPECT().UpdateWorkspaceBuildOrchestrationCompletedByID(gomock.Any(), arg).Return(orchestration, nil).AnyTimes()
+		check.Args(arg).
+			Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionUpdate).
+			Returns(orchestration)
+	}))
+	s.Run("UpdateWorkspaceBuildOrchestrationFailedByID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		arg := database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{
+			Error:     sql.NullString{String: "failed", Valid: true},
+			UpdatedAt: dbtime.Now(),
+			ID:        orchestration.ID,
+		}
+		dbm.EXPECT().UpdateWorkspaceBuildOrchestrationFailedByID(gomock.Any(), arg).Return(orchestration, nil).AnyTimes()
+		check.Args(arg).
+			Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionUpdate).
+			Returns(orchestration)
+	}))
+	s.Run("UpdateWorkspaceBuildOrchestrationRetryByID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		orchestration := testutil.Fake(s.T(), faker, database.WorkspaceBuildOrchestration{})
+		arg := database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+			MaxAttemptCount: 3,
+			NextRetryAfter:  dbtime.Now(),
+			Error:           sql.NullString{String: "retry", Valid: true},
+			UpdatedAt:       dbtime.Now(),
+			ID:              orchestration.ID,
+		}
+		dbm.EXPECT().UpdateWorkspaceBuildOrchestrationRetryByID(gomock.Any(), arg).Return(orchestration, nil).AnyTimes()
+		check.Args(arg).
+			Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionUpdate).
+			Returns(orchestration)
+	}))
+	s.Run("DeleteOldWorkspaceBuildOrchestrations", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
+		arg := database.DeleteOldWorkspaceBuildOrchestrationsParams{
+			BeforeTime: dbtime.Now(),
+			LimitCount: 100,
+		}
+		dbm.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), arg).Return(int64(0), nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourceWorkspaceBuildOrchestration.AnyOrganization(), policy.ActionDelete)
+	}))
 	s.Run("Start/InsertWorkspaceBuildParameters", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
 		w := testutil.Fake(s.T(), faker, database.Workspace{})
 		b := testutil.Fake(s.T(), faker, database.WorkspaceBuild{

coderd/database/dbmetrics/querymetrics.go

@@ -793,6 +793,14 @@ func (m queryMetricsStore) DeleteOldWorkspaceAgentStats(ctx context.Context) err
 	return r0
 }
 
+func (m queryMetricsStore) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) {
+	start := time.Now()
+	r0, r1 := m.s.DeleteOldWorkspaceBuildOrchestrations(ctx, arg)
+	m.queryLatencies.WithLabelValues("DeleteOldWorkspaceBuildOrchestrations").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "DeleteOldWorkspaceBuildOrchestrations").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error {
 	start := time.Now()
 	r0 := m.s.DeleteOrganizationMember(ctx, arg)
@@ -2337,6 +2345,14 @@ func (m queryMetricsStore) GetMCPServerUserTokensByUserID(ctx context.Context, u
 	return r0, r1
 }
 
+func (m queryMetricsStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) (database.WorkspaceBuildOrchestration, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx)
+	m.queryLatencies.WithLabelValues("GetNextPendingWorkspaceBuildOrchestrationForUpdate").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetNextPendingWorkspaceBuildOrchestrationForUpdate").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetNotificationMessagesByStatus(ctx context.Context, arg database.GetNotificationMessagesByStatusParams) ([]database.NotificationMessage, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetNotificationMessagesByStatus(ctx, arg)
@@ -4529,6 +4545,14 @@ func (m queryMetricsStore) InsertWorkspaceBuild(ctx context.Context, arg databas
 	return r0
 }
 
+func (m queryMetricsStore) InsertWorkspaceBuildOrchestration(ctx context.Context, arg database.InsertWorkspaceBuildOrchestrationParams) (database.WorkspaceBuildOrchestration, error) {
+	start := time.Now()
+	r0, r1 := m.s.InsertWorkspaceBuildOrchestration(ctx, arg)
+	m.queryLatencies.WithLabelValues("InsertWorkspaceBuildOrchestration").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "InsertWorkspaceBuildOrchestration").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) InsertWorkspaceBuildParameters(ctx context.Context, arg database.InsertWorkspaceBuildParametersParams) error {
 	start := time.Now()
 	r0 := m.s.InsertWorkspaceBuildParameters(ctx, arg)
@@ -5953,6 +5977,38 @@ func (m queryMetricsStore) UpdateWorkspaceBuildNotifiedAutostopDeadline(ctx cont
 	return r0
 }
 
+func (m queryMetricsStore) UpdateWorkspaceBuildOrchestrationCanceledByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	start := time.Now()
+	r0, r1 := m.s.UpdateWorkspaceBuildOrchestrationCanceledByID(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateWorkspaceBuildOrchestrationCanceledByID").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpdateWorkspaceBuildOrchestrationCanceledByID").Inc()
+	return r0, r1
+}
+
+func (m queryMetricsStore) UpdateWorkspaceBuildOrchestrationCompletedByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	start := time.Now()
+	r0, r1 := m.s.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateWorkspaceBuildOrchestrationCompletedByID").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpdateWorkspaceBuildOrchestrationCompletedByID").Inc()
+	return r0, r1
+}
+
+func (m queryMetricsStore) UpdateWorkspaceBuildOrchestrationFailedByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationFailedByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	start := time.Now()
+	r0, r1 := m.s.UpdateWorkspaceBuildOrchestrationFailedByID(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateWorkspaceBuildOrchestrationFailedByID").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpdateWorkspaceBuildOrchestrationFailedByID").Inc()
+	return r0, r1
+}
+
+func (m queryMetricsStore) UpdateWorkspaceBuildOrchestrationRetryByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationRetryByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	start := time.Now()
+	r0, r1 := m.s.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, arg)
+	m.queryLatencies.WithLabelValues("UpdateWorkspaceBuildOrchestrationRetryByID").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "UpdateWorkspaceBuildOrchestrationRetryByID").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) UpdateWorkspaceBuildProvisionerStateByID(ctx context.Context, arg database.UpdateWorkspaceBuildProvisionerStateByIDParams) error {
 	start := time.Now()
 	r0 := m.s.UpdateWorkspaceBuildProvisionerStateByID(ctx, arg)

coderd/database/dbmock/dbmock.go

@@ -1337,6 +1337,21 @@ func (mr *MockStoreMockRecorder) DeleteOldWorkspaceAgentStats(ctx any) *gomock.C
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldWorkspaceAgentStats", reflect.TypeOf((*MockStore)(nil).DeleteOldWorkspaceAgentStats), ctx)
 }
 
+// DeleteOldWorkspaceBuildOrchestrations mocks base method.
+func (m *MockStore) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg database.DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "DeleteOldWorkspaceBuildOrchestrations", ctx, arg)
+	ret0, _ := ret[0].(int64)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// DeleteOldWorkspaceBuildOrchestrations indicates an expected call of DeleteOldWorkspaceBuildOrchestrations.
+func (mr *MockStoreMockRecorder) DeleteOldWorkspaceBuildOrchestrations(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteOldWorkspaceBuildOrchestrations", reflect.TypeOf((*MockStore)(nil).DeleteOldWorkspaceBuildOrchestrations), ctx, arg)
+}
+
 // DeleteOrganizationMember mocks base method.
 func (m *MockStore) DeleteOrganizationMember(ctx context.Context, arg database.DeleteOrganizationMemberParams) error {
 	m.ctrl.T.Helper()
@@ -4333,6 +4348,21 @@ func (mr *MockStoreMockRecorder) GetMCPServerUserTokensByUserID(ctx, userID any)
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMCPServerUserTokensByUserID", reflect.TypeOf((*MockStore)(nil).GetMCPServerUserTokensByUserID), ctx, userID)
 }
 
+// GetNextPendingWorkspaceBuildOrchestrationForUpdate mocks base method.
+func (m *MockStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) (database.WorkspaceBuildOrchestration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetNextPendingWorkspaceBuildOrchestrationForUpdate", ctx)
+	ret0, _ := ret[0].(database.WorkspaceBuildOrchestration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetNextPendingWorkspaceBuildOrchestrationForUpdate indicates an expected call of GetNextPendingWorkspaceBuildOrchestrationForUpdate.
+func (mr *MockStoreMockRecorder) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNextPendingWorkspaceBuildOrchestrationForUpdate", reflect.TypeOf((*MockStore)(nil).GetNextPendingWorkspaceBuildOrchestrationForUpdate), ctx)
+}
+
 // GetNotificationMessagesByStatus mocks base method.
 func (m *MockStore) GetNotificationMessagesByStatus(ctx context.Context, arg database.GetNotificationMessagesByStatusParams) ([]database.NotificationMessage, error) {
 	m.ctrl.T.Helper()
@@ -8471,6 +8501,21 @@ func (mr *MockStoreMockRecorder) InsertWorkspaceBuild(ctx, arg any) *gomock.Call
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertWorkspaceBuild", reflect.TypeOf((*MockStore)(nil).InsertWorkspaceBuild), ctx, arg)
 }
 
+// InsertWorkspaceBuildOrchestration mocks base method.
+func (m *MockStore) InsertWorkspaceBuildOrchestration(ctx context.Context, arg database.InsertWorkspaceBuildOrchestrationParams) (database.WorkspaceBuildOrchestration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "InsertWorkspaceBuildOrchestration", ctx, arg)
+	ret0, _ := ret[0].(database.WorkspaceBuildOrchestration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// InsertWorkspaceBuildOrchestration indicates an expected call of InsertWorkspaceBuildOrchestration.
+func (mr *MockStoreMockRecorder) InsertWorkspaceBuildOrchestration(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertWorkspaceBuildOrchestration", reflect.TypeOf((*MockStore)(nil).InsertWorkspaceBuildOrchestration), ctx, arg)
+}
+
 // InsertWorkspaceBuildParameters mocks base method.
 func (m *MockStore) InsertWorkspaceBuildParameters(ctx context.Context, arg database.InsertWorkspaceBuildParametersParams) error {
 	m.ctrl.T.Helper()
@@ -11166,6 +11211,66 @@ func (mr *MockStoreMockRecorder) UpdateWorkspaceBuildNotifiedAutostopDeadline(ct
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateWorkspaceBuildNotifiedAutostopDeadline", reflect.TypeOf((*MockStore)(nil).UpdateWorkspaceBuildNotifiedAutostopDeadline), ctx, arg)
 }
 
+// UpdateWorkspaceBuildOrchestrationCanceledByID mocks base method.
+func (m *MockStore) UpdateWorkspaceBuildOrchestrationCanceledByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateWorkspaceBuildOrchestrationCanceledByID", ctx, arg)
+	ret0, _ := ret[0].(database.WorkspaceBuildOrchestration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateWorkspaceBuildOrchestrationCanceledByID indicates an expected call of UpdateWorkspaceBuildOrchestrationCanceledByID.
+func (mr *MockStoreMockRecorder) UpdateWorkspaceBuildOrchestrationCanceledByID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateWorkspaceBuildOrchestrationCanceledByID", reflect.TypeOf((*MockStore)(nil).UpdateWorkspaceBuildOrchestrationCanceledByID), ctx, arg)
+}
+
+// UpdateWorkspaceBuildOrchestrationCompletedByID mocks base method.
+func (m *MockStore) UpdateWorkspaceBuildOrchestrationCompletedByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateWorkspaceBuildOrchestrationCompletedByID", ctx, arg)
+	ret0, _ := ret[0].(database.WorkspaceBuildOrchestration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateWorkspaceBuildOrchestrationCompletedByID indicates an expected call of UpdateWorkspaceBuildOrchestrationCompletedByID.
+func (mr *MockStoreMockRecorder) UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateWorkspaceBuildOrchestrationCompletedByID", reflect.TypeOf((*MockStore)(nil).UpdateWorkspaceBuildOrchestrationCompletedByID), ctx, arg)
+}
+
+// UpdateWorkspaceBuildOrchestrationFailedByID mocks base method.
+func (m *MockStore) UpdateWorkspaceBuildOrchestrationFailedByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationFailedByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateWorkspaceBuildOrchestrationFailedByID", ctx, arg)
+	ret0, _ := ret[0].(database.WorkspaceBuildOrchestration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateWorkspaceBuildOrchestrationFailedByID indicates an expected call of UpdateWorkspaceBuildOrchestrationFailedByID.
+func (mr *MockStoreMockRecorder) UpdateWorkspaceBuildOrchestrationFailedByID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateWorkspaceBuildOrchestrationFailedByID", reflect.TypeOf((*MockStore)(nil).UpdateWorkspaceBuildOrchestrationFailedByID), ctx, arg)
+}
+
+// UpdateWorkspaceBuildOrchestrationRetryByID mocks base method.
+func (m *MockStore) UpdateWorkspaceBuildOrchestrationRetryByID(ctx context.Context, arg database.UpdateWorkspaceBuildOrchestrationRetryByIDParams) (database.WorkspaceBuildOrchestration, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "UpdateWorkspaceBuildOrchestrationRetryByID", ctx, arg)
+	ret0, _ := ret[0].(database.WorkspaceBuildOrchestration)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// UpdateWorkspaceBuildOrchestrationRetryByID indicates an expected call of UpdateWorkspaceBuildOrchestrationRetryByID.
+func (mr *MockStoreMockRecorder) UpdateWorkspaceBuildOrchestrationRetryByID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateWorkspaceBuildOrchestrationRetryByID", reflect.TypeOf((*MockStore)(nil).UpdateWorkspaceBuildOrchestrationRetryByID), ctx, arg)
+}
+
 // UpdateWorkspaceBuildProvisionerStateByID mocks base method.
 func (m *MockStore) UpdateWorkspaceBuildProvisionerStateByID(ctx context.Context, arg database.UpdateWorkspaceBuildProvisionerStateByIDParams) error {
 	m.ctrl.T.Helper()

coderd/database/dbpurge/dbpurge.go

@@ -39,6 +39,11 @@ const (
 	// long enough to cover the maximum interval of a heartbeat event (currently
 	// 1 hour) plus some buffer.
 	maxTelemetryHeartbeatAge = 24 * time.Hour
+	// Operational handoff state; terminal rows are kept for debugging, then
+	// purged.
+	workspaceBuildOrchestrationTerminalRetention = 24 * time.Hour
+	// Batch size for workspace build orchestration deletion.
+	workspaceBuildOrchestrationsBatchSize = 10000
 	// Chat and chat file batch sizes stay smaller than audit/connection
 	// log batches because chat_files rows carry bytea blobs.
 	chatsBatchSize     = 1000
@@ -275,6 +280,15 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
 			}
 		}
 
+		deleteOldWorkspaceBuildOrchestrationsBefore := start.Add(-workspaceBuildOrchestrationTerminalRetention)
+		purgedWorkspaceBuildOrchestrations, err := tx.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{
+			BeforeTime: deleteOldWorkspaceBuildOrchestrationsBefore,
+			LimitCount: workspaceBuildOrchestrationsBatchSize,
+		})
+		if err != nil {
+			return xerrors.Errorf("failed to delete old workspace build orchestrations: %w", err)
+		}
+
 		var purgedChats, purgedChatFiles, purgedChatDebugRuns int64
 		if purgeChats {
 			purgedChats, purgedChatFiles, err = i.purgeChatsInTx(ctx, tx, start, chatRetentionDays)
@@ -304,6 +318,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
 			slog.F("audit_logs", purgedAuditLogs),
 			slog.F("boundary_logs", purgedBoundaryLogs),
 			slog.F("boundary_sessions", purgedBoundarySessions),
+			slog.F("workspace_build_orchestrations", purgedWorkspaceBuildOrchestrations),
 			slog.F("chats", purgedChats),
 			slog.F("chat_files", purgedChatFiles),
 			slog.F("chat_debug_runs", purgedChatDebugRuns),
@@ -318,6 +333,7 @@ func (i *instance) purgeTick(ctx context.Context, db database.Store, start time.
 			i.recordsPurged.WithLabelValues("audit_logs").Add(float64(purgedAuditLogs))
 			i.recordsPurged.WithLabelValues("boundary_logs").Add(float64(purgedBoundaryLogs))
 			i.recordsPurged.WithLabelValues("boundary_sessions").Add(float64(purgedBoundarySessions))
+			i.recordsPurged.WithLabelValues("workspace_build_orchestrations").Add(float64(purgedWorkspaceBuildOrchestrations))
 			i.recordsPurged.WithLabelValues("chats").Add(float64(purgedChats))
 			i.recordsPurged.WithLabelValues("chat_debug_runs").Add(float64(purgedChatDebugRuns))
 			i.recordsPurged.WithLabelValues("chat_files").Add(float64(purgedChatFiles))

coderd/database/dbpurge/dbpurge_test.go

@@ -129,6 +129,11 @@ func TestMetrics(t *testing.T) {
 		})
 		require.GreaterOrEqual(t, auditLogs, 0)
 
+		workspaceBuildOrchestrations := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{
+			"record_type": "workspace_build_orchestrations",
+		})
+		require.GreaterOrEqual(t, workspaceBuildOrchestrations, 0)
+
 		chats := promhelp.CounterValue(t, reg, "coderd_dbpurge_records_purged_total", prometheus.Labels{
 			"record_type": "chats",
 		})
@@ -247,6 +252,7 @@ func TestMetrics(t *testing.T) {
 		mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+		mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
 		mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldChatDebugRuns(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatDebugRunsParams{})).Return(int64(0), nil).MinTimes(1)
 		mDB.EXPECT().InTx(gomock.Any(), database.DefaultTXOptions().WithID("db_purge")).
@@ -297,6 +303,7 @@ func TestMetrics(t *testing.T) {
 		mDB.EXPECT().DeleteOldNotificationMessages(gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().ExpirePrebuildsAPIKeys(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldTelemetryLocks(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+		mDB.EXPECT().DeleteOldWorkspaceBuildOrchestrations(gomock.Any(), gomock.Any()).Return(int64(0), nil).AnyTimes()
 		mDB.EXPECT().DeleteOldAuditLogConnectionEvents(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
 		mDB.EXPECT().DeleteOldChats(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatsParams{})).Return(int64(0), nil).MinTimes(1)
 		mDB.EXPECT().DeleteOldChatFiles(gomock.Any(), gomock.AssignableToTypeOf(database.DeleteOldChatFilesParams{})).Return(int64(0), nil).MinTimes(1)
@@ -1117,6 +1124,146 @@ func TestDeleteOldTelemetryHeartbeats(t *testing.T) {
 	}, testutil.WaitShort, testutil.IntervalFast, "it should delete old telemetry heartbeats")
 }
 
+func TestDeleteOldWorkspaceBuildOrchestrations(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	db, _, rawDB := dbtestutil.NewDBWithSQLDB(t)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		JobID:          versionJob.ID,
+		CreatedBy:      user.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OwnerID:        user.ID,
+		OrganizationID: org.ID,
+		TemplateID:     template.ID,
+	})
+
+	now := dbtime.Now()
+	cutoff := now.Add(-24 * time.Hour)
+	buildTime := cutoff.Add(-time.Hour)
+	oldCompletedTime := cutoff.Add(-3 * time.Minute)
+	oldFailedTime := cutoff.Add(-2 * time.Minute)
+	oldCanceledTime := cutoff.Add(-time.Minute)
+	oldPendingTime := cutoff.Add(-time.Minute)
+	recentTime := cutoff.Add(time.Minute)
+
+	createBuild := func(buildNumber int32, createdAt time.Time) database.WorkspaceBuild {
+		return mustCreateWorkspaceBuild(t, db, org, version, workspace.ID, createdAt, buildNumber)
+	}
+	insertOrchestration := func(parentBuild database.WorkspaceBuild, updatedAt time.Time) database.WorkspaceBuildOrchestration {
+		orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+			ID:                       uuid.New(),
+			CreatedAt:                updatedAt,
+			UpdatedAt:                updatedAt,
+			ParentBuildID:            parentBuild.ID,
+			ChildTransition:          database.WorkspaceTransitionStart,
+			ChildRichParameterValues: json.RawMessage("[]"),
+		})
+		require.NoError(t, err)
+		return orchestration
+	}
+
+	// Given: old terminal orchestration rows (completed, failed,
+	// canceled), an old pending row, and a recent terminal row.
+	oldCompletedParent := createBuild(1, buildTime)
+	oldCompletedChild := createBuild(2, buildTime)
+	oldCompleted := insertOrchestration(oldCompletedParent, oldCompletedTime)
+	_, err := db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{
+		ID:           oldCompleted.ID,
+		ChildBuildID: uuid.NullUUID{UUID: oldCompletedChild.ID, Valid: true},
+		UpdatedAt:    oldCompletedTime,
+	})
+	require.NoError(t, err)
+
+	oldFailedParent := createBuild(3, buildTime)
+	oldFailed := insertOrchestration(oldFailedParent, oldFailedTime)
+	_, err = db.UpdateWorkspaceBuildOrchestrationFailedByID(ctx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{
+		ID:        oldFailed.ID,
+		Error:     sql.NullString{String: "failed", Valid: true},
+		UpdatedAt: oldFailedTime,
+	})
+	require.NoError(t, err)
+
+	oldCanceledParent := createBuild(4, buildTime)
+	oldCanceled := insertOrchestration(oldCanceledParent, oldCanceledTime)
+	_, err = db.UpdateWorkspaceBuildOrchestrationCanceledByID(ctx, database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams{
+		ID:        oldCanceled.ID,
+		UpdatedAt: oldCanceledTime,
+	})
+	require.NoError(t, err)
+
+	oldPendingParent := createBuild(5, buildTime)
+	oldPending := insertOrchestration(oldPendingParent, oldPendingTime)
+
+	recentCompletedParent := createBuild(6, buildTime)
+	recentCompletedChild := createBuild(7, buildTime)
+	recentCompleted := insertOrchestration(recentCompletedParent, recentTime)
+	_, err = db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{
+		ID:           recentCompleted.ID,
+		ChildBuildID: uuid.NullUUID{UUID: recentCompletedChild.ID, Valid: true},
+		UpdatedAt:    recentTime,
+	})
+	require.NoError(t, err)
+
+	// When: old workspace build orchestrations are deleted with LimitCount 1
+	deleted, err := db.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{
+		BeforeTime: cutoff,
+		LimitCount: 1,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, 1, deleted)
+
+	// Then: only the oldest terminal row is deleted.
+	assertOrchestrationDeleted(ctx, t, rawDB, oldCompletedParent.ID)
+	assertOrchestrationExists(ctx, t, rawDB, oldFailedParent.ID, oldFailed.ID)
+	assertOrchestrationExists(ctx, t, rawDB, oldCanceledParent.ID, oldCanceled.ID)
+	assertOrchestrationExists(ctx, t, rawDB, oldPendingParent.ID, oldPending.ID)
+	assertOrchestrationExists(ctx, t, rawDB, recentCompletedParent.ID, recentCompleted.ID)
+
+	// When: old workspace build orchestrations are deleted again.
+	deleted, err = db.DeleteOldWorkspaceBuildOrchestrations(ctx, database.DeleteOldWorkspaceBuildOrchestrationsParams{
+		BeforeTime: cutoff,
+		LimitCount: 10,
+	})
+	require.NoError(t, err)
+	require.EqualValues(t, 2, deleted)
+
+	// Then: the remaining old terminal rows are deleted.
+	assertOrchestrationDeleted(ctx, t, rawDB, oldFailedParent.ID)
+	assertOrchestrationDeleted(ctx, t, rawDB, oldCanceledParent.ID)
+	assertOrchestrationExists(ctx, t, rawDB, oldPendingParent.ID, oldPending.ID)
+	assertOrchestrationExists(ctx, t, rawDB, recentCompletedParent.ID, recentCompleted.ID)
+}
+
+func assertOrchestrationDeleted(ctx context.Context, t *testing.T, rawDB *sql.DB, parentBuildID uuid.UUID) {
+	t.Helper()
+
+	_, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuildID)
+	require.ErrorIs(t, err, sql.ErrNoRows)
+}
+
+func assertOrchestrationExists(ctx context.Context, t *testing.T, rawDB *sql.DB, parentBuildID uuid.UUID, orchestrationID uuid.UUID) {
+	t.Helper()
+
+	orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuildID)
+	require.NoError(t, err)
+	require.Equal(t, orchestrationID, orchestration.ID)
+}
+
 func TestDeleteOldConnectionLogs(t *testing.T) {
 	t.Parallel()
 

coderd/database/dbtestutil/workspacebuildorchestrations.go

@@ -0,0 +1,32 @@
+package dbtestutil
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/google/uuid"
+	"github.com/jmoiron/sqlx"
+
+	"github.com/coder/coder/v2/coderd/database"
+)
+
+// GetWorkspaceBuildOrchestrationByParentBuildID reads a workspace
+// build orchestration row directly from the database for tests.
+//
+// It scans into the struct by column name so new columns are picked up
+// automatically without updating this helper.
+func GetWorkspaceBuildOrchestrationByParentBuildID(
+	ctx context.Context,
+	sqlDB *sql.DB,
+	parentBuildID uuid.UUID,
+) (database.WorkspaceBuildOrchestration, error) {
+	db := sqlx.NewDb(sqlDB, "postgres")
+	var orchestration database.WorkspaceBuildOrchestration
+	err := db.GetContext(
+		ctx,
+		&orchestration,
+		`SELECT * FROM workspace_build_orchestrations WHERE parent_build_id = $1`,
+		parentBuildID,
+	)
+	return orchestration, err
+}

coderd/database/dump.sql

@@ -258,7 +258,12 @@ CREATE TYPE api_key_scope AS ENUM (
     'ai_gateway_key:create',
     'ai_gateway_key:delete',
     'ai_gateway_key:read',
-    'ai_gateway_key:update'
+    'ai_gateway_key:update',
+    'workspace_build_orchestration:*',
+    'workspace_build_orchestration:create',
+    'workspace_build_orchestration:delete',
+    'workspace_build_orchestration:read',
+    'workspace_build_orchestration:update'
 );
 
 CREATE TYPE app_sharing_level AS ENUM (
@@ -3868,6 +3873,44 @@ CREATE TABLE workspace_app_statuses (
     uri text
 );
 
+CREATE TABLE workspace_build_orchestrations (
+    id uuid NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    workspace_id uuid NOT NULL,
+    parent_build_id uuid NOT NULL,
+    child_build_id uuid,
+    child_transition workspace_transition NOT NULL,
+    child_template_version_id uuid,
+    child_template_version_preset_id uuid,
+    child_rich_parameter_values jsonb DEFAULT '[]'::jsonb NOT NULL,
+    child_log_level text DEFAULT ''::text NOT NULL,
+    child_reason build_reason,
+    attempt_count integer DEFAULT 0 NOT NULL,
+    next_retry_after timestamp with time zone,
+    status text DEFAULT 'pending'::text NOT NULL,
+    error text,
+    CONSTRAINT workspace_build_orchestrations_attempt_count_check CHECK ((attempt_count >= 0)),
+    CONSTRAINT workspace_build_orchestrations_child_log_level_check CHECK ((child_log_level = ANY (ARRAY[''::text, 'debug'::text]))),
+    CONSTRAINT workspace_build_orchestrations_child_parameters_check CHECK ((jsonb_typeof(child_rich_parameter_values) = 'array'::text)),
+    CONSTRAINT workspace_build_orchestrations_child_preset_version_check CHECK (((child_template_version_preset_id IS NULL) OR (child_template_version_id IS NOT NULL))),
+    CONSTRAINT workspace_build_orchestrations_completed_child_check CHECK (((status <> 'completed'::text) OR (child_build_id IS NOT NULL))),
+    CONSTRAINT workspace_build_orchestrations_next_retry_after_check CHECK (((status = 'pending'::text) OR (next_retry_after IS NULL))),
+    CONSTRAINT workspace_build_orchestrations_status_check CHECK ((status = ANY (ARRAY['pending'::text, 'completed'::text, 'failed'::text, 'canceled'::text])))
+);
+
+COMMENT ON TABLE workspace_build_orchestrations IS 'Tracks durable follow-up workspace build operations, such as server-side restart, where one child build is created after a parent build completes successfully.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.workspace_id IS 'Copied from the parent build so the database can enforce that parent and child builds belong to the same workspace.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.parent_build_id IS 'Unique because we only support sequences with one child build per parent build.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.child_build_id IS 'Nullable because the child build is created only after the parent build completes successfully.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.attempt_count IS 'Counts retryable child build creation failures for this orchestration row.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.next_retry_after IS 'When set, the orchestrator skips this pending row until the timestamp has passed.';
+
 CREATE TABLE workspace_build_parameters (
     workspace_build_id uuid NOT NULL,
     name text NOT NULL,
@@ -4382,6 +4425,9 @@ ALTER TABLE ONLY template_version_preset_parameters
 ALTER TABLE ONLY template_version_preset_prebuild_schedules
     ADD CONSTRAINT template_version_preset_prebuild_schedules_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY template_version_presets
+    ADD CONSTRAINT template_version_presets_id_template_version_id_key UNIQUE (id, template_version_id);
+
 ALTER TABLE ONLY template_version_presets
     ADD CONSTRAINT template_version_presets_pkey PRIMARY KEY (id);
 
@@ -4499,9 +4545,21 @@ ALTER TABLE ONLY workspace_apps
 ALTER TABLE ONLY workspace_apps
     ADD CONSTRAINT workspace_apps_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_child_build_id_key UNIQUE (child_build_id);
+
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_parent_build_id_key UNIQUE (parent_build_id);
+
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY workspace_build_parameters
     ADD CONSTRAINT workspace_build_parameters_workspace_build_id_name_key UNIQUE (workspace_build_id, name);
 
+ALTER TABLE ONLY workspace_builds
+    ADD CONSTRAINT workspace_builds_id_workspace_id_key UNIQUE (id, workspace_id);
+
 ALTER TABLE ONLY workspace_builds
     ADD CONSTRAINT workspace_builds_job_id_key UNIQUE (job_id);
 
@@ -4747,6 +4805,8 @@ CREATE UNIQUE INDEX idx_users_username ON users USING btree (username) WHERE (de
 
 CREATE INDEX idx_workspace_app_statuses_workspace_id_created_at ON workspace_app_statuses USING btree (workspace_id, created_at DESC);
 
+CREATE INDEX idx_workspace_build_orchestrations_pending ON workspace_build_orchestrations USING btree (created_at) WHERE (status = 'pending'::text);
+
 CREATE INDEX idx_workspace_builds_initiator_id ON workspace_builds USING btree (initiator_id);
 
 CREATE UNIQUE INDEX notification_messages_dedupe_hash_idx ON notification_messages USING btree (dedupe_hash);
@@ -5376,6 +5436,21 @@ ALTER TABLE ONLY workspace_app_statuses
 ALTER TABLE ONLY workspace_apps
     ADD CONSTRAINT workspace_apps_agent_id_fkey FOREIGN KEY (agent_id) REFERENCES workspace_agents(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_child_build_workspace_id_fkey FOREIGN KEY (child_build_id, workspace_id) REFERENCES workspace_builds(id, workspace_id) ON DELETE CASCADE;
+
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_child_preset_id_fkey FOREIGN KEY (child_template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE SET NULL;
+
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_child_preset_version_fkey FOREIGN KEY (child_template_version_preset_id, child_template_version_id) REFERENCES template_version_presets(id, template_version_id);
+
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_child_template_version_id_fkey FOREIGN KEY (child_template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
+
+ALTER TABLE ONLY workspace_build_orchestrations
+    ADD CONSTRAINT workspace_build_orchestrations_parent_build_workspace_id_fkey FOREIGN KEY (parent_build_id, workspace_id) REFERENCES workspace_builds(id, workspace_id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY workspace_build_parameters
     ADD CONSTRAINT workspace_build_parameters_workspace_build_id_fkey FOREIGN KEY (workspace_build_id) REFERENCES workspace_builds(id) ON DELETE CASCADE;
 

coderd/database/foreign_key_constraint.go

@@ -145,6 +145,11 @@ const (
 	ForeignKeyWorkspaceAppStatusesAppID                           ForeignKeyConstraint = "workspace_app_statuses_app_id_fkey"                              // ALTER TABLE ONLY workspace_app_statuses ADD CONSTRAINT workspace_app_statuses_app_id_fkey FOREIGN KEY (app_id) REFERENCES workspace_apps(id);
 	ForeignKeyWorkspaceAppStatusesWorkspaceID                     ForeignKeyConstraint = "workspace_app_statuses_workspace_id_fkey"                        // ALTER TABLE ONLY workspace_app_statuses ADD CONSTRAINT workspace_app_statuses_workspace_id_fkey FOREIGN KEY (workspace_id) REFERENCES workspaces(id);
 	ForeignKeyWorkspaceAppsAgentID                                ForeignKeyConstraint = "workspace_apps_agent_id_fkey"                                    // ALTER TABLE ONLY workspace_apps ADD CONSTRAINT workspace_apps_agent_id_fkey FOREIGN KEY (agent_id) REFERENCES workspace_agents(id) ON DELETE CASCADE;
+	ForeignKeyWorkspaceBuildOrchestrationsChildBuildWorkspaceID   ForeignKeyConstraint = "workspace_build_orchestrations_child_build_workspace_id_fkey"    // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_child_build_workspace_id_fkey FOREIGN KEY (child_build_id, workspace_id) REFERENCES workspace_builds(id, workspace_id) ON DELETE CASCADE;
+	ForeignKeyWorkspaceBuildOrchestrationsChildPresetID           ForeignKeyConstraint = "workspace_build_orchestrations_child_preset_id_fkey"             // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_child_preset_id_fkey FOREIGN KEY (child_template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE SET NULL;
+	ForeignKeyWorkspaceBuildOrchestrationsChildPresetVersion      ForeignKeyConstraint = "workspace_build_orchestrations_child_preset_version_fkey"        // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_child_preset_version_fkey FOREIGN KEY (child_template_version_preset_id, child_template_version_id) REFERENCES template_version_presets(id, template_version_id);
+	ForeignKeyWorkspaceBuildOrchestrationsChildTemplateVersionID  ForeignKeyConstraint = "workspace_build_orchestrations_child_template_version_id_fkey"   // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_child_template_version_id_fkey FOREIGN KEY (child_template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
+	ForeignKeyWorkspaceBuildOrchestrationsParentBuildWorkspaceID  ForeignKeyConstraint = "workspace_build_orchestrations_parent_build_workspace_id_fkey"   // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_parent_build_workspace_id_fkey FOREIGN KEY (parent_build_id, workspace_id) REFERENCES workspace_builds(id, workspace_id) ON DELETE CASCADE;
 	ForeignKeyWorkspaceBuildParametersWorkspaceBuildID            ForeignKeyConstraint = "workspace_build_parameters_workspace_build_id_fkey"              // ALTER TABLE ONLY workspace_build_parameters ADD CONSTRAINT workspace_build_parameters_workspace_build_id_fkey FOREIGN KEY (workspace_build_id) REFERENCES workspace_builds(id) ON DELETE CASCADE;
 	ForeignKeyWorkspaceBuildsJobID                                ForeignKeyConstraint = "workspace_builds_job_id_fkey"                                    // ALTER TABLE ONLY workspace_builds ADD CONSTRAINT workspace_builds_job_id_fkey FOREIGN KEY (job_id) REFERENCES provisioner_jobs(id) ON DELETE CASCADE;
 	ForeignKeyWorkspaceBuildsTemplateVersionID                    ForeignKeyConstraint = "workspace_builds_template_version_id_fkey"                       // ALTER TABLE ONLY workspace_builds ADD CONSTRAINT workspace_builds_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;

coderd/database/migrations/000540_workspace_build_orchestrations.down.sql

@@ -0,0 +1,10 @@
+-- Enum additions to api_key_scope are intentionally not reversed
+-- because Postgres cannot drop enum values safely.
+
+DROP TABLE IF EXISTS workspace_build_orchestrations;
+
+ALTER TABLE template_version_presets
+    DROP CONSTRAINT IF EXISTS template_version_presets_id_template_version_id_key;
+
+ALTER TABLE workspace_builds
+    DROP CONSTRAINT IF EXISTS workspace_builds_id_workspace_id_key;

coderd/database/migrations/000540_workspace_build_orchestrations.up.sql

@@ -0,0 +1,103 @@
+-- Postgres requires the referenced column set of a composite foreign
+-- key to have its own unique constraint, even though id is already
+-- unique.
+ALTER TABLE workspace_builds
+    ADD CONSTRAINT workspace_builds_id_workspace_id_key
+        UNIQUE (id, workspace_id);
+
+ALTER TABLE template_version_presets
+    ADD CONSTRAINT template_version_presets_id_template_version_id_key
+        UNIQUE (id, template_version_id);
+
+CREATE TABLE workspace_build_orchestrations (
+    id UUID PRIMARY KEY NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL,
+    updated_at TIMESTAMPTZ NOT NULL,
+    workspace_id UUID NOT NULL,
+    parent_build_id UUID UNIQUE NOT NULL,
+    child_build_id UUID UNIQUE,
+    child_transition workspace_transition NOT NULL,
+    child_template_version_id UUID REFERENCES template_versions(id) ON DELETE CASCADE,
+    child_template_version_preset_id UUID, -- a constraint is added below
+    child_rich_parameter_values JSONB DEFAULT '[]'::JSONB NOT NULL,
+    child_log_level TEXT DEFAULT '' NOT NULL,
+    child_reason build_reason,
+    attempt_count INTEGER DEFAULT 0 NOT NULL,
+    next_retry_after TIMESTAMPTZ,
+    status TEXT DEFAULT 'pending' NOT NULL,
+    error TEXT,
+    CONSTRAINT workspace_build_orchestrations_status_check CHECK (
+        status IN ('pending', 'completed', 'failed', 'canceled')
+    ),
+    CONSTRAINT workspace_build_orchestrations_completed_child_check CHECK (
+        status <> 'completed' OR child_build_id IS NOT NULL
+    ),
+    CONSTRAINT workspace_build_orchestrations_child_parameters_check CHECK (
+        jsonb_typeof(child_rich_parameter_values) = 'array'
+    ),
+    CONSTRAINT workspace_build_orchestrations_attempt_count_check CHECK (
+        attempt_count >= 0
+    ),
+    CONSTRAINT workspace_build_orchestrations_next_retry_after_check CHECK (
+        status = 'pending' OR next_retry_after IS NULL
+    ),
+    CONSTRAINT workspace_build_orchestrations_child_preset_version_check CHECK (
+        child_template_version_preset_id IS NULL OR child_template_version_id IS NOT NULL
+    ),
+    -- Mirrors CreateWorkspaceBuildRequest validation, where the optional
+    -- log level is either unset or debug.
+    CONSTRAINT workspace_build_orchestrations_child_log_level_check CHECK (
+        child_log_level IN ('', 'debug')
+    ),
+    -- These constraints enforce that any stored child preset belongs to
+    -- the requested child template version, while preset deletion still
+    -- clears only the preset column.
+    CONSTRAINT workspace_build_orchestrations_child_preset_id_fkey
+        FOREIGN KEY (child_template_version_preset_id)
+        REFERENCES template_version_presets(id)
+        ON DELETE SET NULL,
+    CONSTRAINT workspace_build_orchestrations_child_preset_version_fkey
+        FOREIGN KEY (child_template_version_preset_id, child_template_version_id)
+        REFERENCES template_version_presets(id, template_version_id),
+    -- Composite foreign keys enforce that the parent and child builds
+    -- belong to the same workspace.
+    CONSTRAINT workspace_build_orchestrations_parent_build_workspace_id_fkey
+        FOREIGN KEY (parent_build_id, workspace_id)
+        REFERENCES workspace_builds(id, workspace_id)
+        ON DELETE CASCADE,
+    CONSTRAINT workspace_build_orchestrations_child_build_workspace_id_fkey
+        FOREIGN KEY (child_build_id, workspace_id)
+        REFERENCES workspace_builds(id, workspace_id)
+        ON DELETE CASCADE
+);
+
+-- The orchestrator scans eligible pending rows oldest first and skips
+-- terminal rows and retry rows whose delay has not elapsed.
+CREATE INDEX idx_workspace_build_orchestrations_pending
+    ON workspace_build_orchestrations (created_at)
+    WHERE status = 'pending';
+
+COMMENT ON TABLE workspace_build_orchestrations IS
+    'Tracks durable follow-up workspace build operations, such as server-side restart, where one child build is created after a parent build completes successfully.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.parent_build_id IS
+    'Unique because we only support sequences with one child build per parent build.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.workspace_id IS
+    'Copied from the parent build so the database can enforce that parent and child builds belong to the same workspace.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.child_build_id IS
+    'Nullable because the child build is created only after the parent build completes successfully.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.attempt_count IS
+    'Counts retryable child build creation failures for this orchestration row.';
+
+COMMENT ON COLUMN workspace_build_orchestrations.next_retry_after IS
+    'When set, the orchestrator skips this pending row until the timestamp has passed.';
+
+-- Add workspace_build_orchestration scopes for RBAC.
+ALTER TYPE api_key_scope ADD VALUE IF NOT EXISTS 'workspace_build_orchestration:*';
+ALTER TYPE api_key_scope ADD VALUE IF NOT EXISTS 'workspace_build_orchestration:create';
+ALTER TYPE api_key_scope ADD VALUE IF NOT EXISTS 'workspace_build_orchestration:delete';
+ALTER TYPE api_key_scope ADD VALUE IF NOT EXISTS 'workspace_build_orchestration:read';
+ALTER TYPE api_key_scope ADD VALUE IF NOT EXISTS 'workspace_build_orchestration:update';

coderd/database/migrations/testdata/fixtures/000540_workspace_build_orchestrations.up.sql

@@ -0,0 +1,21 @@
+INSERT INTO workspace_build_orchestrations (
+	id,
+	created_at,
+	updated_at,
+	workspace_id,
+	parent_build_id,
+	child_transition
+)
+SELECT
+	'4e983a68-9b8a-4d4e-a4d6-5f2dd73551c2'::uuid,
+	NOW(),
+	NOW(),
+	workspace_id,
+	id,
+	'start'::workspace_transition
+FROM
+	workspace_builds
+ORDER BY
+	created_at, id
+LIMIT 1
+ON CONFLICT DO NOTHING;

coderd/database/models.go

@@ -387,6 +387,11 @@ const (
 	ApiKeyScopeAIGatewayKeyDelete                  APIKeyScope = "ai_gateway_key:delete"
 	ApiKeyScopeAIGatewayKeyRead                    APIKeyScope = "ai_gateway_key:read"
 	ApiKeyScopeAIGatewayKeyUpdate                  APIKeyScope = "ai_gateway_key:update"
+	ApiKeyScopeWorkspaceBuildOrchestration         APIKeyScope = "workspace_build_orchestration:*"
+	ApiKeyScopeWorkspaceBuildOrchestrationCreate   APIKeyScope = "workspace_build_orchestration:create"
+	ApiKeyScopeWorkspaceBuildOrchestrationDelete   APIKeyScope = "workspace_build_orchestration:delete"
+	ApiKeyScopeWorkspaceBuildOrchestrationRead     APIKeyScope = "workspace_build_orchestration:read"
+	ApiKeyScopeWorkspaceBuildOrchestrationUpdate   APIKeyScope = "workspace_build_orchestration:update"
 )
 
 func (e *APIKeyScope) Scan(src interface{}) error {
@@ -656,7 +661,12 @@ func (e APIKeyScope) Valid() bool {
 		ApiKeyScopeAIGatewayKeyCreate,
 		ApiKeyScopeAIGatewayKeyDelete,
 		ApiKeyScopeAIGatewayKeyRead,
-		ApiKeyScopeAIGatewayKeyUpdate:
+		ApiKeyScopeAIGatewayKeyUpdate,
+		ApiKeyScopeWorkspaceBuildOrchestration,
+		ApiKeyScopeWorkspaceBuildOrchestrationCreate,
+		ApiKeyScopeWorkspaceBuildOrchestrationDelete,
+		ApiKeyScopeWorkspaceBuildOrchestrationRead,
+		ApiKeyScopeWorkspaceBuildOrchestrationUpdate:
 		return true
 	}
 	return false
@@ -895,6 +905,11 @@ func AllAPIKeyScopeValues() []APIKeyScope {
 		ApiKeyScopeAIGatewayKeyDelete,
 		ApiKeyScopeAIGatewayKeyRead,
 		ApiKeyScopeAIGatewayKeyUpdate,
+		ApiKeyScopeWorkspaceBuildOrchestration,
+		ApiKeyScopeWorkspaceBuildOrchestrationCreate,
+		ApiKeyScopeWorkspaceBuildOrchestrationDelete,
+		ApiKeyScopeWorkspaceBuildOrchestrationRead,
+		ApiKeyScopeWorkspaceBuildOrchestrationUpdate,
 	}
 }
 
@@ -6461,6 +6476,31 @@ type WorkspaceBuild struct {
 	InitiatorByName          string              `db:"initiator_by_name" json:"initiator_by_name"`
 }
 
+// Tracks durable follow-up workspace build operations, such as server-side restart, where one child build is created after a parent build completes successfully.
+type WorkspaceBuildOrchestration struct {
+	ID        uuid.UUID `db:"id" json:"id"`
+	CreatedAt time.Time `db:"created_at" json:"created_at"`
+	UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+	// Copied from the parent build so the database can enforce that parent and child builds belong to the same workspace.
+	WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"`
+	// Unique because we only support sequences with one child build per parent build.
+	ParentBuildID uuid.UUID `db:"parent_build_id" json:"parent_build_id"`
+	// Nullable because the child build is created only after the parent build completes successfully.
+	ChildBuildID                 uuid.NullUUID       `db:"child_build_id" json:"child_build_id"`
+	ChildTransition              WorkspaceTransition `db:"child_transition" json:"child_transition"`
+	ChildTemplateVersionID       uuid.NullUUID       `db:"child_template_version_id" json:"child_template_version_id"`
+	ChildTemplateVersionPresetID uuid.NullUUID       `db:"child_template_version_preset_id" json:"child_template_version_preset_id"`
+	ChildRichParameterValues     json.RawMessage     `db:"child_rich_parameter_values" json:"child_rich_parameter_values"`
+	ChildLogLevel                string              `db:"child_log_level" json:"child_log_level"`
+	ChildReason                  NullBuildReason     `db:"child_reason" json:"child_reason"`
+	// Counts retryable child build creation failures for this orchestration row.
+	AttemptCount int32 `db:"attempt_count" json:"attempt_count"`
+	// When set, the orchestrator skips this pending row until the timestamp has passed.
+	NextRetryAfter sql.NullTime   `db:"next_retry_after" json:"next_retry_after"`
+	Status         string         `db:"status" json:"status"`
+	Error          sql.NullString `db:"error" json:"error"`
+}
+
 type WorkspaceBuildParameter struct {
 	WorkspaceBuildID uuid.UUID `db:"workspace_build_id" json:"workspace_build_id"`
 	// Parameter name

coderd/database/querier.go

@@ -214,6 +214,7 @@ type sqlcQuerier interface {
 	// Logs can take up a lot of space, so it's important we clean up frequently.
 	DeleteOldWorkspaceAgentLogs(ctx context.Context, threshold time.Time) (int64, error)
 	DeleteOldWorkspaceAgentStats(ctx context.Context) error
+	DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error)
 	DeleteOrganizationMember(ctx context.Context, arg DeleteOrganizationMemberParams) error
 	DeleteProvisionerKey(ctx context.Context, id uuid.UUID) error
 	DeleteReplicasUpdatedBefore(ctx context.Context, updatedAt time.Time) error
@@ -610,6 +611,9 @@ type sqlcQuerier interface {
 	GetMCPServerConfigsByIDs(ctx context.Context, ids []uuid.UUID) ([]MCPServerConfig, error)
 	GetMCPServerUserToken(ctx context.Context, arg GetMCPServerUserTokenParams) (MCPServerUserToken, error)
 	GetMCPServerUserTokensByUserID(ctx context.Context, userID uuid.UUID) ([]MCPServerUserToken, error)
+	// Must be called from within a transaction. The row lock is released
+	// when the transaction ends.
+	GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) (WorkspaceBuildOrchestration, error)
 	GetNotificationMessagesByStatus(ctx context.Context, arg GetNotificationMessagesByStatusParams) ([]NotificationMessage, error)
 	// Fetch the notification report generator log indicating recent activity.
 	GetNotificationReportGeneratorLogByTemplate(ctx context.Context, templateID uuid.UUID) (NotificationReportGeneratorLog, error)
@@ -1117,6 +1121,7 @@ type sqlcQuerier interface {
 	InsertWorkspaceAppStats(ctx context.Context, arg InsertWorkspaceAppStatsParams) error
 	InsertWorkspaceAppStatus(ctx context.Context, arg InsertWorkspaceAppStatusParams) (WorkspaceAppStatus, error)
 	InsertWorkspaceBuild(ctx context.Context, arg InsertWorkspaceBuildParams) error
+	InsertWorkspaceBuildOrchestration(ctx context.Context, arg InsertWorkspaceBuildOrchestrationParams) (WorkspaceBuildOrchestration, error)
 	InsertWorkspaceBuildParameters(ctx context.Context, arg InsertWorkspaceBuildParametersParams) error
 	InsertWorkspaceModule(ctx context.Context, arg InsertWorkspaceModuleParams) (WorkspaceModule, error)
 	InsertWorkspaceProxy(ctx context.Context, arg InsertWorkspaceProxyParams) (WorkspaceProxy, error)
@@ -1488,6 +1493,10 @@ type sqlcQuerier interface {
 	// reminder idempotent and HA-safe. It re-arms automatically when the deadline
 	// changes (e.g. an activity bump).
 	UpdateWorkspaceBuildNotifiedAutostopDeadline(ctx context.Context, arg UpdateWorkspaceBuildNotifiedAutostopDeadlineParams) error
+	UpdateWorkspaceBuildOrchestrationCanceledByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationCanceledByIDParams) (WorkspaceBuildOrchestration, error)
+	UpdateWorkspaceBuildOrchestrationCompletedByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationCompletedByIDParams) (WorkspaceBuildOrchestration, error)
+	UpdateWorkspaceBuildOrchestrationFailedByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationFailedByIDParams) (WorkspaceBuildOrchestration, error)
+	UpdateWorkspaceBuildOrchestrationRetryByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationRetryByIDParams) (WorkspaceBuildOrchestration, error)
 	UpdateWorkspaceBuildProvisionerStateByID(ctx context.Context, arg UpdateWorkspaceBuildProvisionerStateByIDParams) error
 	UpdateWorkspaceDeletedByID(ctx context.Context, arg UpdateWorkspaceDeletedByIDParams) error
 	UpdateWorkspaceDormantDeletingAt(ctx context.Context, arg UpdateWorkspaceDormantDeletingAtParams) (WorkspaceTable, error)

coderd/database/querier_test.go

@@ -8800,6 +8800,586 @@ func TestUserSecretsAuthorization(t *testing.T) {
 	}
 }
 
+func TestUpdateWorkspaceBuildOrchestrationRetryByIDMaxAttempts(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionJob.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+	buildJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeWorkspaceBuild,
+	})
+	parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       workspace.ID,
+		TemplateVersionID: version.ID,
+		InitiatorID:       user.ID,
+		JobID:             buildJob.ID,
+		Transition:        database.WorkspaceTransitionStop,
+	})
+
+	// Given: a pending orchestration row.
+	now := dbtime.Now()
+	orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+		ID:                       uuid.New(),
+		CreatedAt:                now,
+		UpdatedAt:                now,
+		ParentBuildID:            parentBuild.ID,
+		ChildTransition:          database.WorkspaceTransitionStart,
+		ChildRichParameterValues: json.RawMessage("[]"),
+	})
+	require.NoError(t, err)
+	require.Equal(t, workspace.ID, orchestration.WorkspaceID)
+
+	const maxAttemptCount = 3
+	const retryError = "some retryable child build failure"
+	recordRetry := func(t *testing.T, wantAttempt int32, wantStatus string, wantNextRetry bool) {
+		t.Helper()
+
+		now := dbtime.Now()
+		nextRetryAfter := now.Add(time.Minute)
+		got, err := db.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+			Error: sql.NullString{
+				String: retryError,
+				Valid:  true,
+			},
+			NextRetryAfter:  nextRetryAfter,
+			UpdatedAt:       now,
+			ID:              orchestration.ID,
+			MaxAttemptCount: maxAttemptCount,
+		})
+		require.NoError(t, err)
+		require.Equal(t, wantAttempt, got.AttemptCount)
+		require.Equal(t, wantStatus, got.Status)
+		require.True(t, got.Error.Valid)
+		require.Equal(t, retryError, got.Error.String)
+		require.Equal(t, wantNextRetry, got.NextRetryAfter.Valid)
+		if wantNextRetry {
+			require.False(t, got.NextRetryAfter.Time.Before(nextRetryAfter))
+		}
+	}
+
+	// When: retryable child build failures are recorded until one
+	// attempt remains.
+	recordRetry(t, 1, "pending", true)
+	recordRetry(t, 2, "pending", true)
+
+	// Then: the next attempt fails the row, clears its retry delay,
+	// and prevents further retry updates.
+	recordRetry(t, 3, "failed", false)
+	_, err = db.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+		Error: sql.NullString{
+			String: retryError,
+			Valid:  true,
+		},
+		NextRetryAfter:  dbtime.Now().Add(time.Minute),
+		UpdatedAt:       dbtime.Now(),
+		ID:              orchestration.ID,
+		MaxAttemptCount: maxAttemptCount,
+	})
+	require.ErrorIs(t, err, sql.ErrNoRows)
+}
+
+func TestUpdateWorkspaceBuildOrchestrationRetryByIDPendingGateUnderContention(t *testing.T) {
+	t.Parallel()
+
+	db, _, _ := dbtestutil.NewDBWithSQLDB(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionJob.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+	buildJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeWorkspaceBuild,
+	})
+	parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       workspace.ID,
+		TemplateVersionID: version.ID,
+		InitiatorID:       user.ID,
+		JobID:             buildJob.ID,
+		Transition:        database.WorkspaceTransitionStop,
+	})
+
+	// Given: a pending orchestration row.
+	now := dbtime.Now()
+	orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+		ID:                       uuid.New(),
+		CreatedAt:                now,
+		UpdatedAt:                now,
+		ParentBuildID:            parentBuild.ID,
+		ChildTransition:          database.WorkspaceTransitionStart,
+		ChildRichParameterValues: json.RawMessage("[]"),
+	})
+	require.NoError(t, err)
+
+	const retryError = "some retryable child build failure"
+	type retryResult struct {
+		orchestration database.WorkspaceBuildOrchestration
+		err           error
+	}
+	firstUpdated := make(chan retryResult, 1)
+	releaseFirst := make(chan struct{})
+	firstErr := make(chan error, 1)
+	secondStarted := make(chan struct{}, 1)
+	secondErr := make(chan error, 1)
+	secondDone := make(chan struct{})
+
+	// When: one retry update terminalizes the row while another
+	// worker races to retry the same pending row.
+	go func() {
+		err := db.InTx(func(tx database.Store) error {
+			got, err := tx.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+				Error: sql.NullString{
+					String: retryError,
+					Valid:  true,
+				},
+				NextRetryAfter: dbtime.Now().Add(time.Minute),
+				UpdatedAt:      dbtime.Now(),
+				ID:             orchestration.ID,
+				// Use a single allowed retry so the winning update
+				// immediately terminalizes the row.
+				MaxAttemptCount: 1,
+			})
+			if err != nil {
+				firstUpdated <- retryResult{err: err}
+				return err
+			}
+			firstUpdated <- retryResult{orchestration: got}
+			<-releaseFirst
+			return nil
+		}, nil)
+		firstErr <- err
+	}()
+
+	firstResult := <-firstUpdated
+	require.NoError(t, firstResult.err)
+
+	go func() {
+		secondStarted <- struct{}{}
+		_, err := db.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+			Error: sql.NullString{
+				String: retryError,
+				Valid:  true,
+			},
+			NextRetryAfter:  dbtime.Now().Add(time.Minute),
+			UpdatedAt:       dbtime.Now(),
+			ID:              orchestration.ID,
+			MaxAttemptCount: 1,
+		})
+		secondErr <- err
+		close(secondDone)
+	}()
+
+	<-secondStarted
+	// Then: while the first transaction is held open, the second
+	// update does not complete.
+	require.Never(t, func() bool {
+		select {
+		case <-secondDone:
+			return true
+		default:
+			return false
+		}
+	}, time.Second, testutil.IntervalFast)
+
+	close(releaseFirst)
+	require.NoError(t, <-firstErr)
+
+	// Then: after the first transaction commits the failed status, the
+	// second update rechecks the pending gate and affects no rows.
+	require.ErrorIs(t, <-secondErr, sql.ErrNoRows)
+}
+
+func TestGetNextPendingWorkspaceBuildOrchestrationForUpdateRetryDelay(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionJob.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+
+	var buildNumber int32
+	createOrchestration := func(t *testing.T, createdAt time.Time) database.WorkspaceBuildOrchestration {
+		t.Helper()
+
+		buildNumber++
+		buildJob := database.ProvisionerJob{
+			OrganizationID: org.ID,
+			Type:           database.ProvisionerJobTypeWorkspaceBuild,
+		}
+		setJobStatus(t, database.ProvisionerJobStatusSucceeded, &buildJob)
+		buildJob = dbgen.ProvisionerJob(t, db, nil, buildJob)
+		parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+			WorkspaceID:       workspace.ID,
+			TemplateVersionID: version.ID,
+			InitiatorID:       user.ID,
+			JobID:             buildJob.ID,
+			BuildNumber:       buildNumber,
+			Transition:        database.WorkspaceTransitionStop,
+		})
+
+		orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+			ID:                       uuid.New(),
+			CreatedAt:                createdAt,
+			UpdatedAt:                createdAt,
+			ParentBuildID:            parentBuild.ID,
+			ChildTransition:          database.WorkspaceTransitionStart,
+			ChildRichParameterValues: json.RawMessage("[]"),
+		})
+		require.NoError(t, err)
+		require.Equal(t, workspace.ID, orchestration.WorkspaceID)
+		return orchestration
+	}
+
+	claimNext := func(t *testing.T) (database.WorkspaceBuildOrchestration, error) {
+		t.Helper()
+
+		var orchestration database.WorkspaceBuildOrchestration
+		err := db.InTx(func(tx database.Store) error {
+			var err error
+			orchestration, err = tx.GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx)
+			return err
+		}, nil)
+		return orchestration, err
+	}
+
+	baseTime := dbtime.Now().Add(-time.Hour)
+
+	// Given: an old pending orchestration row with a future retry delay.
+	delayed := createOrchestration(t, baseTime)
+	_, err := db.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+		Error: sql.NullString{
+			String: "retry later",
+			Valid:  true,
+		},
+		NextRetryAfter:  dbtime.Now().Add(time.Hour),
+		UpdatedAt:       dbtime.Now(),
+		ID:              delayed.ID,
+		MaxAttemptCount: 3,
+	})
+	require.NoError(t, err)
+
+	// When: the orchestrator claims the next eligible row.
+	_, err = claimNext(t)
+
+	// Then: no row is claimed before the retry time.
+	require.ErrorIs(t, err, sql.ErrNoRows)
+
+	// When: a later row is eligible immediately.
+	eligible := createOrchestration(t, baseTime.Add(time.Minute))
+
+	// Then: the old delayed row does not block the later eligible row.
+	got, err := claimNext(t)
+	require.NoError(t, err)
+	require.Equal(t, eligible.ID, got.ID)
+
+	// When: the delayed row's retry time has passed.
+	_, err = db.UpdateWorkspaceBuildOrchestrationRetryByID(ctx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+		Error: sql.NullString{
+			String: "retry now",
+			Valid:  true,
+		},
+		NextRetryAfter:  dbtime.Now().Add(-time.Minute),
+		UpdatedAt:       dbtime.Now(),
+		ID:              delayed.ID,
+		MaxAttemptCount: 3,
+	})
+	require.NoError(t, err)
+
+	// Then: the delayed row is eligible again and is claimed first
+	// because it is older than the later row.
+	got, err = claimNext(t)
+	require.NoError(t, err)
+	require.Equal(t, delayed.ID, got.ID)
+}
+
+func TestUpdateWorkspaceBuildOrchestrationCompletedByIDWorkspaceMismatch(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionJob.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+
+	parentWorkspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+	otherWorkspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+
+	parentJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeWorkspaceBuild,
+	})
+	parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       parentWorkspace.ID,
+		TemplateVersionID: version.ID,
+		InitiatorID:       user.ID,
+		JobID:             parentJob.ID,
+		Transition:        database.WorkspaceTransitionStop,
+	})
+
+	// Given: a pending orchestration row.
+	orchestration, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+		ID:                       uuid.New(),
+		CreatedAt:                dbtime.Now(),
+		UpdatedAt:                dbtime.Now(),
+		ParentBuildID:            parentBuild.ID,
+		ChildTransition:          database.WorkspaceTransitionStart,
+		ChildRichParameterValues: json.RawMessage("[]"),
+	})
+	require.NoError(t, err)
+	require.Equal(t, parentWorkspace.ID, orchestration.WorkspaceID)
+
+	// Given: a child build whose workspace does not match the parent
+	// build's workspace.
+	childJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeWorkspaceBuild,
+	})
+	childBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       otherWorkspace.ID,
+		TemplateVersionID: version.ID,
+		InitiatorID:       user.ID,
+		JobID:             childJob.ID,
+		Transition:        database.WorkspaceTransitionStart,
+	})
+
+	// When: the orchestration is completed with the child build.
+	_, err = db.UpdateWorkspaceBuildOrchestrationCompletedByID(ctx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{
+		ID:           orchestration.ID,
+		ChildBuildID: uuid.NullUUID{UUID: childBuild.ID, Valid: true},
+		UpdatedAt:    dbtime.Now(),
+	})
+
+	// Then: the composite foreign key rejects the mismatched child build.
+	require.Error(t, err)
+	require.True(t, database.IsForeignKeyViolation(err))
+}
+
+func TestInsertWorkspaceBuildOrchestrationPresetRequiresVersion(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionJob.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+	parentJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeWorkspaceBuild,
+	})
+
+	// Given: a parent build, a child preset, no child template
+	// version.
+	parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       workspace.ID,
+		TemplateVersionID: version.ID,
+		InitiatorID:       user.ID,
+		JobID:             parentJob.ID,
+		Transition:        database.WorkspaceTransitionStop,
+	})
+	preset := dbgen.Preset(t, db, database.InsertPresetParams{
+		TemplateVersionID: version.ID,
+	})
+
+	// When: the orchestration row is inserted.
+	_, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+		ID:              uuid.New(),
+		CreatedAt:       dbtime.Now(),
+		UpdatedAt:       dbtime.Now(),
+		ParentBuildID:   parentBuild.ID,
+		ChildTransition: database.WorkspaceTransitionStart,
+		ChildTemplateVersionPresetID: uuid.NullUUID{
+			UUID:  preset.ID,
+			Valid: true,
+		},
+		ChildRichParameterValues: json.RawMessage("[]"),
+	})
+
+	// Then: the check constraint rejects the missing child template
+	// version.
+	require.Error(t, err)
+	require.True(t, database.IsCheckViolation(err))
+}
+
+func TestInsertWorkspaceBuildOrchestrationPresetVersionMismatch(t *testing.T) {
+	t.Parallel()
+
+	db, _ := dbtestutil.NewDB(t)
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionOneJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	versionOne := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionOneJob.ID,
+	})
+	versionTwoJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	versionTwo := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+		JobID:          versionTwoJob.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: versionOne.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     template.ID,
+	})
+	parentJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeWorkspaceBuild,
+	})
+
+	// Given: a parent build and a child preset.
+	parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       workspace.ID,
+		TemplateVersionID: versionOne.ID,
+		InitiatorID:       user.ID,
+		JobID:             parentJob.ID,
+		Transition:        database.WorkspaceTransitionStop,
+	})
+	preset := dbgen.Preset(t, db, database.InsertPresetParams{
+		TemplateVersionID: versionOne.ID,
+	})
+
+	// When: the orchestration row is inserted with a different child
+	// template version.
+	_, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+		ID:              uuid.New(),
+		CreatedAt:       dbtime.Now(),
+		UpdatedAt:       dbtime.Now(),
+		ParentBuildID:   parentBuild.ID,
+		ChildTransition: database.WorkspaceTransitionStart,
+		ChildTemplateVersionID: uuid.NullUUID{
+			UUID:  versionTwo.ID,
+			Valid: true,
+		},
+		ChildTemplateVersionPresetID: uuid.NullUUID{
+			UUID:  preset.ID,
+			Valid: true,
+		},
+		ChildRichParameterValues: json.RawMessage("[]"),
+	})
+
+	// Then: the composite foreign key rejects the preset/version
+	// mismatch.
+	require.Error(t, err)
+	require.True(t, database.IsForeignKeyViolation(err))
+}
+
 func TestWorkspaceBuildDeadlineConstraint(t *testing.T) {
 	t.Parallel()
 

coderd/database/queries.sql.go

@@ -35141,6 +35141,359 @@ func (q *sqlQuerier) InsertWorkspaceAppStats(ctx context.Context, arg InsertWork
 	return err
 }
 
+const deleteOldWorkspaceBuildOrchestrations = `-- name: DeleteOldWorkspaceBuildOrchestrations :execrows
+WITH deletable AS (
+    SELECT
+        id
+    FROM
+        workspace_build_orchestrations
+    WHERE
+        status IN ('completed', 'failed', 'canceled')
+        AND updated_at < $1::timestamptz
+    ORDER BY
+        updated_at ASC
+    LIMIT $2::int
+)
+DELETE FROM workspace_build_orchestrations
+USING deletable
+WHERE workspace_build_orchestrations.id = deletable.id
+`
+
+type DeleteOldWorkspaceBuildOrchestrationsParams struct {
+	BeforeTime time.Time `db:"before_time" json:"before_time"`
+	LimitCount int32     `db:"limit_count" json:"limit_count"`
+}
+
+func (q *sqlQuerier) DeleteOldWorkspaceBuildOrchestrations(ctx context.Context, arg DeleteOldWorkspaceBuildOrchestrationsParams) (int64, error) {
+	result, err := q.db.ExecContext(ctx, deleteOldWorkspaceBuildOrchestrations, arg.BeforeTime, arg.LimitCount)
+	if err != nil {
+		return 0, err
+	}
+	return result.RowsAffected()
+}
+
+const getNextPendingWorkspaceBuildOrchestrationForUpdate = `-- name: GetNextPendingWorkspaceBuildOrchestrationForUpdate :one
+SELECT
+    wbo.id, wbo.created_at, wbo.updated_at, wbo.workspace_id, wbo.parent_build_id, wbo.child_build_id, wbo.child_transition, wbo.child_template_version_id, wbo.child_template_version_preset_id, wbo.child_rich_parameter_values, wbo.child_log_level, wbo.child_reason, wbo.attempt_count, wbo.next_retry_after, wbo.status, wbo.error
+FROM
+    workspace_build_orchestrations wbo
+    JOIN workspace_builds wb ON wbo.parent_build_id = wb.id
+    JOIN provisioner_jobs pj ON wb.job_id = pj.id
+WHERE
+    wbo.status = 'pending'
+    AND (
+        wbo.next_retry_after IS NULL
+        OR wbo.next_retry_after <= NOW()
+    )
+    -- Include all terminal parent states so pending orchestration
+    -- rows are processed and resolved even when no child build should
+    -- be created.
+    AND pj.job_status IN ('succeeded', 'failed', 'canceled')
+ORDER BY
+    wbo.created_at ASC
+LIMIT 1
+FOR UPDATE OF wbo SKIP LOCKED
+`
+
+// Must be called from within a transaction. The row lock is released
+// when the transaction ends.
+func (q *sqlQuerier) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) (WorkspaceBuildOrchestration, error) {
+	row := q.db.QueryRowContext(ctx, getNextPendingWorkspaceBuildOrchestrationForUpdate)
+	var i WorkspaceBuildOrchestration
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.WorkspaceID,
+		&i.ParentBuildID,
+		&i.ChildBuildID,
+		&i.ChildTransition,
+		&i.ChildTemplateVersionID,
+		&i.ChildTemplateVersionPresetID,
+		&i.ChildRichParameterValues,
+		&i.ChildLogLevel,
+		&i.ChildReason,
+		&i.AttemptCount,
+		&i.NextRetryAfter,
+		&i.Status,
+		&i.Error,
+	)
+	return i, err
+}
+
+const insertWorkspaceBuildOrchestration = `-- name: InsertWorkspaceBuildOrchestration :one
+INSERT INTO workspace_build_orchestrations (
+    id,
+    created_at,
+    updated_at,
+    parent_build_id,
+    workspace_id,
+    child_transition,
+    child_template_version_id,
+    child_template_version_preset_id,
+    child_rich_parameter_values,
+    child_log_level,
+    child_reason,
+    status,
+    error
+)
+VALUES (
+    $1,
+    $2,
+    $3,
+    $4,
+    (SELECT workspace_id FROM workspace_builds WHERE id = $4),
+    $5,
+    $6,
+    $7,
+    $8,
+    $9,
+    $10,
+    'pending',
+    NULL
+)
+RETURNING id, created_at, updated_at, workspace_id, parent_build_id, child_build_id, child_transition, child_template_version_id, child_template_version_preset_id, child_rich_parameter_values, child_log_level, child_reason, attempt_count, next_retry_after, status, error
+`
+
+type InsertWorkspaceBuildOrchestrationParams struct {
+	ID                           uuid.UUID           `db:"id" json:"id"`
+	CreatedAt                    time.Time           `db:"created_at" json:"created_at"`
+	UpdatedAt                    time.Time           `db:"updated_at" json:"updated_at"`
+	ParentBuildID                uuid.UUID           `db:"parent_build_id" json:"parent_build_id"`
+	ChildTransition              WorkspaceTransition `db:"child_transition" json:"child_transition"`
+	ChildTemplateVersionID       uuid.NullUUID       `db:"child_template_version_id" json:"child_template_version_id"`
+	ChildTemplateVersionPresetID uuid.NullUUID       `db:"child_template_version_preset_id" json:"child_template_version_preset_id"`
+	ChildRichParameterValues     json.RawMessage     `db:"child_rich_parameter_values" json:"child_rich_parameter_values"`
+	ChildLogLevel                string              `db:"child_log_level" json:"child_log_level"`
+	ChildReason                  NullBuildReason     `db:"child_reason" json:"child_reason"`
+}
+
+func (q *sqlQuerier) InsertWorkspaceBuildOrchestration(ctx context.Context, arg InsertWorkspaceBuildOrchestrationParams) (WorkspaceBuildOrchestration, error) {
+	row := q.db.QueryRowContext(ctx, insertWorkspaceBuildOrchestration,
+		arg.ID,
+		arg.CreatedAt,
+		arg.UpdatedAt,
+		arg.ParentBuildID,
+		arg.ChildTransition,
+		arg.ChildTemplateVersionID,
+		arg.ChildTemplateVersionPresetID,
+		arg.ChildRichParameterValues,
+		arg.ChildLogLevel,
+		arg.ChildReason,
+	)
+	var i WorkspaceBuildOrchestration
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.WorkspaceID,
+		&i.ParentBuildID,
+		&i.ChildBuildID,
+		&i.ChildTransition,
+		&i.ChildTemplateVersionID,
+		&i.ChildTemplateVersionPresetID,
+		&i.ChildRichParameterValues,
+		&i.ChildLogLevel,
+		&i.ChildReason,
+		&i.AttemptCount,
+		&i.NextRetryAfter,
+		&i.Status,
+		&i.Error,
+	)
+	return i, err
+}
+
+const updateWorkspaceBuildOrchestrationCanceledByID = `-- name: UpdateWorkspaceBuildOrchestrationCanceledByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    status = 'canceled',
+    next_retry_after = NULL,
+    error = NULL,
+    updated_at = $1
+WHERE
+    id = $2
+    AND status = 'pending'
+RETURNING id, created_at, updated_at, workspace_id, parent_build_id, child_build_id, child_transition, child_template_version_id, child_template_version_preset_id, child_rich_parameter_values, child_log_level, child_reason, attempt_count, next_retry_after, status, error
+`
+
+type UpdateWorkspaceBuildOrchestrationCanceledByIDParams struct {
+	UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
+	ID        uuid.UUID `db:"id" json:"id"`
+}
+
+func (q *sqlQuerier) UpdateWorkspaceBuildOrchestrationCanceledByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationCanceledByIDParams) (WorkspaceBuildOrchestration, error) {
+	row := q.db.QueryRowContext(ctx, updateWorkspaceBuildOrchestrationCanceledByID, arg.UpdatedAt, arg.ID)
+	var i WorkspaceBuildOrchestration
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.WorkspaceID,
+		&i.ParentBuildID,
+		&i.ChildBuildID,
+		&i.ChildTransition,
+		&i.ChildTemplateVersionID,
+		&i.ChildTemplateVersionPresetID,
+		&i.ChildRichParameterValues,
+		&i.ChildLogLevel,
+		&i.ChildReason,
+		&i.AttemptCount,
+		&i.NextRetryAfter,
+		&i.Status,
+		&i.Error,
+	)
+	return i, err
+}
+
+const updateWorkspaceBuildOrchestrationCompletedByID = `-- name: UpdateWorkspaceBuildOrchestrationCompletedByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    child_build_id = $1,
+    status = 'completed',
+    next_retry_after = NULL,
+    error = NULL,
+    updated_at = $2
+WHERE
+    id = $3
+    AND status = 'pending'
+RETURNING id, created_at, updated_at, workspace_id, parent_build_id, child_build_id, child_transition, child_template_version_id, child_template_version_preset_id, child_rich_parameter_values, child_log_level, child_reason, attempt_count, next_retry_after, status, error
+`
+
+type UpdateWorkspaceBuildOrchestrationCompletedByIDParams struct {
+	ChildBuildID uuid.NullUUID `db:"child_build_id" json:"child_build_id"`
+	UpdatedAt    time.Time     `db:"updated_at" json:"updated_at"`
+	ID           uuid.UUID     `db:"id" json:"id"`
+}
+
+func (q *sqlQuerier) UpdateWorkspaceBuildOrchestrationCompletedByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationCompletedByIDParams) (WorkspaceBuildOrchestration, error) {
+	row := q.db.QueryRowContext(ctx, updateWorkspaceBuildOrchestrationCompletedByID, arg.ChildBuildID, arg.UpdatedAt, arg.ID)
+	var i WorkspaceBuildOrchestration
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.WorkspaceID,
+		&i.ParentBuildID,
+		&i.ChildBuildID,
+		&i.ChildTransition,
+		&i.ChildTemplateVersionID,
+		&i.ChildTemplateVersionPresetID,
+		&i.ChildRichParameterValues,
+		&i.ChildLogLevel,
+		&i.ChildReason,
+		&i.AttemptCount,
+		&i.NextRetryAfter,
+		&i.Status,
+		&i.Error,
+	)
+	return i, err
+}
+
+const updateWorkspaceBuildOrchestrationFailedByID = `-- name: UpdateWorkspaceBuildOrchestrationFailedByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    status = 'failed',
+    next_retry_after = NULL,
+    error = $1,
+    updated_at = $2
+WHERE
+    id = $3
+    AND status = 'pending'
+RETURNING id, created_at, updated_at, workspace_id, parent_build_id, child_build_id, child_transition, child_template_version_id, child_template_version_preset_id, child_rich_parameter_values, child_log_level, child_reason, attempt_count, next_retry_after, status, error
+`
+
+type UpdateWorkspaceBuildOrchestrationFailedByIDParams struct {
+	Error     sql.NullString `db:"error" json:"error"`
+	UpdatedAt time.Time      `db:"updated_at" json:"updated_at"`
+	ID        uuid.UUID      `db:"id" json:"id"`
+}
+
+func (q *sqlQuerier) UpdateWorkspaceBuildOrchestrationFailedByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationFailedByIDParams) (WorkspaceBuildOrchestration, error) {
+	row := q.db.QueryRowContext(ctx, updateWorkspaceBuildOrchestrationFailedByID, arg.Error, arg.UpdatedAt, arg.ID)
+	var i WorkspaceBuildOrchestration
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.WorkspaceID,
+		&i.ParentBuildID,
+		&i.ChildBuildID,
+		&i.ChildTransition,
+		&i.ChildTemplateVersionID,
+		&i.ChildTemplateVersionPresetID,
+		&i.ChildRichParameterValues,
+		&i.ChildLogLevel,
+		&i.ChildReason,
+		&i.AttemptCount,
+		&i.NextRetryAfter,
+		&i.Status,
+		&i.Error,
+	)
+	return i, err
+}
+
+const updateWorkspaceBuildOrchestrationRetryByID = `-- name: UpdateWorkspaceBuildOrchestrationRetryByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    attempt_count = attempt_count + 1,
+    next_retry_after = CASE
+        WHEN attempt_count + 1 >= $1::int THEN NULL
+        ELSE $2::timestamptz
+    END,
+    status = CASE
+        WHEN attempt_count + 1 >= $1::int THEN 'failed'
+        ELSE status
+    END,
+    error = $3,
+    updated_at = $4
+WHERE
+    id = $5
+    AND status = 'pending'
+RETURNING id, created_at, updated_at, workspace_id, parent_build_id, child_build_id, child_transition, child_template_version_id, child_template_version_preset_id, child_rich_parameter_values, child_log_level, child_reason, attempt_count, next_retry_after, status, error
+`
+
+type UpdateWorkspaceBuildOrchestrationRetryByIDParams struct {
+	MaxAttemptCount int32          `db:"max_attempt_count" json:"max_attempt_count"`
+	NextRetryAfter  time.Time      `db:"next_retry_after" json:"next_retry_after"`
+	Error           sql.NullString `db:"error" json:"error"`
+	UpdatedAt       time.Time      `db:"updated_at" json:"updated_at"`
+	ID              uuid.UUID      `db:"id" json:"id"`
+}
+
+func (q *sqlQuerier) UpdateWorkspaceBuildOrchestrationRetryByID(ctx context.Context, arg UpdateWorkspaceBuildOrchestrationRetryByIDParams) (WorkspaceBuildOrchestration, error) {
+	row := q.db.QueryRowContext(ctx, updateWorkspaceBuildOrchestrationRetryByID,
+		arg.MaxAttemptCount,
+		arg.NextRetryAfter,
+		arg.Error,
+		arg.UpdatedAt,
+		arg.ID,
+	)
+	var i WorkspaceBuildOrchestration
+	err := row.Scan(
+		&i.ID,
+		&i.CreatedAt,
+		&i.UpdatedAt,
+		&i.WorkspaceID,
+		&i.ParentBuildID,
+		&i.ChildBuildID,
+		&i.ChildTransition,
+		&i.ChildTemplateVersionID,
+		&i.ChildTemplateVersionPresetID,
+		&i.ChildRichParameterValues,
+		&i.ChildLogLevel,
+		&i.ChildReason,
+		&i.AttemptCount,
+		&i.NextRetryAfter,
+		&i.Status,
+		&i.Error,
+	)
+	return i, err
+}
+
 const getUserWorkspaceBuildParameters = `-- name: GetUserWorkspaceBuildParameters :many
 SELECT name, value
 FROM (

coderd/database/queries/workspacebuildorchestrations.sql

@@ -0,0 +1,133 @@
+-- name: InsertWorkspaceBuildOrchestration :one
+INSERT INTO workspace_build_orchestrations (
+    id,
+    created_at,
+    updated_at,
+    parent_build_id,
+    workspace_id,
+    child_transition,
+    child_template_version_id,
+    child_template_version_preset_id,
+    child_rich_parameter_values,
+    child_log_level,
+    child_reason,
+    status,
+    error
+)
+VALUES (
+    @id,
+    @created_at,
+    @updated_at,
+    @parent_build_id,
+    (SELECT workspace_id FROM workspace_builds WHERE id = @parent_build_id),
+    @child_transition,
+    @child_template_version_id,
+    @child_template_version_preset_id,
+    @child_rich_parameter_values,
+    @child_log_level,
+    @child_reason,
+    'pending',
+    NULL
+)
+RETURNING *;
+
+-- name: GetNextPendingWorkspaceBuildOrchestrationForUpdate :one
+-- Must be called from within a transaction. The row lock is released
+-- when the transaction ends.
+SELECT
+    wbo.*
+FROM
+    workspace_build_orchestrations wbo
+    JOIN workspace_builds wb ON wbo.parent_build_id = wb.id
+    JOIN provisioner_jobs pj ON wb.job_id = pj.id
+WHERE
+    wbo.status = 'pending'
+    AND (
+        wbo.next_retry_after IS NULL
+        OR wbo.next_retry_after <= NOW()
+    )
+    -- Include all terminal parent states so pending orchestration
+    -- rows are processed and resolved even when no child build should
+    -- be created.
+    AND pj.job_status IN ('succeeded', 'failed', 'canceled')
+ORDER BY
+    wbo.created_at ASC
+LIMIT 1
+FOR UPDATE OF wbo SKIP LOCKED;
+
+-- name: UpdateWorkspaceBuildOrchestrationCompletedByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    child_build_id = @child_build_id,
+    status = 'completed',
+    next_retry_after = NULL,
+    error = NULL,
+    updated_at = @updated_at
+WHERE
+    id = @id
+    AND status = 'pending'
+RETURNING *;
+
+-- name: UpdateWorkspaceBuildOrchestrationFailedByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    status = 'failed',
+    next_retry_after = NULL,
+    error = @error,
+    updated_at = @updated_at
+WHERE
+    id = @id
+    AND status = 'pending'
+RETURNING *;
+
+-- name: UpdateWorkspaceBuildOrchestrationRetryByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    attempt_count = attempt_count + 1,
+    next_retry_after = CASE
+        WHEN attempt_count + 1 >= @max_attempt_count::int THEN NULL
+        ELSE @next_retry_after::timestamptz
+    END,
+    status = CASE
+        WHEN attempt_count + 1 >= @max_attempt_count::int THEN 'failed'
+        ELSE status
+    END,
+    error = @error,
+    updated_at = @updated_at
+WHERE
+    id = @id
+    AND status = 'pending'
+RETURNING *;
+
+-- name: UpdateWorkspaceBuildOrchestrationCanceledByID :one
+UPDATE
+    workspace_build_orchestrations
+SET
+    status = 'canceled',
+    next_retry_after = NULL,
+    error = NULL,
+    updated_at = @updated_at
+WHERE
+    id = @id
+    AND status = 'pending'
+RETURNING *;
+
+-- name: DeleteOldWorkspaceBuildOrchestrations :execrows
+WITH deletable AS (
+    SELECT
+        id
+    FROM
+        workspace_build_orchestrations
+    WHERE
+        status IN ('completed', 'failed', 'canceled')
+        AND updated_at < @before_time::timestamptz
+    ORDER BY
+        updated_at ASC
+    LIMIT @limit_count::int
+)
+DELETE FROM workspace_build_orchestrations
+USING deletable
+WHERE workspace_build_orchestrations.id = deletable.id;

coderd/database/unique_constraint.go

@@ -92,6 +92,7 @@ const (
 	UniqueTemplateVersionParametersTemplateVersionIDNameKey   UniqueConstraint = "template_version_parameters_template_version_id_name_key"        // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_name_key UNIQUE (template_version_id, name);
 	UniqueTemplateVersionPresetParametersPkey                 UniqueConstraint = "template_version_preset_parameters_pkey"                         // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_parameters_pkey PRIMARY KEY (id);
 	UniqueTemplateVersionPresetPrebuildSchedulesPkey          UniqueConstraint = "template_version_preset_prebuild_schedules_pkey"                 // ALTER TABLE ONLY template_version_preset_prebuild_schedules ADD CONSTRAINT template_version_preset_prebuild_schedules_pkey PRIMARY KEY (id);
+	UniqueTemplateVersionPresetsIDTemplateVersionIDKey        UniqueConstraint = "template_version_presets_id_template_version_id_key"             // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_id_template_version_id_key UNIQUE (id, template_version_id);
 	UniqueTemplateVersionPresetsPkey                          UniqueConstraint = "template_version_presets_pkey"                                   // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_pkey PRIMARY KEY (id);
 	UniqueTemplateVersionTerraformValuesTemplateVersionIDKey  UniqueConstraint = "template_version_terraform_values_template_version_id_key"       // ALTER TABLE ONLY template_version_terraform_values ADD CONSTRAINT template_version_terraform_values_template_version_id_key UNIQUE (template_version_id);
 	UniqueTemplateVersionVariablesTemplateVersionIDNameKey    UniqueConstraint = "template_version_variables_template_version_id_name_key"         // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_name_key UNIQUE (template_version_id, name);
@@ -131,7 +132,11 @@ const (
 	UniqueWorkspaceAppStatusesPkey                            UniqueConstraint = "workspace_app_statuses_pkey"                                     // ALTER TABLE ONLY workspace_app_statuses ADD CONSTRAINT workspace_app_statuses_pkey PRIMARY KEY (id);
 	UniqueWorkspaceAppsAgentIDSlugIndex                       UniqueConstraint = "workspace_apps_agent_id_slug_idx"                                // ALTER TABLE ONLY workspace_apps ADD CONSTRAINT workspace_apps_agent_id_slug_idx UNIQUE (agent_id, slug);
 	UniqueWorkspaceAppsPkey                                   UniqueConstraint = "workspace_apps_pkey"                                             // ALTER TABLE ONLY workspace_apps ADD CONSTRAINT workspace_apps_pkey PRIMARY KEY (id);
+	UniqueWorkspaceBuildOrchestrationsChildBuildIDKey         UniqueConstraint = "workspace_build_orchestrations_child_build_id_key"               // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_child_build_id_key UNIQUE (child_build_id);
+	UniqueWorkspaceBuildOrchestrationsParentBuildIDKey        UniqueConstraint = "workspace_build_orchestrations_parent_build_id_key"              // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_parent_build_id_key UNIQUE (parent_build_id);
+	UniqueWorkspaceBuildOrchestrationsPkey                    UniqueConstraint = "workspace_build_orchestrations_pkey"                             // ALTER TABLE ONLY workspace_build_orchestrations ADD CONSTRAINT workspace_build_orchestrations_pkey PRIMARY KEY (id);
 	UniqueWorkspaceBuildParametersWorkspaceBuildIDNameKey     UniqueConstraint = "workspace_build_parameters_workspace_build_id_name_key"          // ALTER TABLE ONLY workspace_build_parameters ADD CONSTRAINT workspace_build_parameters_workspace_build_id_name_key UNIQUE (workspace_build_id, name);
+	UniqueWorkspaceBuildsIDWorkspaceIDKey                     UniqueConstraint = "workspace_builds_id_workspace_id_key"                            // ALTER TABLE ONLY workspace_builds ADD CONSTRAINT workspace_builds_id_workspace_id_key UNIQUE (id, workspace_id);
 	UniqueWorkspaceBuildsJobIDKey                             UniqueConstraint = "workspace_builds_job_id_key"                                     // ALTER TABLE ONLY workspace_builds ADD CONSTRAINT workspace_builds_job_id_key UNIQUE (job_id);
 	UniqueWorkspaceBuildsPkey                                 UniqueConstraint = "workspace_builds_pkey"                                           // ALTER TABLE ONLY workspace_builds ADD CONSTRAINT workspace_builds_pkey PRIMARY KEY (id);
 	UniqueWorkspaceBuildsWorkspaceIDBuildNumberKey            UniqueConstraint = "workspace_builds_workspace_id_build_number_key"                  // ALTER TABLE ONLY workspace_builds ADD CONSTRAINT workspace_builds_workspace_id_build_number_key UNIQUE (workspace_id, build_number);

coderd/pproflabel/pproflabel.go

@@ -35,6 +35,9 @@ const (
 	// ServiceTallymanPublisher publishes usage events to coder/tallyman.
 	ServiceTallymanPublisher = "tallyman-publisher"
 	ServiceUsageEventCron    = "usage-event-cron"
+	// ServiceWorkspaceBuildOrchestrator fulfills workspace build
+	// orchestrations once their parent build reaches a terminal state.
+	ServiceWorkspaceBuildOrchestrator = "workspace-build-orchestrator"
 
 	RequestTypeTag = "coder_request_type"
 )

coderd/provisionerdserver/provisionerdserver.go

@@ -1285,6 +1285,13 @@ func (s *server) FailJob(ctx context.Context, failJob *proto.FailedJob) (*proto.
 
 		s.notifyWorkspaceBuildFailed(ctx, workspace, build)
 
+		// Wake the orchestrator before the workspace event publish
+		// below, which returns on error, so a failed UI event cannot
+		// skip the wake.
+		if err := wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, s.Pubsub); err != nil {
+			s.Logger.Warn(ctx, "failed to publish workspace build orchestration wake", slog.Error(err))
+		}
+
 		msg, err := json.Marshal(wspubsub.WorkspaceEvent{
 			Kind:        wspubsub.WorkspaceEventKindStateChange,
 			WorkspaceID: workspace.ID,
@@ -2552,6 +2559,15 @@ func (s *server) completeWorkspaceBuildJob(ctx context.Context, job database.Pro
 		}
 	}
 
+	// Wake the orchestrator before the workspace event publish below,
+	// which returns on error, so a failed UI event cannot skip the
+	// wake.
+	if err := wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, s.Pubsub); err != nil {
+		s.Logger.Warn(ctx, "failed to publish workspace build orchestration wake",
+			slog.Error(err),
+		)
+	}
+
 	msg, err := json.Marshal(wspubsub.WorkspaceEvent{
 		Kind:        wspubsub.WorkspaceEventKindStateChange,
 		WorkspaceID: workspace.ID,

coderd/rbac/object_gen.go

@@ -457,6 +457,16 @@ var (
 		Type: "workspace_agent_resource_monitor",
 	}
 
+	// ResourceWorkspaceBuildOrchestration
+	// Valid Actions
+	//  - "ActionCreate" :: create a workspace build orchestration
+	//  - "ActionDelete" :: delete a workspace build orchestration
+	//  - "ActionRead" :: read a workspace build orchestration
+	//  - "ActionUpdate" :: update a workspace build orchestration
+	ResourceWorkspaceBuildOrchestration = Object{
+		Type: "workspace_build_orchestration",
+	}
+
 	// ResourceWorkspaceDormant
 	// Valid Actions
 	//  - "ActionApplicationConnect" :: connect to workspace apps via browser
@@ -536,6 +546,7 @@ func AllResources() []Objecter {
 		ResourceWorkspace,
 		ResourceWorkspaceAgentDevcontainers,
 		ResourceWorkspaceAgentResourceMonitor,
+		ResourceWorkspaceBuildOrchestration,
 		ResourceWorkspaceDormant,
 		ResourceWorkspaceProxy,
 	}

coderd/rbac/policy/policy.go

@@ -140,6 +140,14 @@ var RBACPermissions = map[string]PermissionDefinition{
 			ActionRead:   "read and use a workspace proxy",
 		},
 	},
+	"workspace_build_orchestration": {
+		Actions: map[Action]ActionDefinition{
+			ActionCreate: "create a workspace build orchestration",
+			ActionRead:   "read a workspace build orchestration",
+			ActionUpdate: "update a workspace build orchestration",
+			ActionDelete: "delete a workspace build orchestration",
+		},
+	},
 	"license": {
 		Actions: map[Action]ActionDefinition{
 			ActionCreate: "create a license",

coderd/rbac/roles.go

@@ -585,7 +585,7 @@ func ReloadBuiltinRoles(opts *RoleOptions) {
 					// Org admins should not have workspace exec perms.
 					organizationID.String(): {
 						Org: append(
-							allPermsExcept(ResourceWorkspace, ResourceWorkspaceDormant, ResourcePrebuiltWorkspace, ResourceAssignRole, ResourceUserSecret, ResourceBoundaryUsage, ResourceBoundaryLog, ResourceAiSeat),
+							allPermsExcept(ResourceWorkspace, ResourceWorkspaceDormant, ResourcePrebuiltWorkspace, ResourceAssignRole, ResourceUserSecret, ResourceBoundaryUsage, ResourceBoundaryLog, ResourceAiSeat, ResourceWorkspaceBuildOrchestration),
 							Permissions(map[string][]policy.Action{
 								ResourceWorkspace.Type:        slice.Omit(ResourceWorkspace.AvailableActions(), policy.ActionApplicationConnect, policy.ActionSSH),
 								ResourceWorkspaceDormant.Type: {policy.ActionRead, policy.ActionDelete, policy.ActionCreate, policy.ActionUpdate, policy.ActionWorkspaceStop, policy.ActionCreateAgent, policy.ActionDeleteAgent, policy.ActionUpdateAgent},

coderd/rbac/roles_test.go

@@ -987,6 +987,15 @@ func TestRolePermissions(t *testing.T) {
 				false: {},
 			},
 		},
+		{
+			Name:     "WorkspaceBuildOrchestration",
+			Actions:  crud,
+			Resource: rbac.ResourceWorkspaceBuildOrchestration.InOrg(orgID),
+			AuthorizeMap: map[bool][]hasAuthSubjects{
+				true:  {owner},
+				false: {setOrgNotMe, setOtherOrg, memberMe, agentsAccessUser, templateAdmin, userAdmin, orgWorkspaceAccessUser},
+			},
+		},
 		{
 			// Any owner/admin across may access any users' preferences
 			// Members may not access other members' preferences

coderd/rbac/scopes_constants_gen.go

@@ -164,6 +164,10 @@ const (
 	ScopeWorkspaceAgentResourceMonitorCreate ScopeName = "workspace_agent_resource_monitor:create"
 	ScopeWorkspaceAgentResourceMonitorRead   ScopeName = "workspace_agent_resource_monitor:read"
 	ScopeWorkspaceAgentResourceMonitorUpdate ScopeName = "workspace_agent_resource_monitor:update"
+	ScopeWorkspaceBuildOrchestrationCreate   ScopeName = "workspace_build_orchestration:create"
+	ScopeWorkspaceBuildOrchestrationDelete   ScopeName = "workspace_build_orchestration:delete"
+	ScopeWorkspaceBuildOrchestrationRead     ScopeName = "workspace_build_orchestration:read"
+	ScopeWorkspaceBuildOrchestrationUpdate   ScopeName = "workspace_build_orchestration:update"
 	ScopeWorkspaceDormantApplicationConnect  ScopeName = "workspace_dormant:application_connect"
 	ScopeWorkspaceDormantCreate              ScopeName = "workspace_dormant:create"
 	ScopeWorkspaceDormantCreateAgent         ScopeName = "workspace_dormant:create_agent"
@@ -348,6 +352,10 @@ func (e ScopeName) Valid() bool {
 		ScopeWorkspaceAgentResourceMonitorCreate,
 		ScopeWorkspaceAgentResourceMonitorRead,
 		ScopeWorkspaceAgentResourceMonitorUpdate,
+		ScopeWorkspaceBuildOrchestrationCreate,
+		ScopeWorkspaceBuildOrchestrationDelete,
+		ScopeWorkspaceBuildOrchestrationRead,
+		ScopeWorkspaceBuildOrchestrationUpdate,
 		ScopeWorkspaceDormantApplicationConnect,
 		ScopeWorkspaceDormantCreate,
 		ScopeWorkspaceDormantCreateAgent,
@@ -533,6 +541,10 @@ func AllScopeNameValues() []ScopeName {
 		ScopeWorkspaceAgentResourceMonitorCreate,
 		ScopeWorkspaceAgentResourceMonitorRead,
 		ScopeWorkspaceAgentResourceMonitorUpdate,
+		ScopeWorkspaceBuildOrchestrationCreate,
+		ScopeWorkspaceBuildOrchestrationDelete,
+		ScopeWorkspaceBuildOrchestrationRead,
+		ScopeWorkspaceBuildOrchestrationUpdate,
 		ScopeWorkspaceDormantApplicationConnect,
 		ScopeWorkspaceDormantCreate,
 		ScopeWorkspaceDormantCreateAgent,

coderd/workspacebuilds.go

@@ -375,6 +375,30 @@ func (api *API) postWorkspaceBuildsInternal(
 	codersdk.WorkspaceBuild,
 	error,
 ) {
+	if err := validateCreateWorkspaceBuildOnSuccess(createBuild); err != nil {
+		return codersdk.WorkspaceBuild{}, err
+	}
+
+	var childParameterValuesJSON json.RawMessage
+	if createBuild.OnSuccess != nil {
+		childParameterValues := createBuild.OnSuccess.RichParameterValues
+		if childParameterValues == nil {
+			childParameterValues = []codersdk.WorkspaceBuildParameter{}
+		}
+
+		var err error
+		childParameterValuesJSON, err = json.Marshal(childParameterValues)
+		if err != nil {
+			return codersdk.WorkspaceBuild{}, httperror.NewResponseError(
+				http.StatusInternalServerError,
+				codersdk.Response{
+					Message: "Internal error preparing follow-up workspace build parameters",
+					Detail:  err.Error(),
+				},
+			)
+		}
+	}
+
 	transition := database.WorkspaceTransition(createBuild.Transition)
 	builder := wsbuilder.New(workspace, transition, *api.BuildUsageChecker.Load()).
 		Initiator(apiKey.UserID).
@@ -481,7 +505,62 @@ func (api *API) postWorkspaceBuildsInternal(
 			},
 			workspaceBuildBaggage,
 		)
-		return err
+		if err != nil {
+			return err
+		}
+
+		if createBuild.OnSuccess != nil {
+			onSuccessReq := createBuild.OnSuccess
+			// Reuse the parent build's timestamps so the orchestration row and
+			// the parent build agree on created_at.
+			_, err = tx.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+				ID:              uuid.New(),
+				CreatedAt:       workspaceBuild.CreatedAt,
+				UpdatedAt:       workspaceBuild.UpdatedAt,
+				ParentBuildID:   workspaceBuild.ID,
+				ChildTransition: database.WorkspaceTransition(onSuccessReq.Transition),
+				ChildTemplateVersionID: uuid.NullUUID{
+					UUID:  onSuccessReq.TemplateVersionID,
+					Valid: onSuccessReq.TemplateVersionID != uuid.Nil,
+				},
+				ChildTemplateVersionPresetID: uuid.NullUUID{
+					UUID:  onSuccessReq.TemplateVersionPresetID,
+					Valid: onSuccessReq.TemplateVersionPresetID != uuid.Nil,
+				},
+				ChildRichParameterValues: childParameterValuesJSON,
+				ChildLogLevel:            string(createBuild.LogLevel),
+				ChildReason: database.NullBuildReason{
+					BuildReason: database.BuildReason(createBuild.Reason),
+					Valid:       createBuild.Reason != "",
+				},
+			})
+			if err != nil {
+				if dbauthz.IsNotAuthorizedError(err) {
+					detail := "Queuing the follow-up workspace build requires permission to start the workspace."
+					if onSuccessReq.TemplateVersionID != uuid.Nil {
+						detail = "Pinning a template version on the follow-up build requires template update permission. Omit template_version_id to use the active version."
+					}
+					return httperror.NewResponseError(http.StatusForbidden, codersdk.Response{
+						Message: "Unauthorized to queue follow-up workspace build.",
+						Detail:  detail,
+					})
+				}
+				api.Logger.Error(ctx, "failed to queue follow-up workspace build",
+					slog.F("workspace_id", workspace.ID),
+					slog.F("workspace_build_id", workspaceBuild.ID),
+					slog.Error(err),
+				)
+				return httperror.NewResponseError(
+					http.StatusInternalServerError,
+					codersdk.Response{
+						Message: "Internal error queueing follow-up workspace build",
+						Detail:  err.Error(),
+					},
+				)
+			}
+		}
+
+		return nil
 	}, nil)
 	if err != nil {
 		return codersdk.WorkspaceBuild{}, err
@@ -584,6 +663,48 @@ func (api *API) postWorkspaceBuildsInternal(
 	return apiBuild, nil
 }
 
+// validateCreateWorkspaceBuildOnSuccess enforces the subset of build options
+// that currently has well-defined stop-then-start semantics.
+func validateCreateWorkspaceBuildOnSuccess(createBuild codersdk.CreateWorkspaceBuildRequest) error {
+	onSuccess := createBuild.OnSuccess
+	if onSuccess == nil {
+		return nil
+	}
+
+	if createBuild.Transition != codersdk.WorkspaceTransitionStop {
+		return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{
+			Message: "OnSuccess is only permitted when stopping a workspace.",
+		})
+	}
+	if onSuccess.Transition != codersdk.WorkspaceTransitionStart {
+		return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{
+			Message: "OnSuccess transition must be start.",
+		})
+	}
+	if createBuild.DryRun {
+		return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{
+			Message: "OnSuccess cannot be set alongside DryRun.",
+		})
+	}
+	if createBuild.Orphan {
+		return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{
+			Message: "OnSuccess cannot be set alongside Orphan.",
+		})
+	}
+	if len(createBuild.ProvisionerState) > 0 {
+		return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{
+			Message: "OnSuccess cannot be set alongside ProvisionerState.",
+		})
+	}
+	if onSuccess.TemplateVersionPresetID != uuid.Nil && onSuccess.TemplateVersionID == uuid.Nil {
+		return httperror.NewResponseError(http.StatusBadRequest, codersdk.Response{
+			Message: "OnSuccess TemplateVersionPresetID requires TemplateVersionID.",
+		})
+	}
+
+	return nil
+}
+
 func (api *API) notifyWorkspaceUpdated(
 	ctx context.Context,
 	initiatorID uuid.UUID,
@@ -765,6 +886,10 @@ func (api *API) patchCancelWorkspaceBuild(rw http.ResponseWriter, r *http.Reques
 		Kind:        wspubsub.WorkspaceEventKindStateChange,
 		WorkspaceID: workspace.ID,
 	})
+	err = wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, api.Pubsub)
+	if err != nil {
+		api.Logger.Warn(ctx, "failed to publish workspace build orchestration wake", slog.Error(err))
+	}
 
 	// Publish workspace build update to the all builds channel if the experiment is enabled.
 	if api.Experiments.Enabled(codersdk.ExperimentWorkspaceBuildUpdates) {

coderd/workspacebuilds_on_success_test.go

@@ -0,0 +1,840 @@
+package coderd_test
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/provisioner/echo"
+	"github.com/coder/coder/v2/provisionersdk/proto"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestPostWorkspaceBuildsOnSuccessRestart(t *testing.T) {
+	t.Parallel()
+
+	const paramName = "foo"
+
+	// GIVEN: a running workspace with an existing rich parameter value.
+	deploymentValues := coderdtest.DeploymentValues(t)
+	deploymentValues.EnableTerraformDebugMode = true
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+		DeploymentValues:         deploymentValues,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID,
+		echoResponsesWithRichParameter(paramName, echoResponseOptions{
+			blockStopApply: false,
+		}),
+	)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID, func(request *codersdk.CreateWorkspaceRequest) {
+		request.RichParameterValues = []codersdk.WorkspaceBuildParameter{
+			{Name: paramName, Value: "bar"},
+		}
+	})
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: a stop build is created with an on_success start build.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	user, err := client.User(ctx, codersdk.Me)
+	require.NoError(t, err)
+
+	stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		Reason:     codersdk.CreateWorkspaceBuildReasonCLI,
+		LogLevel:   codersdk.ProvisionerLogLevelDebug,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: template.ActiveVersionID,
+			RichParameterValues: []codersdk.WorkspaceBuildParameter{
+				{Name: paramName, Value: "baz"},
+			},
+		},
+	})
+	require.NoError(t, err)
+	require.Equal(t, codersdk.WorkspaceTransitionStop, stopBuild.Transition)
+	require.Equal(t, codersdk.BuildReasonCLI, stopBuild.Reason)
+
+	// THEN: the server persists the child start build intent.
+	orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+	require.NoError(t, err)
+	require.Equal(t, "pending", orchestration.Status)
+	require.Equal(t, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransition(orchestration.ChildTransition))
+	require.True(t, orchestration.ChildTemplateVersionID.Valid)
+	require.Equal(t, template.ActiveVersionID, orchestration.ChildTemplateVersionID.UUID)
+	require.False(t, orchestration.ChildTemplateVersionPresetID.Valid)
+	require.Equal(t, string(codersdk.ProvisionerLogLevelDebug), orchestration.ChildLogLevel)
+	require.True(t, orchestration.ChildReason.Valid)
+	require.Equal(t, codersdk.BuildReasonCLI, codersdk.BuildReason(orchestration.ChildReason.BuildReason))
+
+	var childRichParameterValues []codersdk.WorkspaceBuildParameter
+	require.NoError(t, json.Unmarshal(orchestration.ChildRichParameterValues, &childRichParameterValues))
+	require.ElementsMatch(t, []codersdk.WorkspaceBuildParameter{
+		{Name: paramName, Value: "baz"},
+	}, childRichParameterValues)
+
+	// THEN: the returned parent stop build completes successfully.
+	stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status)
+
+	// THEN: the server creates and completes the child start build.
+	var childBuild codersdk.WorkspaceBuild
+	require.Eventually(t, func() bool {
+		childBuild, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+			ctx,
+			user.Username,
+			workspace.Name,
+			strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10),
+		)
+		return err == nil &&
+			childBuild.Transition == codersdk.WorkspaceTransitionStart
+	}, testutil.WaitMedium, testutil.IntervalFast)
+
+	childBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, childBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, childBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, childBuild.Status)
+	require.Equal(t, codersdk.BuildReasonCLI, childBuild.Reason)
+	require.Equal(t, template.ActiveVersionID, childBuild.TemplateVersionID)
+
+	// THEN: the child build uses the on_success parameter values.
+	params, err := client.WorkspaceBuildParameters(ctx, childBuild.ID)
+	require.NoError(t, err)
+	require.ElementsMatch(t, []codersdk.WorkspaceBuildParameter{
+		{Name: paramName, Value: "baz"},
+	}, params)
+}
+
+func TestPostWorkspaceBuildsOnSuccessTemplateVersionPreset(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace and a preset on its active template
+	// version.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	preset := dbgen.Preset(t, db, database.InsertPresetParams{
+		Name:              "on-success-preset",
+		TemplateVersionID: version.ID,
+	})
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: a stop build is created with an on_success start build
+	// that requests the preset.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	user, err := client.User(ctx, codersdk.Me)
+	require.NoError(t, err)
+
+	stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:              codersdk.WorkspaceTransitionStart,
+			TemplateVersionID:       template.ActiveVersionID,
+			TemplateVersionPresetID: preset.ID,
+		},
+	})
+	require.NoError(t, err)
+
+	// THEN: the server persists the child preset intent.
+	orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+	require.NoError(t, err)
+	require.True(t, orchestration.ChildTemplateVersionPresetID.Valid)
+	require.Equal(t, preset.ID, orchestration.ChildTemplateVersionPresetID.UUID)
+
+	stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status)
+
+	// THEN: the child start build uses the preset.
+	var childBuild codersdk.WorkspaceBuild
+	require.Eventually(t, func() bool {
+		childBuild, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+			ctx,
+			user.Username,
+			workspace.Name,
+			strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10),
+		)
+		return err == nil &&
+			childBuild.Transition == codersdk.WorkspaceTransitionStart
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	childBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, childBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, childBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, childBuild.Status)
+	require.NotNil(t, childBuild.TemplateVersionPresetID)
+	require.Equal(t, preset.ID, *childBuild.TemplateVersionPresetID)
+}
+
+func TestPostWorkspaceBuildsOnSuccessUnpinnedChildUsesActiveTemplateVersion(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace and a second completed template
+	// version that is not active yet.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, provisionerCloser, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	userClient, user := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
+
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, userClient, template.ID)
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	newVersion := coderdtest.UpdateTemplateVersion(t, client, first.OrganizationID, nil, template.ID)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, newVersion.ID)
+
+	// WHEN: a non-template-admin queues an unpinned on_success child
+	// build.
+
+	// Stop the provisioner so the parent build cannot complete before
+	// the test updates the active template version.
+	require.NoError(t, provisionerCloser.Close())
+	ctx := testutil.Context(t, testutil.WaitLong)
+	stopBuild, err := userClient.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition: codersdk.WorkspaceTransitionStart,
+		},
+	})
+	require.NoError(t, err)
+
+	// THEN: the child build remains unpinned in the orchestration row.
+	orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+	require.NoError(t, err)
+	require.False(t, orchestration.ChildTemplateVersionID.Valid, "child build should remain unpinned")
+
+	// WHEN: the active version changes before the parent succeeds.
+	coderdtest.UpdateActiveTemplateVersion(t, client, template.ID, newVersion.ID)
+	coderdtest.NewProvisionerDaemon(t, api)
+
+	stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, stopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status)
+
+	// THEN: the child build uses the active version when the
+	// orchestrator creates it.
+	var childBuild codersdk.WorkspaceBuild
+	require.Eventually(t, func() bool {
+		childBuild, err = userClient.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+			ctx,
+			user.Username,
+			workspace.Name,
+			strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10),
+		)
+		return err == nil &&
+			childBuild.Transition == codersdk.WorkspaceTransitionStart
+	}, testutil.WaitMedium, testutil.IntervalFast)
+
+	childBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, childBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, childBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, childBuild.Status)
+	require.Equal(t, newVersion.ID, childBuild.TemplateVersionID)
+}
+
+func TestPostWorkspaceBuildsOnSuccessUnpinnedChildNoParams(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace owned by a non-template-admin.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
+
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, userClient, template.ID)
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: the non-template-admin queues a stop build with an unpinned
+	// on_success start build that supplies no parameters, reason, or log level.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	stopBuild, err := userClient.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition: codersdk.WorkspaceTransitionStart,
+		},
+	})
+	// THEN: the request is permitted without template-update privileges,
+	// because no durable template version pin is requested.
+	require.NoError(t, err)
+	require.Equal(t, codersdk.WorkspaceTransitionStop, stopBuild.Transition)
+
+	// THEN: the persisted child build intent leaves the optional fields unset.
+	orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+	require.NoError(t, err)
+	require.Equal(t, "pending", orchestration.Status)
+	require.Equal(t, workspace.ID, orchestration.WorkspaceID)
+	require.Equal(t, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransition(orchestration.ChildTransition))
+	require.False(t, orchestration.ChildTemplateVersionID.Valid)
+	require.False(t, orchestration.ChildTemplateVersionPresetID.Valid)
+	require.False(t, orchestration.ChildReason.Valid)
+	require.Empty(t, orchestration.ChildLogLevel)
+
+	// THEN: nil parameters are coerced to an empty JSON array, not null, to
+	// satisfy the database CHECK constraint.
+	require.JSONEq(t, "[]", string(orchestration.ChildRichParameterValues))
+	var childRichParameterValues []codersdk.WorkspaceBuildParameter
+	require.NoError(t, json.Unmarshal(orchestration.ChildRichParameterValues, &childRichParameterValues))
+	require.Empty(t, childRichParameterValues)
+}
+
+func TestPostWorkspaceBuildsOnSuccessPinnedChildVersionRequiresTemplateUpdate(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace owned by a non-template-admin.
+	client, _ := coderdtest.NewWithDatabase(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
+	first := coderdtest.CreateFirstUser(t, client)
+	userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
+
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, userClient, template.ID)
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: the non-template-admin tries to queue a stop build with a
+	// pinned on_success child version.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	_, err := userClient.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: version.ID,
+		},
+	})
+	require.Error(t, err)
+
+	// THEN: the API rejects the durable child version pin and explains the
+	// missing template update permission.
+	var apiErr *codersdk.Error
+	require.ErrorAs(t, err, &apiErr)
+	require.Equal(t, http.StatusForbidden, apiErr.StatusCode())
+	require.Contains(t, apiErr.Response.Detail, "template update permission")
+
+	// THEN: no new workspace build is created.
+	builds, err := userClient.WorkspaceBuilds(ctx, codersdk.WorkspaceBuildsRequest{WorkspaceID: workspace.ID})
+	require.NoError(t, err)
+	require.Len(t, builds, 1)
+	require.Equal(t, initialBuild.ID, builds[0].ID)
+}
+
+func TestPostWorkspaceBuildsOnSuccessValidation(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		request     codersdk.CreateWorkspaceBuildRequest
+		wantMessage string
+	}{
+		{
+			name: "ParentMustBeStop",
+			request: codersdk.CreateWorkspaceBuildRequest{
+				Transition: codersdk.WorkspaceTransitionStart,
+				OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+					Transition: codersdk.WorkspaceTransitionStart,
+				},
+			},
+			wantMessage: "OnSuccess is only permitted when stopping a workspace.",
+		},
+		{
+			// The oneof=start struct tag on OnSuccess.Transition rejects this
+			// during httpapi.Read, before the explicit check is reached.
+			name: "ChildMustBeStart",
+			request: codersdk.CreateWorkspaceBuildRequest{
+				Transition: codersdk.WorkspaceTransitionStop,
+				OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+					Transition: codersdk.WorkspaceTransitionStop,
+				},
+			},
+			wantMessage: "Validation failed.",
+		},
+		{
+			name: "ParentDryRunRejected",
+			request: codersdk.CreateWorkspaceBuildRequest{
+				Transition: codersdk.WorkspaceTransitionStop,
+				DryRun:     true,
+				OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+					Transition: codersdk.WorkspaceTransitionStart,
+				},
+			},
+			wantMessage: "OnSuccess cannot be set alongside DryRun.",
+		},
+		{
+			name: "ParentOrphanRejected",
+			request: codersdk.CreateWorkspaceBuildRequest{
+				Transition: codersdk.WorkspaceTransitionStop,
+				Orphan:     true,
+				OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+					Transition: codersdk.WorkspaceTransitionStart,
+				},
+			},
+			wantMessage: "OnSuccess cannot be set alongside Orphan.",
+		},
+		{
+			name: "ParentProvisionerStateRejected",
+			request: codersdk.CreateWorkspaceBuildRequest{
+				Transition:       codersdk.WorkspaceTransitionStop,
+				ProvisionerState: []byte("state"),
+				OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+					Transition: codersdk.WorkspaceTransitionStart,
+				},
+			},
+			wantMessage: "OnSuccess cannot be set alongside ProvisionerState.",
+		},
+		{
+			name: "ChildPresetWithoutVersionRejected",
+			request: codersdk.CreateWorkspaceBuildRequest{
+				Transition: codersdk.WorkspaceTransitionStop,
+				OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+					Transition:              codersdk.WorkspaceTransitionStart,
+					TemplateVersionPresetID: uuid.New(),
+				},
+			},
+			wantMessage: "OnSuccess TemplateVersionPresetID requires TemplateVersionID.",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			// GIVEN: a running workspace.
+			client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
+			first := coderdtest.CreateFirstUser(t, client)
+			version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
+			coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+			template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+			workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+			coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+			// WHEN: an invalid on_success request is posted.
+			_, err := client.CreateWorkspaceBuild(testutil.Context(t, testutil.WaitLong), workspace.ID, tt.request)
+			require.Error(t, err)
+
+			// THEN: the API rejects the request before creating a build.
+			var apiErr *codersdk.Error
+			require.ErrorAs(t, err, &apiErr)
+			require.Equal(t, http.StatusBadRequest, apiErr.StatusCode())
+			require.Contains(t, apiErr.Message, tt.wantMessage)
+		})
+	}
+}
+
+// Canceling an already-running job resolves the orchestration as
+// "failed", not "canceled". This hits the same orchestrator branch as
+// TestPostWorkspaceBuildsOnSuccessParentFailed below; despite that
+// overlap, the test pins this non-obvious end-to-end behavior.
+func TestPostWorkspaceBuildsOnSuccessParentCanceledMidFlight(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace whose stop apply will block.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID,
+		echoResponsesWithRichParameter("foo", echoResponseOptions{
+			blockStopApply: true,
+		}),
+	)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: a stop build is created with an on_success start build.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	user, err := client.User(ctx, codersdk.Me)
+	require.NoError(t, err)
+
+	stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: template.ActiveVersionID,
+		},
+	})
+	require.NoError(t, err)
+	require.Equal(t, codersdk.WorkspaceTransitionStop, stopBuild.Transition)
+	require.Equal(t, codersdk.BuildReasonInitiator, stopBuild.Reason)
+
+	// WHEN: the parent stop build starts running and is canceled.
+	require.Eventually(t, func() bool {
+		var err error
+		stopBuild, err = client.WorkspaceBuild(ctx, stopBuild.ID)
+		return err == nil &&
+			stopBuild.Job.Status == codersdk.ProvisionerJobRunning
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	require.NoError(t, client.CancelWorkspaceBuild(ctx, stopBuild.ID, codersdk.CancelWorkspaceBuildParams{}))
+	require.Eventually(t, func() bool {
+		var err error
+		stopBuild, err = client.WorkspaceBuild(ctx, stopBuild.ID)
+		if err != nil {
+			return false
+		}
+		return stopBuild.Job.Status == codersdk.ProvisionerJobFailed &&
+			stopBuild.Job.Error == "canceled"
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	// THEN: the server resolves the orchestration without creating the
+	// child start build.
+	require.Eventually(t, func() bool {
+		orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+		return err == nil &&
+			orchestration.Status == "failed" &&
+			!orchestration.ChildBuildID.Valid &&
+			orchestration.Error.Valid &&
+			orchestration.Error.String == "parent workspace build failed: canceled"
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	_, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+		ctx,
+		user.Username,
+		workspace.Name,
+		strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10),
+	)
+	var apiErr *codersdk.Error
+	require.ErrorAs(t, err, &apiErr)
+	require.Equal(t, http.StatusNotFound, apiErr.StatusCode())
+}
+
+func TestPostWorkspaceBuildsOnSuccessParentFailed(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace whose stop apply will fail.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID,
+		echoResponsesWithRichParameter("foo", echoResponseOptions{
+			failStopApply: true,
+		}),
+	)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: a stop build is created with an on_success start build.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	user, err := client.User(ctx, codersdk.Me)
+	require.NoError(t, err)
+
+	stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: template.ActiveVersionID,
+		},
+	})
+	require.NoError(t, err)
+
+	// WHEN: the parent stop build fails.
+	stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobFailed, stopBuild.Job.Status)
+
+	// THEN: the server resolves the orchestration without creating the
+	// child start build.
+	require.Eventually(t, func() bool {
+		orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+		return err == nil &&
+			orchestration.Status == "failed" &&
+			!orchestration.ChildBuildID.Valid &&
+			orchestration.Error.Valid &&
+			orchestration.Error.String == "parent workspace build failed: failed!"
+	}, testutil.WaitShort, testutil.IntervalFast)
+
+	_, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+		ctx,
+		user.Username,
+		workspace.Name,
+		strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10),
+	)
+	var apiErr *codersdk.Error
+	require.ErrorAs(t, err, &apiErr)
+	require.Equal(t, http.StatusNotFound, apiErr.StatusCode())
+}
+
+func TestPostWorkspaceBuildsOnSuccessNonRetryableChildBuildFailure(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a running workspace with a rich parameter value that
+	// satisfies the template regex validation.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	first := coderdtest.CreateFirstUser(t, client)
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID,
+		echoResponsesWithRichParameter("foo", echoResponseOptions{
+			validationRegex: "^good$",
+		}),
+	)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID, func(request *codersdk.CreateWorkspaceRequest) {
+		request.RichParameterValues = []codersdk.WorkspaceBuildParameter{
+			{Name: "foo", Value: "good"},
+		}
+	})
+	initialBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, initialBuild.Status)
+
+	// WHEN: a stop build is created with an on_success child build
+	// that has an invalid rich parameter value.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	user, err := client.User(ctx, codersdk.Me)
+	require.NoError(t, err)
+
+	stopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: template.ActiveVersionID,
+			// The invalid value triggers a non-retryable child build
+			// creation error when the orchestrator processes the row.
+			RichParameterValues: []codersdk.WorkspaceBuildParameter{
+				{Name: "foo", Value: "bad"},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	stopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, stopBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusStopped, stopBuild.Status)
+
+	// THEN: the server marks the orchestration as failed without
+	// retrying or creating the child start build.
+	var orchestration database.WorkspaceBuildOrchestration
+	require.Eventually(t, func() bool {
+		orchestration, err = dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, stopBuild.ID)
+		return err == nil &&
+			orchestration.Status == "failed" &&
+			!orchestration.ChildBuildID.Valid &&
+			orchestration.AttemptCount == 0 &&
+			!orchestration.NextRetryAfter.Valid &&
+			orchestration.Error.Valid
+	}, testutil.WaitShort, testutil.IntervalFast)
+	require.Contains(t, orchestration.Error.String, "Unable to validate parameters")
+
+	_, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+		ctx,
+		user.Username,
+		workspace.Name,
+		strconv.FormatInt(int64(stopBuild.BuildNumber+1), 10),
+	)
+	var apiErr *codersdk.Error
+	require.ErrorAs(t, err, &apiErr)
+	require.Equal(t, http.StatusNotFound, apiErr.StatusCode())
+}
+
+func TestPostWorkspaceBuildsOnSuccessRetryableChildBuildFailureDoesNotBlockLaterRestart(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: two provisioners, one holding a template import job
+	// open to make the child build fail retryably while the other
+	// processes workspace builds.
+	db, ps, sqlDB := dbtestutil.NewDBWithSQLDB(t)
+	client, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
+		Database:                 db,
+		Pubsub:                   ps,
+		IncludeProvisionerDaemon: true,
+	})
+	coderdtest.NewProvisionerDaemon(t, api)
+
+	first := coderdtest.CreateFirstUser(t, client)
+	version := coderdtest.CreateTemplateVersion(t, client, first.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, first.OrganizationID, version.ID)
+
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+	startBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, startBuild.Status)
+
+	blockedVersion := coderdtest.UpdateTemplateVersion(t, client, first.OrganizationID,
+		// Without a PlanComplete response, the echo provisioner will
+		// keep the template version import job running.
+		&echo.Responses{
+			Parse: echo.ParseComplete,
+			ProvisionPlan: []*proto.Response{{
+				Type: &proto.Response_Log{
+					Log: &proto.Log{},
+				},
+			}},
+		}, template.ID,
+	)
+	coderdtest.AwaitTemplateVersionJobRunning(t, client, blockedVersion.ID)
+
+	// WHEN: a stop build is created with an on_success child build
+	// request that references a template version whose import job is
+	// still running, causing child build creation to be retried later.
+	ctx := testutil.Context(t, testutil.WaitLong)
+	badStopBuild, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: blockedVersion.ID,
+		},
+	})
+	require.NoError(t, err)
+
+	badStopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, badStopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, badStopBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusStopped, badStopBuild.Status)
+
+	// THEN: the orchestrator records a delayed retry after child
+	// build creation fails because the requested template version
+	// is still importing.
+	var badOrchestration database.WorkspaceBuildOrchestration
+	require.Eventually(t, func() bool {
+		badOrchestration, err = dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, sqlDB, badStopBuild.ID)
+		return err == nil &&
+			badOrchestration.Status == "pending" &&
+			badOrchestration.AttemptCount == 1 &&
+			badOrchestration.NextRetryAfter.Valid
+	}, testutil.WaitShort, testutil.IntervalFast)
+	require.False(t, badOrchestration.ChildBuildID.Valid)
+	require.True(t, badOrchestration.Error.Valid)
+	require.Contains(t, badOrchestration.Error.String, "template version is running")
+
+	// WHEN: a later restart uses a valid child build request.
+	goodWorkspace := coderdtest.CreateWorkspace(t, client, template.ID)
+	goodStartBuild := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, goodWorkspace.LatestBuild.ID)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, goodStartBuild.Status)
+
+	user, err := client.User(ctx, codersdk.Me)
+	require.NoError(t, err)
+
+	goodStopBuild, err := client.CreateWorkspaceBuild(ctx, goodWorkspace.ID, codersdk.CreateWorkspaceBuildRequest{
+		Transition: codersdk.WorkspaceTransitionStop,
+		OnSuccess: &codersdk.CreateWorkspaceBuildOnSuccessRequest{
+			Transition:        codersdk.WorkspaceTransitionStart,
+			TemplateVersionID: template.ActiveVersionID,
+		},
+	})
+	require.NoError(t, err)
+
+	goodStopBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, goodStopBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, goodStopBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusStopped, goodStopBuild.Status)
+
+	// THEN: the delayed retry row does not block the later orchestration.
+	var goodChildBuild codersdk.WorkspaceBuild
+	require.Eventually(t, func() bool {
+		goodChildBuild, err = client.WorkspaceBuildByUsernameAndWorkspaceNameAndBuildNumber(
+			ctx,
+			user.Username,
+			goodWorkspace.Name,
+			strconv.FormatInt(int64(goodStopBuild.BuildNumber+1), 10),
+		)
+		return err == nil &&
+			goodChildBuild.Transition == codersdk.WorkspaceTransitionStart
+	}, testutil.WaitMedium, testutil.IntervalFast)
+
+	goodChildBuild = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, goodChildBuild.ID)
+	require.Equal(t, codersdk.ProvisionerJobSucceeded, goodChildBuild.Job.Status)
+	require.Equal(t, codersdk.WorkspaceStatusRunning, goodChildBuild.Status)
+}
+
+type echoResponseOptions struct {
+	blockStopApply  bool
+	failStopApply   bool
+	validationRegex string
+}
+
+func echoResponsesWithRichParameter(paramName string, options echoResponseOptions) *echo.Responses {
+	validationError := ""
+	if options.validationRegex != "" {
+		validationError = "invalid parameter value"
+	}
+
+	responses := &echo.Responses{
+		Parse:         echo.ParseComplete,
+		ProvisionInit: echo.InitComplete,
+		ProvisionGraph: []*proto.Response{{
+			Type: &proto.Response_Graph{
+				Graph: &proto.GraphComplete{
+					Parameters: []*proto.RichParameter{{
+						Name:            paramName,
+						Type:            "string",
+						DefaultValue:    "bar",
+						Mutable:         true,
+						FormType:        proto.ParameterFormType_INPUT,
+						ValidationRegex: options.validationRegex,
+						ValidationError: validationError,
+					}},
+				},
+			},
+		}},
+		ProvisionPlan:  echo.PlanComplete,
+		ProvisionApply: echo.ApplyComplete,
+	}
+	if options.blockStopApply {
+		responses.ProvisionApplyMap = map[proto.WorkspaceTransition][]*proto.Response{
+			proto.WorkspaceTransition_START: echo.ApplyComplete,
+			proto.WorkspaceTransition_STOP: {{
+				Type: &proto.Response_Log{
+					Log: &proto.Log{},
+				},
+			}},
+		}
+	}
+	if options.failStopApply {
+		responses.ProvisionApplyMap = map[proto.WorkspaceTransition][]*proto.Response{
+			proto.WorkspaceTransition_START: echo.ApplyComplete,
+			proto.WorkspaceTransition_STOP:  echo.ApplyFailed,
+		}
+	}
+	return responses
+}

coderd/workspaces.go

@@ -3030,20 +3030,7 @@ func validWorkspaceSchedule(s *string) (sql.NullString, error) {
 }
 
 func (api *API) publishWorkspaceUpdate(ctx context.Context, ownerID uuid.UUID, event wspubsub.WorkspaceEvent) {
-	err := event.Validate()
-	if err != nil {
-		api.Logger.Warn(ctx, "invalid workspace update event",
-			slog.F("workspace_id", event.WorkspaceID),
-			slog.F("event_kind", event.Kind), slog.Error(err))
-		return
-	}
-	msg, err := json.Marshal(event)
-	if err != nil {
-		api.Logger.Warn(ctx, "failed to marshal workspace update",
-			slog.F("workspace_id", event.WorkspaceID), slog.Error(err))
-		return
-	}
-	err = api.Pubsub.Publish(wspubsub.WorkspaceEventChannel(ownerID), msg)
+	err := wspubsub.PublishWorkspaceEvent(ctx, api.Pubsub, ownerID, event)
 	if err != nil {
 		api.Logger.Warn(ctx, "failed to publish workspace update",
 			slog.F("workspace_id", event.WorkspaceID), slog.Error(err))

coderd/wsbuildorchestrator/doc.go

@@ -0,0 +1,4 @@
+// Package wsbuildorchestrator runs the background worker that
+// fulfills workspace build orchestrations once their parent build
+// reaches a terminal state.
+package wsbuildorchestrator

coderd/wsbuildorchestrator/wsbuildorchestrator.go

@@ -0,0 +1,567 @@
+package wsbuildorchestrator
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/audit"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/coderd/database/provisionerjobs"
+	"github.com/coder/coder/v2/coderd/database/pubsub"
+	"github.com/coder/coder/v2/coderd/files"
+	"github.com/coder/coder/v2/coderd/pproflabel"
+	"github.com/coder/coder/v2/coderd/wsbuilder"
+	"github.com/coder/coder/v2/coderd/wspubsub"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/quartz"
+)
+
+const (
+	subscribeMaxBackoff = 10 * time.Second
+	// Pubsub should wake the worker promptly, while occasional
+	// polling prevents missed wakes from leaving rows pending
+	// indefinitely.
+	backupPollInterval = 30 * time.Second
+	maxAttempts        = 3
+	retryDelay         = 30 * time.Second
+)
+
+// Orchestrator fulfills workspace build orchestrations after their
+// parent builds reach a terminal state.
+type Orchestrator struct {
+	logger            slog.Logger
+	db                database.Store
+	pubsub            pubsub.Pubsub
+	fileCache         *files.Cache
+	buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker]
+	deploymentValues  *codersdk.DeploymentValues
+	experiments       codersdk.Experiments
+	builderMetrics    *wsbuilder.Metrics
+	clock             quartz.Clock
+
+	wakeCh chan struct{}
+
+	// startOnce ensures the background goroutines are launched at most
+	// once, even if Start is called more than once.
+	startOnce sync.Once
+	// cancel cancels the context on all running jobs. If the ctx
+	// passed into `Start` is canceled, the jobs will also stop.
+	cancel context.CancelFunc
+	// wg ensures all job goroutines have exited before Close returns.
+	wg sync.WaitGroup
+}
+
+type Options struct {
+	Logger            slog.Logger
+	Database          database.Store
+	Pubsub            pubsub.Pubsub
+	FileCache         *files.Cache
+	BuildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker]
+	DeploymentValues  *codersdk.DeploymentValues
+	Experiments       codersdk.Experiments
+	BuilderMetrics    *wsbuilder.Metrics
+	Clock             quartz.Clock
+}
+
+// New constructs an Orchestrator. Call Start to begin processing.
+func New(opts Options) *Orchestrator {
+	clock := opts.Clock
+	if clock == nil {
+		clock = quartz.NewReal()
+	}
+	return &Orchestrator{
+		logger:            opts.Logger.Named("workspace_build_orchestrator"),
+		db:                opts.Database,
+		pubsub:            opts.Pubsub,
+		fileCache:         opts.FileCache,
+		buildUsageChecker: opts.BuildUsageChecker,
+		deploymentValues:  opts.DeploymentValues,
+		experiments:       opts.Experiments,
+		builderMetrics:    opts.BuilderMetrics,
+		clock:             clock,
+		// Keep one pending wake signal while the worker is between
+		// runs. One is enough because each run drains all ready
+		// orchestration rows.
+		wakeCh: make(chan struct{}, 1),
+	}
+}
+
+// Start launches the orchestrator's background goroutines. It is safe
+// to call more than once; only the first call has any effect. Call
+// Close to stop the goroutines and wait for their exit.
+func (o *Orchestrator) Start(ctx context.Context) {
+	o.startOnce.Do(func() {
+		ctx, o.cancel = context.WithCancel(ctx)
+		o.wg.Add(2)
+		pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceWorkspaceBuildOrchestrator, "goroutine", "subscribe"), func(ctx context.Context) {
+			defer o.wg.Done()
+			o.subscribe(ctx)
+		})
+		pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceWorkspaceBuildOrchestrator, "goroutine", "run"), func(ctx context.Context) {
+			defer o.wg.Done()
+			o.run(ctx)
+		})
+	})
+}
+
+// Close stops the orchestrator and waits for its goroutines to exit.
+func (o *Orchestrator) Close() {
+	if o.cancel != nil {
+		o.cancel()
+	}
+	o.wg.Wait()
+}
+
+func (o *Orchestrator) subscribe(ctx context.Context) {
+	eb := backoff.NewExponentialBackOff()
+	eb.MaxElapsedTime = 0
+	eb.MaxInterval = subscribeMaxBackoff
+	bkoff := backoff.WithContext(eb, ctx)
+
+	var cancelSubscribe func()
+	err := backoff.Retry(func() error {
+		cancelFn, err := o.pubsub.SubscribeWithErr(
+			wspubsub.WorkspaceBuildOrchestrationWakeChannel,
+			o.listen,
+		)
+		if err != nil {
+			o.logger.Warn(ctx, "failed to subscribe to wake channel", slog.Error(err))
+			return err
+		}
+		cancelSubscribe = cancelFn
+		return nil
+	}, bkoff)
+	if err != nil {
+		if ctx.Err() == nil {
+			o.logger.Error(ctx, "code bug: retry failed before context canceled", slog.Error(err))
+		}
+		return
+	}
+	defer cancelSubscribe()
+	o.logger.Debug(ctx, "subscribed to wake channel")
+
+	// Reconcile rows that may have become ready while the worker was
+	// not subscribed.
+	o.wake()
+
+	<-ctx.Done()
+}
+
+func (o *Orchestrator) listen(ctx context.Context, _ []byte, err error) {
+	if xerrors.Is(err, pubsub.ErrDroppedMessages) {
+		o.logger.Warn(ctx, "pubsub may have dropped wake signals")
+		o.wake()
+		return
+	}
+	if err != nil {
+		o.logger.Warn(ctx, "unhandled pubsub error", slog.Error(err))
+		return
+	}
+	o.wake()
+}
+
+func (o *Orchestrator) wake() {
+	select {
+	case o.wakeCh <- struct{}{}:
+	default:
+	}
+}
+
+func (o *Orchestrator) run(ctx context.Context) {
+	ticker := o.clock.NewTicker(backupPollInterval)
+	defer ticker.Stop()
+
+	for {
+		// wakeCh can win the select below even when ctx is canceled,
+		// so re-check here. Once canceled, do not begin another
+		// processing round.
+		if ctx.Err() != nil {
+			return
+		}
+
+		err := o.processAll(ctx)
+		if err != nil && ctx.Err() == nil {
+			o.logger.Error(ctx, "failed to process orchestrations", slog.Error(err))
+		}
+
+		select {
+		case <-o.wakeCh:
+		case <-ticker.C:
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// processAll processes all pending orchestration rows whose parent
+// builds have reached a terminal state.
+func (o *Orchestrator) processAll(ctx context.Context) error {
+	for {
+		found, err := o.processNext(ctx)
+		if err != nil {
+			return err
+		}
+		if !found {
+			return nil
+		}
+	}
+}
+
+func (o *Orchestrator) processNext(ctx context.Context) (bool, error) {
+	//nolint:gocritic // Inserting the orchestration row required
+	// authorization for the parent and child transitions. The worker
+	// uses system authority to fulfill that durable intent after the
+	// parent build completes.
+	sysCtx := dbauthz.AsSystemRestricted(ctx)
+
+	var (
+		found           bool
+		workspace       database.Workspace
+		childJob        *database.ProvisionerJob
+		orchestrationID uuid.UUID
+		childBuildErr   error
+	)
+
+	err := o.db.InTx(func(tx database.Store) error {
+		orchestration, err := tx.GetNextPendingWorkspaceBuildOrchestrationForUpdate(sysCtx)
+		if xerrors.Is(err, sql.ErrNoRows) {
+			return nil
+		}
+		if err != nil {
+			return xerrors.Errorf("get next pending workspace build orchestration: %w", err)
+		}
+
+		found = true
+		orchestrationID = orchestration.ID
+
+		// markFailed resolves the locked orchestration as failed with
+		// a message, so a row that cannot make progress does not keep
+		// blocking later ones.
+		markFailed := func(msg string) error {
+			_, err := tx.UpdateWorkspaceBuildOrchestrationFailedByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{
+				Error:     sql.NullString{String: msg, Valid: true},
+				UpdatedAt: dbtime.Now(),
+				ID:        orchestration.ID,
+			})
+			if err != nil {
+				return xerrors.Errorf("mark workspace build orchestration as failed: %w", err)
+			}
+			return nil
+		}
+
+		// parentBuild and parentJob are guaranteed to exist by
+		// foreign keys on the locked orchestration row, so an error
+		// here is unexpected and likely transient. Return it to
+		// retry, rather than resolving the orchestration as failed.
+		parentBuild, err := tx.GetWorkspaceBuildByID(sysCtx, orchestration.ParentBuildID)
+		if err != nil {
+			return xerrors.Errorf("get parent workspace build: %w", err)
+		}
+
+		parentJob, err := tx.GetProvisionerJobByID(sysCtx, parentBuild.JobID)
+		if err != nil {
+			return xerrors.Errorf("get parent provisioner job: %w", err)
+		}
+
+		// Resolve terminal parent outcomes that do not create a child
+		// build. Successful parents continue below.
+		switch parentJob.JobStatus {
+		case database.ProvisionerJobStatusSucceeded:
+		case database.ProvisionerJobStatusCanceled:
+			_, err = tx.UpdateWorkspaceBuildOrchestrationCanceledByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationCanceledByIDParams{
+				ID:        orchestration.ID,
+				UpdatedAt: dbtime.Now(),
+			})
+			if err != nil {
+				return xerrors.Errorf("mark workspace build orchestration as canceled: %w", err)
+			}
+			return nil
+		case database.ProvisionerJobStatusFailed:
+			parentFailure := "parent workspace build failed"
+			if parentJob.Error.Valid && parentJob.Error.String != "" {
+				parentFailure = fmt.Sprintf("parent workspace build failed: %s", parentJob.Error.String)
+			}
+			return markFailed(parentFailure)
+		default:
+			// This should be unreachable because the row-locking query
+			// only selects terminal parent jobs. Mark the row as failed
+			// because retrying would block later orchestrations.
+			return markFailed(fmt.Sprintf("unexpected parent job status %q", parentJob.JobStatus))
+		}
+
+		childBuildRequest, err := childBuildRequestFromOrchestration(orchestration)
+		if err != nil {
+			// Mark the row failed to avoid retrying work that cannot
+			// make progress.
+			return markFailed(err.Error())
+		}
+
+		workspace, err = tx.GetWorkspaceByID(sysCtx, parentBuild.WorkspaceID)
+		if err != nil {
+			return xerrors.Errorf("get workspace: %w", err)
+		}
+
+		// GetWorkspaceByID returns soft-deleted rows.
+		if workspace.Deleted {
+			return markFailed("workspace was deleted")
+		}
+
+		// A dormant workspace must be woken before it can start.
+		// Starting it while still dormant would leave it running but
+		// still subject to deleting_at, which could auto-delete it.
+		if workspace.DormantAt.Valid {
+			return markFailed("workspace is dormant")
+		}
+
+		childBuild, provisionerJob, err := o.createBuild(sysCtx, tx, workspace, parentBuild.InitiatorID, childBuildRequest)
+		if err != nil {
+			// Carry the builder error out of the transaction; the
+			// fail-vs-retry decision runs after the rollback.
+			childBuildErr = err
+			return xerrors.Errorf("create child workspace build: %w", err)
+		}
+		childJob = provisionerJob
+
+		_, err = tx.UpdateWorkspaceBuildOrchestrationCompletedByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationCompletedByIDParams{
+			ChildBuildID: uuid.NullUUID{
+				UUID:  childBuild.ID,
+				Valid: true,
+			},
+			UpdatedAt: dbtime.Now(),
+			ID:        orchestration.ID,
+		})
+		if err != nil {
+			return xerrors.Errorf("complete workspace build orchestration: %w", err)
+		}
+
+		return nil
+	}, nil)
+	if err != nil {
+		if !found {
+			// A persistent error here blocks the whole queue, but
+			// that is systemic, not a poison row. Surface for retry.
+			return false, err
+		}
+
+		if ctx.Err() != nil {
+			// On shutdown, don't resolve or log it as unexpected
+			// error below.
+			return false, err
+		}
+
+		// A row was locked but processing failed. Resolve so it does
+		// not stay pending and block newer orchestrations.
+		errMsg := err.Error()
+		failNow := false
+		if childBuildErr != nil {
+			// The child build error carries an HTTP status we can
+			// classify into retryable vs permanent.
+			errMsg = childBuildErrorMessage(childBuildErr)
+			failNow = childBuildErrorShouldFailOrchestration(childBuildErr)
+		} else {
+			o.logger.Error(ctx, "unexpected error processing orchestration",
+				slog.F("workspace_build_orchestration_id", orchestrationID),
+				slog.Error(err))
+		}
+
+		var markErr error
+		if failNow {
+			// Mark the orchestration failed so one bad row does not
+			// block later orchestrations.
+			_, markErr = o.db.UpdateWorkspaceBuildOrchestrationFailedByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationFailedByIDParams{
+				Error: sql.NullString{
+					String: errMsg,
+					Valid:  true,
+				},
+				UpdatedAt: dbtime.Now(),
+				ID:        orchestrationID,
+			})
+		} else {
+			// Back off and retry, eventually failing after maxAttempts
+			// so a persistently failing row stops blocking the queue.
+			now := dbtime.Now()
+			_, markErr = o.db.UpdateWorkspaceBuildOrchestrationRetryByID(sysCtx, database.UpdateWorkspaceBuildOrchestrationRetryByIDParams{
+				Error: sql.NullString{
+					String: errMsg,
+					Valid:  true,
+				},
+				NextRetryAfter:  now.Add(retryDelay),
+				UpdatedAt:       now,
+				ID:              orchestrationID,
+				MaxAttemptCount: maxAttempts,
+			})
+		}
+
+		if markErr != nil {
+			if xerrors.Is(markErr, sql.ErrNoRows) {
+				// This update runs after the transaction has ended, so
+				// another worker may have resolved the orchestration
+				// first. Treat that race as success because the row no
+				// longer needs processing.
+				return found, nil
+			}
+			// Preserve the original error because the orchestration row
+			// could not be updated with it.
+			return false, errors.Join(
+				err,
+				xerrors.Errorf("resolve workspace build orchestration: %w", markErr),
+			)
+		}
+
+		return found, nil
+	}
+
+	// These post-commit notifications are best-effort. The child
+	// build and provisioner job are already persisted, so missing
+	// pubsub does not corrupt state. It can delay workers or
+	// subscribers until another wake or refresh.
+	if childJob != nil {
+		if err := provisionerjobs.PostJob(o.pubsub, *childJob); err != nil {
+			o.logger.Error(ctx, "failed to post child provisioner job to pubsub",
+				slog.F("workspace_build_orchestration_id", orchestrationID),
+				slog.F("workspace_id", workspace.ID),
+				slog.Error(err),
+			)
+		}
+
+		err := wspubsub.PublishWorkspaceEvent(ctx, o.pubsub, workspace.OwnerID, wspubsub.WorkspaceEvent{
+			Kind:        wspubsub.WorkspaceEventKindStateChange,
+			WorkspaceID: workspace.ID,
+		})
+		if err != nil {
+			o.logger.Warn(ctx, "failed to publish workspace update",
+				slog.F("workspace_build_orchestration_id", orchestrationID),
+				slog.F("workspace_id", workspace.ID), slog.Error(err))
+		}
+	}
+
+	return found, nil
+}
+
+func childBuildRequestFromOrchestration(orchestration database.WorkspaceBuildOrchestration) (codersdk.CreateWorkspaceBuildRequest, error) {
+	var childParameterValues []codersdk.WorkspaceBuildParameter
+	if len(orchestration.ChildRichParameterValues) > 0 {
+		err := json.Unmarshal(orchestration.ChildRichParameterValues, &childParameterValues)
+		if err != nil {
+			return codersdk.CreateWorkspaceBuildRequest{}, xerrors.Errorf("unmarshal child rich parameter values: %w", err)
+		}
+	}
+	if childParameterValues == nil {
+		childParameterValues = []codersdk.WorkspaceBuildParameter{}
+	}
+
+	request := codersdk.CreateWorkspaceBuildRequest{
+		Transition:          codersdk.WorkspaceTransition(orchestration.ChildTransition),
+		RichParameterValues: childParameterValues,
+		LogLevel:            codersdk.ProvisionerLogLevel(orchestration.ChildLogLevel),
+	}
+
+	if orchestration.ChildTemplateVersionID.Valid {
+		request.TemplateVersionID = orchestration.ChildTemplateVersionID.UUID
+	}
+	if orchestration.ChildTemplateVersionPresetID.Valid {
+		request.TemplateVersionPresetID = orchestration.ChildTemplateVersionPresetID.UUID
+	}
+	if orchestration.ChildReason.Valid {
+		request.Reason = codersdk.CreateWorkspaceBuildReason(orchestration.ChildReason.BuildReason)
+	}
+
+	return request, nil
+}
+
+func (o *Orchestrator) createBuild(
+	ctx context.Context,
+	tx database.Store,
+	workspace database.Workspace,
+	initiatorID uuid.UUID,
+	request codersdk.CreateWorkspaceBuildRequest,
+) (*database.WorkspaceBuild, *database.ProvisionerJob, error) {
+	transition := database.WorkspaceTransition(request.Transition)
+	builder := wsbuilder.New(workspace, transition, *o.buildUsageChecker.Load()).
+		Initiator(initiatorID).
+		RichParameterValues(request.RichParameterValues).
+		LogLevel(string(request.LogLevel)).
+		DeploymentValues(o.deploymentValues).
+		Experiments(o.experiments).
+		TemplateVersionPresetID(request.TemplateVersionPresetID).
+		BuildMetrics(o.builderMetrics)
+
+	if request.TemplateVersionID != uuid.Nil {
+		builder = builder.VersionID(request.TemplateVersionID)
+	} else if transition == database.WorkspaceTransitionStart {
+		builder = builder.ActiveVersion()
+	}
+	if request.Reason != "" {
+		builder = builder.Reason(database.BuildReason(request.Reason))
+	}
+
+	workspaceBuild, provisionerJob, _, err := builder.Build(ctx, tx, o.fileCache,
+		// nil authorization function skips the builder's RBAC and
+		// config checks. The parent and child transitions were
+		// authorized when the orchestration row was inserted, and the
+		// child reuses the parent build's already-validated log
+		// level.
+		nil,
+		// The child build is created by a background worker, so there
+		// is no request IP to attach. Its initiator is still set from
+		// the parent build.
+		audit.WorkspaceBuildBaggage{},
+	)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return workspaceBuild, provisionerJob, nil
+}
+
+// childBuildErrorShouldFailOrchestration reports whether a child build
+// error should be persisted as a failed orchestration instead of retried.
+func childBuildErrorShouldFailOrchestration(err error) bool {
+	buildErr, ok := errors.AsType[wsbuilder.BuildError](err)
+	if !ok {
+		return false
+	}
+
+	switch buildErr.Status {
+	case http.StatusBadRequest, http.StatusForbidden, http.StatusNotFound:
+		// These statuses indicate invalid stored build input or a
+		// permission/resource state that retrying the same request
+		// will not fix.
+		return true
+	default:
+		return false
+	}
+}
+
+// childBuildErrorMessage returns the error text to be stored on the
+// orchestration row. Build errors can expose cleaner response
+// messages than Error(), which may contain only the wrapped cause.
+func childBuildErrorMessage(err error) string {
+	buildErr, ok := errors.AsType[wsbuilder.BuildError](err)
+	if !ok {
+		return err.Error()
+	}
+
+	_, response := buildErr.Response()
+	if response.Detail != "" && response.Detail != response.Message {
+		return fmt.Sprintf("%s: %s", response.Message, response.Detail)
+	}
+	if response.Message != "" {
+		return response.Message
+	}
+	return buildErr.Error()
+}

coderd/wsbuildorchestrator/wsbuildorchestrator_internal_test.go

@@ -0,0 +1,427 @@
+package wsbuildorchestrator
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog/v3/sloggers/slogtest"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/database/dbtime"
+	"github.com/coder/coder/v2/coderd/database/pubsub"
+	"github.com/coder/coder/v2/coderd/wspubsub"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m, testutil.GoleakOptions...)
+}
+
+func newTestOrchestrator(t *testing.T, db database.Store, ps pubsub.Pubsub) *Orchestrator {
+	t.Helper()
+
+	return New(Options{
+		Logger:   testutil.Logger(t),
+		Database: db,
+		Pubsub:   ps,
+	})
+}
+
+func TestWorkspaceBuildOrchestratorSubscribeQueuesWakeOnPubsub(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	ps := pubsub.NewInMemory()
+	o := newTestOrchestrator(t, nil, ps)
+
+	go o.subscribe(ctx)
+
+	// subscribe sends an initial wake after registration. Drain it so
+	// the publish below tests pubsub delivery without racing setup.
+	testutil.RequireReceive(ctx, t, o.wakeCh)
+
+	err := wspubsub.PublishWorkspaceBuildOrchestrationWake(ctx, ps)
+	require.NoError(t, err)
+
+	testutil.RequireReceive(ctx, t, o.wakeCh)
+}
+
+func TestWorkspaceBuildOrchestratorRunProcessesOnWakeAndPoll(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name string
+		// trigger causes the run loop to process another pass, either
+		// via a wake signal or by advancing past the backup poll.
+		trigger func(ctx context.Context, o *Orchestrator, mClock *quartz.Mock)
+	}{
+		{
+			name: "Wake",
+			trigger: func(_ context.Context, o *Orchestrator, _ *quartz.Mock) {
+				o.wake()
+			},
+		},
+		{
+			name: "BackupPoll",
+			trigger: func(ctx context.Context, _ *Orchestrator, mClock *quartz.Mock) {
+				mClock.Advance(backupPollInterval).MustWait(ctx)
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := testutil.Context(t, testutil.WaitShort)
+			mClock := quartz.NewMock(t)
+			store := &runStore{
+				calls: make(chan struct{}),
+			}
+			o := New(Options{
+				Logger:   testutil.Logger(t),
+				Database: store,
+				Clock:    mClock,
+			})
+
+			go o.run(ctx)
+
+			// Drain the initial pass. run() creates the backup poll
+			// ticker and then processes (sends) once before
+			// waiting. So, receiving here also ensures the ticker
+			// exists before we advance the clock.
+			testutil.RequireReceive(ctx, t, store.calls)
+
+			tc.trigger(ctx, o, mClock)
+			// Now this pass can only come from the trigger.
+			testutil.RequireReceive(ctx, t, store.calls)
+		})
+	}
+}
+
+type runStore struct {
+	database.Store
+	calls chan struct{}
+}
+
+func (s *runStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error {
+	return fn(s)
+}
+
+func (s *runStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(ctx context.Context) (
+	database.WorkspaceBuildOrchestration, error,
+) {
+	select {
+	case s.calls <- struct{}{}:
+	case <-ctx.Done():
+		return database.WorkspaceBuildOrchestration{}, ctx.Err()
+	}
+	return database.WorkspaceBuildOrchestration{}, sql.ErrNoRows
+}
+
+// Note: it overwrites parentJob's OrganizationID and Type.
+func seedPendingOrchestration(
+	ctx context.Context,
+	t *testing.T,
+	db database.Store,
+	workspaceDeleted bool,
+	parentJob database.ProvisionerJob,
+) (database.ProvisionerJob, database.WorkspaceBuild) {
+	t.Helper()
+
+	org := dbgen.Organization(t, db, database.Organization{})
+	user := dbgen.User(t, db, database.User{})
+	versionJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
+		OrganizationID: org.ID,
+		Type:           database.ProvisionerJobTypeTemplateVersionImport,
+	})
+	version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		JobID:          versionJob.ID,
+		CreatedBy:      user.ID,
+	})
+	template := dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: version.ID,
+		CreatedBy:       user.ID,
+	})
+	workspace := dbgen.Workspace(t, db, database.WorkspaceTable{
+		OwnerID:        user.ID,
+		OrganizationID: org.ID,
+		TemplateID:     template.ID,
+		Deleted:        workspaceDeleted,
+	})
+
+	parentJob.OrganizationID = org.ID
+	parentJob.Type = database.ProvisionerJobTypeWorkspaceBuild
+	job := dbgen.ProvisionerJob(t, db, nil, parentJob)
+	parentBuild := dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
+		WorkspaceID:       workspace.ID,
+		TemplateVersionID: version.ID,
+		JobID:             job.ID,
+		Transition:        database.WorkspaceTransitionStop,
+		Reason:            database.BuildReasonInitiator,
+	})
+
+	now := dbtime.Now()
+	_, err := db.InsertWorkspaceBuildOrchestration(ctx, database.InsertWorkspaceBuildOrchestrationParams{
+		ID:                       uuid.New(),
+		CreatedAt:                now,
+		UpdatedAt:                now,
+		ParentBuildID:            parentBuild.ID,
+		ChildTransition:          database.WorkspaceTransitionStart,
+		ChildRichParameterValues: json.RawMessage("[]"),
+	})
+	require.NoError(t, err)
+
+	return job, parentBuild
+}
+
+// succeededJob returns a provisioner job in the succeeded state.
+func succeededJob() database.ProvisionerJob {
+	now := dbtime.Now()
+	return database.ProvisionerJob{
+		StartedAt:   sql.NullTime{Time: now, Valid: true},
+		CompletedAt: sql.NullTime{Time: now, Valid: true},
+	}
+}
+
+// A succeeded parent whose workspace can no longer be started
+// (deleted or dormant) must fail the orchestration without creating a
+// child build.
+func TestWorkspaceBuildOrchestratorFailsForUnstartableWorkspace(t *testing.T) {
+	t.Parallel()
+
+	cases := []struct {
+		name string
+		// seed builds a pending orchestration whose workspace cannot
+		// start, and returns its parent build.
+		seed      func(ctx context.Context, t *testing.T, db database.Store) database.WorkspaceBuild
+		wantError string
+	}{
+		{
+			name: "Deleted",
+			seed: func(ctx context.Context, t *testing.T, db database.Store) database.WorkspaceBuild {
+				job, build := seedPendingOrchestration(ctx, t, db, true, succeededJob())
+				require.Equal(t, database.ProvisionerJobStatusSucceeded, job.JobStatus)
+				return build
+			},
+			wantError: "workspace was deleted",
+		},
+		{
+			name: "Dormant",
+			seed: func(ctx context.Context, t *testing.T, db database.Store) database.WorkspaceBuild {
+				job, build := seedPendingOrchestration(ctx, t, db, false, succeededJob())
+				require.Equal(t, database.ProvisionerJobStatusSucceeded, job.JobStatus)
+				_, err := db.UpdateWorkspaceDormantDeletingAt(ctx, database.UpdateWorkspaceDormantDeletingAtParams{
+					ID:        build.WorkspaceID,
+					DormantAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
+				})
+				require.NoError(t, err)
+				return build
+			},
+			wantError: "workspace is dormant",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			// GIVEN: a pending orchestration whose workspace cannot start.
+			ctx := testutil.Context(t, testutil.WaitShort)
+			db, _, rawDB := dbtestutil.NewDBWithSQLDB(t)
+			parentBuild := tc.seed(ctx, t, db)
+
+			o := newTestOrchestrator(t, db, nil)
+
+			// WHEN: the orchestrator processes the row.
+			found, err := o.processNext(ctx)
+			require.NoError(t, err)
+			require.True(t, found)
+
+			// THEN: the orchestration resolves as failed without creating
+			// a child build.
+			orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuild.ID)
+			require.NoError(t, err)
+			require.Equal(t, "failed", orchestration.Status)
+			require.False(t, orchestration.ChildBuildID.Valid)
+			require.True(t, orchestration.Error.Valid)
+			require.Equal(t, tc.wantError, orchestration.Error.String)
+		})
+	}
+}
+
+// If a pending parent build is canceled before any provisioner
+// acquires it, the orchestration resolves as canceled, with no error
+// and without creating a child build.
+func TestWorkspaceBuildOrchestratorCancelsForCanceledParent(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a workspace whose parent stop build was canceled
+	// (without a provisioner acquiring it) and a pending
+	// orchestration to start it.
+	ctx := testutil.Context(t, testutil.WaitShort)
+	db, _, rawDB := dbtestutil.NewDBWithSQLDB(t)
+
+	now := dbtime.Now()
+	parentJob, parentBuild := seedPendingOrchestration(ctx, t, db, false, database.ProvisionerJob{
+		CanceledAt:  sql.NullTime{Time: now, Valid: true},
+		CompletedAt: sql.NullTime{Time: now, Valid: true},
+	})
+	require.Equal(t, database.ProvisionerJobStatusCanceled, parentJob.JobStatus)
+
+	o := newTestOrchestrator(t, db, nil)
+
+	// WHEN: the orchestrator processes the row.
+	found, err := o.processNext(ctx)
+	require.NoError(t, err)
+	require.True(t, found)
+
+	// THEN: the orchestration resolves as canceled, with no error and
+	// without creating a child build.
+	orchestration, err := dbtestutil.GetWorkspaceBuildOrchestrationByParentBuildID(ctx, rawDB, parentBuild.ID)
+	require.NoError(t, err)
+	require.Equal(t, "canceled", orchestration.Status)
+	require.False(t, orchestration.ChildBuildID.Valid)
+	require.False(t, orchestration.Error.Valid)
+}
+
+// emptyStore reports no pending orchestrations so the run loop stays
+// idle.
+type emptyStore struct {
+	database.Store
+}
+
+func (s emptyStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error {
+	return fn(s)
+}
+
+func (emptyStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(context.Context) (
+	database.WorkspaceBuildOrchestration, error,
+) {
+	return database.WorkspaceBuildOrchestration{}, sql.ErrNoRows
+}
+
+func TestWorkspaceBuildOrchestratorCloseStopsGoroutines(t *testing.T) {
+	t.Parallel()
+
+	o := New(Options{
+		Logger:   testutil.Logger(t),
+		Database: emptyStore{},
+		Pubsub:   pubsub.NewInMemory(),
+	})
+	o.Start(context.Background())
+
+	// Close blocks on wg.Wait, so it returns only once both
+	// background goroutines have exited. A timeout here means a
+	// goroutine never exited after cancellation, leaving Close
+	// blocked.
+	closed := make(chan struct{})
+	go func() {
+		o.Close()
+		close(closed)
+	}()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	select {
+	case <-ctx.Done():
+		t.Fatal("Close did not stop background goroutines")
+	case <-closed:
+	}
+}
+
+// lookupErrorStore returns a pending orchestration, then fails the
+// parent provisioner job lookup. This exercises the non-child-build
+// error path in processNext and captures the resulting retry update.
+type lookupErrorStore struct {
+	database.Store
+	orchestrationID uuid.UUID
+	jobErr          error
+
+	retryCalled bool
+	retryParams database.UpdateWorkspaceBuildOrchestrationRetryByIDParams
+}
+
+func (s *lookupErrorStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error {
+	return fn(s)
+}
+
+func (s *lookupErrorStore) GetNextPendingWorkspaceBuildOrchestrationForUpdate(context.Context) (
+	database.WorkspaceBuildOrchestration, error,
+) {
+	return database.WorkspaceBuildOrchestration{
+		ID:            s.orchestrationID,
+		ParentBuildID: uuid.New(),
+	}, nil
+}
+
+func (*lookupErrorStore) GetWorkspaceBuildByID(_ context.Context, id uuid.UUID) (
+	database.WorkspaceBuild, error,
+) {
+	return database.WorkspaceBuild{ID: id, JobID: uuid.New()}, nil
+}
+
+func (s *lookupErrorStore) GetProvisionerJobByID(context.Context, uuid.UUID) (
+	database.ProvisionerJob, error,
+) {
+	return database.ProvisionerJob{}, s.jobErr
+}
+
+func (s *lookupErrorStore) UpdateWorkspaceBuildOrchestrationRetryByID(
+	_ context.Context,
+	arg database.UpdateWorkspaceBuildOrchestrationRetryByIDParams,
+) (database.WorkspaceBuildOrchestration, error) {
+	s.retryCalled = true
+	s.retryParams = arg
+	return database.WorkspaceBuildOrchestration{}, nil
+}
+
+// TestWorkspaceBuildOrchestratorRetriesUnexpectedError verifies that
+// an unexpected error while processing a row makes processNext
+// request a bounded retry rather than surfacing the error, which
+// would leave the row pending and block newer orchestrations.
+func TestWorkspaceBuildOrchestratorRetriesUnexpectedError(t *testing.T) {
+	t.Parallel()
+
+	// GIVEN: a store that returns a pending orchestration, then fails
+	// the parent provisioner job lookup with an unexpected
+	// (non-child-build) error.
+	store := &lookupErrorStore{
+		orchestrationID: uuid.New(),
+		jobErr:          xerrors.New("boom"),
+	}
+	o := New(Options{
+		// The unexpected-error path logs at error level by design, so
+		// tolerate it here instead of failing via slogtest.
+		Logger:   slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}),
+		Database: store,
+		Pubsub:   pubsub.NewInMemory(),
+	})
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	// WHEN: the orchestrator processes the row.
+	found, err := o.processNext(ctx)
+
+	// THEN: processNext requests a bounded retry (next_retry_after
+	// and maxAttempts passed) and returns without error, instead of
+	// surfacing the error, which would leave the row pending.
+	require.NoError(t, err)
+	require.True(t, found)
+	require.True(t, store.retryCalled)
+	require.Equal(t, store.orchestrationID, store.retryParams.ID)
+	require.Equal(t, int32(maxAttempts), store.retryParams.MaxAttemptCount)
+	require.False(t, store.retryParams.NextRetryAfter.IsZero())
+	require.True(t, store.retryParams.Error.Valid)
+	require.Contains(t, store.retryParams.Error.String, "boom")
+}

coderd/wspubsub/wspubsub.go

@@ -17,6 +17,10 @@ import (
 // creating N separate subscriptions.
 const AllWorkspaceEventChannel = "workspace_updates:all"
 
+// WorkspaceBuildOrchestrationWakeChannel wakes the internal worker that
+// processes pending workspace build orchestration rows.
+const WorkspaceBuildOrchestrationWakeChannel = "workspace_build_orchestrations:wake"
+
 // HandleWorkspaceBuildUpdate wraps a callback to parse WorkspaceBuildUpdate
 // messages from the pubsub.
 func HandleWorkspaceBuildUpdate(cb func(ctx context.Context, payload codersdk.WorkspaceBuildUpdate, err error)) func(ctx context.Context, message []byte, err error) {
@@ -48,12 +52,38 @@ func PublishWorkspaceBuildUpdate(_ context.Context, ps pubsub.Pubsub, update cod
 	return nil
 }
 
+// PublishWorkspaceBuildOrchestrationWake wakes coderd instances that can
+// process pending workspace build orchestration rows. Call this after any
+// workspace build reaches a terminal state: succeeded, failed, or canceled.
+func PublishWorkspaceBuildOrchestrationWake(_ context.Context, ps pubsub.Pubsub) error {
+	if err := ps.Publish(WorkspaceBuildOrchestrationWakeChannel, []byte("{}")); err != nil {
+		return xerrors.Errorf("publish workspace build orchestration wake: %w", err)
+	}
+	return nil
+}
+
 // WorkspaceEventChannel can be used to subscribe to events for
 // workspaces owned by the provided user ID.
 func WorkspaceEventChannel(ownerID uuid.UUID) string {
 	return fmt.Sprintf("workspace_owner:%s", ownerID)
 }
 
+// PublishWorkspaceEvent validates and publishes a workspace event to
+// the owner's event channel.
+func PublishWorkspaceEvent(_ context.Context, ps pubsub.Pubsub, ownerID uuid.UUID, event WorkspaceEvent) error {
+	if err := event.Validate(); err != nil {
+		return xerrors.Errorf("validate workspace event: %w", err)
+	}
+	msg, err := json.Marshal(event)
+	if err != nil {
+		return xerrors.Errorf("marshal workspace event: %w", err)
+	}
+	if err := ps.Publish(WorkspaceEventChannel(ownerID), msg); err != nil {
+		return xerrors.Errorf("publish workspace event: %w", err)
+	}
+	return nil
+}
+
 func HandleWorkspaceEvent(cb func(ctx context.Context, payload WorkspaceEvent, err error)) func(ctx context.Context, message []byte, err error) {
 	return func(ctx context.Context, message []byte, err error) {
 		if err != nil {