feat: use proto streams to increase maximum module files payload (#18268)

This PR implements protobuf streaming to handle large module files by:
1. **Streaming large payloads**: When module files exceed the 4MB limit,
they're streamed in chunks using a new UploadFile RPC method
2. **Database storage**: Streamed files are stored in the database and
referenced by hash for deduplication
3. **Backward compatibility**: Small module files continue using the
existing direct payload method

provisionerd/runner/runner.go

@@ -1,6 +1,7 @@
 package runner
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -555,7 +556,7 @@ func (r *Runner) runTemplateImport(ctx context.Context) (*proto.CompletedJob, *p
 		CoderUrl:             r.job.GetTemplateImport().Metadata.CoderUrl,
 		WorkspaceOwnerGroups: r.job.GetTemplateImport().Metadata.WorkspaceOwnerGroups,
 		WorkspaceTransition:  sdkproto.WorkspaceTransition_START,
-	})
+	}, false)
 	if err != nil {
 		return nil, r.failedJobf("template import provision for start: %s", err)
 	}
@@ -571,7 +572,8 @@ func (r *Runner) runTemplateImport(ctx context.Context) (*proto.CompletedJob, *p
 		CoderUrl:             r.job.GetTemplateImport().Metadata.CoderUrl,
 		WorkspaceOwnerGroups: r.job.GetTemplateImport().Metadata.WorkspaceOwnerGroups,
 		WorkspaceTransition:  sdkproto.WorkspaceTransition_STOP,
-	})
+	}, true, // Modules downloaded on the start provision
+	)
 	if err != nil {
 		return nil, r.failedJobf("template import provision for stop: %s", err)
 	}
@@ -597,7 +599,10 @@ func (r *Runner) runTemplateImport(ctx context.Context) (*proto.CompletedJob, *p
 				StopModules:                stopProvision.Modules,
 				Presets:                    startProvision.Presets,
 				Plan:                       startProvision.Plan,
-				ModuleFiles:                startProvision.ModuleFiles,
+				// ModuleFiles are not on the stopProvision. So grab from the startProvision.
+				ModuleFiles: startProvision.ModuleFiles,
+				// ModuleFileHash will be populated if the file is uploaded async
+				ModuleFilesHash: []byte{},
 			},
 		},
 	}, nil
@@ -666,8 +671,8 @@ type templateImportProvision struct {
 // Performs a dry-run provision when importing a template.
 // This is used to detect resources that would be provisioned for a workspace in various states.
 // It doesn't define values for rich parameters as they're unknown during template import.
-func (r *Runner) runTemplateImportProvision(ctx context.Context, variableValues []*sdkproto.VariableValue, metadata *sdkproto.Metadata) (*templateImportProvision, error) {
-	return r.runTemplateImportProvisionWithRichParameters(ctx, variableValues, nil, metadata)
+func (r *Runner) runTemplateImportProvision(ctx context.Context, variableValues []*sdkproto.VariableValue, metadata *sdkproto.Metadata, omitModules bool) (*templateImportProvision, error) {
+	return r.runTemplateImportProvisionWithRichParameters(ctx, variableValues, nil, metadata, omitModules)
 }
 
 // Performs a dry-run provision with provided rich parameters.
@@ -677,6 +682,7 @@ func (r *Runner) runTemplateImportProvisionWithRichParameters(
 	variableValues []*sdkproto.VariableValue,
 	richParameterValues []*sdkproto.RichParameterValue,
 	metadata *sdkproto.Metadata,
+	omitModules bool,
 ) (*templateImportProvision, error) {
 	ctx, span := r.startTrace(ctx, tracing.FuncName())
 	defer span.End()
@@ -696,6 +702,7 @@ func (r *Runner) runTemplateImportProvisionWithRichParameters(
 		// Template import has no previous values
 		PreviousParameterValues: make([]*sdkproto.RichParameterValue, 0),
 		VariableValues:          variableValues,
+		OmitModuleFiles:         omitModules,
 	}}})
 	if err != nil {
 		return nil, xerrors.Errorf("start provision: %w", err)
@@ -717,11 +724,13 @@ func (r *Runner) runTemplateImportProvisionWithRichParameters(
 		}
 	}()
 
+	var moduleFilesUpload *sdkproto.DataBuilder
 	for {
 		msg, err := r.session.Recv()
 		if err != nil {
 			return nil, xerrors.Errorf("recv import provision: %w", err)
 		}
+
 		switch msgType := msg.Type.(type) {
 		case *sdkproto.Response_Log:
 			r.logProvisionerJobLog(context.Background(), msgType.Log.Level, "template import provision job logged",
@@ -735,6 +744,30 @@ func (r *Runner) runTemplateImportProvisionWithRichParameters(
 				Output:    msgType.Log.Output,
 				Stage:     stage,
 			})
+		case *sdkproto.Response_DataUpload:
+			c := msgType.DataUpload
+			if c.UploadType != sdkproto.DataUploadType_UPLOAD_TYPE_MODULE_FILES {
+				return nil, xerrors.Errorf("invalid data upload type: %q", c.UploadType)
+			}
+
+			if moduleFilesUpload != nil {
+				return nil, xerrors.New("multiple module data uploads received, only expect 1")
+			}
+
+			moduleFilesUpload, err = sdkproto.NewDataBuilder(c)
+			if err != nil {
+				return nil, xerrors.Errorf("create data builder: %w", err)
+			}
+		case *sdkproto.Response_ChunkPiece:
+			c := msgType.ChunkPiece
+			if moduleFilesUpload == nil {
+				return nil, xerrors.New("received chunk piece before module files data upload")
+			}
+
+			_, err := moduleFilesUpload.Add(c)
+			if err != nil {
+				return nil, xerrors.Errorf("module files, add chunk piece: %w", err)
+			}
 		case *sdkproto.Response_Plan:
 			c := msgType.Plan
 			if c.Error != "" {
@@ -745,11 +778,27 @@ func (r *Runner) runTemplateImportProvisionWithRichParameters(
 				return nil, xerrors.New(c.Error)
 			}
 
+			if moduleFilesUpload != nil && len(c.ModuleFiles) > 0 {
+				return nil, xerrors.New("module files were uploaded and module files were returned in the plan response. Only one of these should be set")
+			}
+
 			r.logger.Info(context.Background(), "parse dry-run provision successful",
 				slog.F("resource_count", len(c.Resources)),
 				slog.F("resources", resourceNames(c.Resources)),
 			)
 
+			moduleFilesData := c.ModuleFiles
+			if moduleFilesUpload != nil {
+				uploadData, err := moduleFilesUpload.Complete()
+				if err != nil {
+					return nil, xerrors.Errorf("module files, complete upload: %w", err)
+				}
+				moduleFilesData = uploadData
+				if !bytes.Equal(c.ModuleFilesHash, moduleFilesUpload.Hash) {
+					return nil, xerrors.Errorf("module files hash mismatch, uploaded: %x, expected: %x", moduleFilesUpload.Hash, c.ModuleFilesHash)
+				}
+			}
+
 			return &templateImportProvision{
 				Resources:             c.Resources,
 				Parameters:            c.Parameters,
@@ -757,7 +806,7 @@ func (r *Runner) runTemplateImportProvisionWithRichParameters(
 				Modules:               c.Modules,
 				Presets:               c.Presets,
 				Plan:                  c.Plan,
-				ModuleFiles:           c.ModuleFiles,
+				ModuleFiles:           moduleFilesData,
 			}, nil
 		default:
 			return nil, xerrors.Errorf("invalid message type %q received from provisioner",
@@ -810,6 +859,7 @@ func (r *Runner) runTemplateDryRun(ctx context.Context) (*proto.CompletedJob, *p
 		r.job.GetTemplateDryRun().GetVariableValues(),
 		r.job.GetTemplateDryRun().GetRichParameterValues(),
 		metadata,
+		false,
 	)
 	if err != nil {
 		return nil, r.failedJobf("run dry-run provision job: %s", err)
@@ -872,6 +922,10 @@ func (r *Runner) buildWorkspace(ctx context.Context, stage string, req *sdkproto
 				Output:    msgType.Log.Output,
 				Stage:     stage,
 			})
+		case *sdkproto.Response_DataUpload:
+			continue // Only for template imports
+		case *sdkproto.Response_ChunkPiece:
+			continue // Only for template imports
 		default:
 			// Stop looping!
 			return msg, nil
@@ -964,6 +1018,7 @@ func (r *Runner) runWorkspaceBuild(ctx context.Context) (*proto.CompletedJob, *p
 	resp, failed := r.buildWorkspace(ctx, "Planning infrastructure", &sdkproto.Request{
 		Type: &sdkproto.Request_Plan{
 			Plan: &sdkproto.PlanRequest{
+				OmitModuleFiles:         true, // Only useful for template imports
 				Metadata:                r.job.GetWorkspaceBuild().Metadata,
 				RichParameterValues:     r.job.GetWorkspaceBuild().RichParameterValues,
 				PreviousParameterValues: r.job.GetWorkspaceBuild().PreviousParameterValues,