feat: use proto streams to increase maximum module files payload (#18268)

This PR implements protobuf streaming to handle large module files by:
1. **Streaming large payloads**: When module files exceed the 4MB limit,
they're streamed in chunks using a new UploadFile RPC method
2. **Database storage**: Streamed files are stored in the database and
referenced by hash for deduplication
3. **Backward compatibility**: Small module files continue using the
existing direct payload method

provisioner/terraform/executor.go

@@ -19,13 +19,11 @@ import (
 	tfjson "github.com/hashicorp/terraform-json"
 	"go.opentelemetry.io/otel/attribute"
 	"golang.org/x/xerrors"
-	protobuf "google.golang.org/protobuf/proto"
 
 	"cdr.dev/slog"
 
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/tracing"
-	"github.com/coder/coder/v2/codersdk/drpcsdk"
 	"github.com/coder/coder/v2/provisionersdk/proto"
 )
 
@@ -260,13 +258,15 @@ func getStateFilePath(workdir string) string {
 }
 
 // revive:disable-next-line:flag-parameter
-func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr logSink, metadata *proto.Metadata) (*proto.PlanComplete, error) {
+func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr logSink, req *proto.PlanRequest) (*proto.PlanComplete, error) {
 	ctx, span := e.server.startTrace(ctx, tracing.FuncName())
 	defer span.End()
 
 	e.mut.Lock()
 	defer e.mut.Unlock()
 
+	metadata := req.Metadata
+
 	planfilePath := getPlanFilePath(e.workdir)
 	args := []string{
 		"plan",
@@ -314,10 +314,16 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 
 	graphTimings.ingest(createGraphTimingsEvent(timingGraphComplete))
 
-	moduleFiles, err := GetModulesArchive(os.DirFS(e.workdir))
-	if err != nil {
-		// TODO: we probably want to persist this error or make it louder eventually
-		e.logger.Warn(ctx, "failed to archive terraform modules", slog.Error(err))
+	var moduleFiles []byte
+	// Skipping modules archiving is useful if the caller does not need it, eg during
+	// a workspace build. This removes some added costs of sending the modules
+	// payload back to coderd if coderd is just going to ignore it.
+	if !req.OmitModuleFiles {
+		moduleFiles, err = GetModulesArchive(os.DirFS(e.workdir))
+		if err != nil {
+			// TODO: we probably want to persist this error or make it louder eventually
+			e.logger.Warn(ctx, "failed to archive terraform modules", slog.Error(err))
+		}
 	}
 
 	// When a prebuild claim attempt is made, log a warning if a resource is due to be replaced, since this will obviate
@@ -357,11 +363,6 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l
 		ModuleFiles:           moduleFiles,
 	}
 
-	if protobuf.Size(msg) > drpcsdk.MaxMessageSize {
-		e.logger.Warn(ctx, "cannot persist terraform modules, message payload too big", slog.F("archive_size", len(msg.ModuleFiles)))
-		msg.ModuleFiles = nil
-	}
-
 	return msg, nil
 }
 

provisioner/terraform/modules.go

@@ -13,9 +13,21 @@ import (
 
 	"golang.org/x/xerrors"
 
+	"github.com/coder/coder/v2/coderd/util/xio"
 	"github.com/coder/coder/v2/provisionersdk/proto"
 )
 
+const (
+	// MaximumModuleArchiveSize limits the total size of a module archive.
+	// At some point, the user should take steps to reduce the size of their
+	// template modules, as this can lead to performance issues
+	// TODO: Determine what a reasonable limit is for modules
+	//  If we start hitting this limit, we might want to consider adding
+	//  configurable filters? Files like images could blow up the size of a
+	//  module.
+	MaximumModuleArchiveSize = 20 * 1024 * 1024 // 20MB
+)
+
 type module struct {
 	Source  string `json:"Source"`
 	Version string `json:"Version"`
@@ -85,7 +97,9 @@ func GetModulesArchive(root fs.FS) ([]byte, error) {
 
 	empty := true
 	var b bytes.Buffer
-	w := tar.NewWriter(&b)
+
+	lw := xio.NewLimitWriter(&b, MaximumModuleArchiveSize)
+	w := tar.NewWriter(lw)
 
 	for _, it := range m.Modules {
 		// Check to make sure that the module is a remote module fetched by

provisioner/terraform/provision.go

@@ -163,7 +163,7 @@ func (s *server) Plan(
 		return provisionersdk.PlanErrorf("plan vars: %s", err)
 	}
 
-	resp, err := e.plan(ctx, killCtx, env, vars, sess, request.Metadata)
+	resp, err := e.plan(ctx, killCtx, env, vars, sess, request)
 	if err != nil {
 		return provisionersdk.PlanErrorf("%s", err.Error())
 	}