feat: use proto streams to increase maximum module files payload (#18268)

This PR implements protobuf streaming to handle large module files by:
1. **Streaming large payloads**: When module files exceed the 4MB limit,
they're streamed in chunks using a new UploadFile RPC method
2. **Database storage**: Streamed files are stored in the database and
referenced by hash for deduplication
3. **Backward compatibility**: Small module files continue using the
existing direct payload method

provisionersdk/session.go

@@ -17,6 +17,9 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
+	"github.com/coder/coder/v2/codersdk/drpcsdk"
+
+	protobuf "google.golang.org/protobuf/proto"
 
 	"github.com/coder/coder/v2/provisionersdk/proto"
 )
@@ -161,6 +164,33 @@ func (s *Session) handleRequests() error {
 				return err
 			}
 			resp.Type = &proto.Response_Plan{Plan: complete}
+
+			if protobuf.Size(resp) > drpcsdk.MaxMessageSize {
+				// It is likely the modules that is pushing the message size over the limit.
+				// Send the modules over a stream of messages instead.
+				s.Logger.Info(s.Context(), "plan response too large, sending modules as stream",
+					slog.F("size_bytes", len(complete.ModuleFiles)),
+				)
+				dataUp, chunks := proto.BytesToDataUpload(proto.DataUploadType_UPLOAD_TYPE_MODULE_FILES, complete.ModuleFiles)
+
+				complete.ModuleFiles = nil // sent over the stream
+				complete.ModuleFilesHash = dataUp.DataHash
+				resp.Type = &proto.Response_Plan{Plan: complete}
+
+				err := s.stream.Send(&proto.Response{Type: &proto.Response_DataUpload{DataUpload: dataUp}})
+				if err != nil {
+					complete.Error = fmt.Sprintf("send data upload: %s", err.Error())
+				} else {
+					for i, chunk := range chunks {
+						err := s.stream.Send(&proto.Response{Type: &proto.Response_ChunkPiece{ChunkPiece: chunk}})
+						if err != nil {
+							complete.Error = fmt.Sprintf("send data piece upload %d/%d: %s", i, dataUp.Chunks, err.Error())
+							break
+						}
+					}
+				}
+			}
+
 			if complete.Error == "" {
 				planned = true
 			}