feat: use proto streams to increase maximum module files payload (#18268)

This PR implements protobuf streaming to handle large module files by:
1. **Streaming large payloads**: When module files exceed the 4MB limit,
they're streamed in chunks using a new UploadFile RPC method
2. **Database storage**: Streamed files are stored in the database and
referenced by hash for deduplication
3. **Backward compatibility**: Small module files continue using the
existing direct payload method

site/e2e/helpers.ts

@@ -584,6 +584,7 @@ const createTemplateVersionTar = async (
 					resourceReplacements: [],
 					plan: emptyPlan,
 					moduleFiles: new Uint8Array(),
+					moduleFilesHash: new Uint8Array(),
 				},
 			};
 		});
@@ -711,6 +712,7 @@ const createTemplateVersionTar = async (
 			resourceReplacements: [],
 			plan: emptyPlan,
 			moduleFiles: new Uint8Array(),
+			moduleFilesHash: new Uint8Array(),
 			...response.plan,
 		} as PlanComplete;
 		response.plan.resources = response.plan.resources?.map(fillResource);

site/e2e/provisionerGenerated.ts

@@ -70,6 +70,17 @@ export enum TimingState {
   UNRECOGNIZED = -1,
 }
 
+export enum DataUploadType {
+  UPLOAD_TYPE_UNKNOWN = 0,
+  /**
+   * UPLOAD_TYPE_MODULE_FILES - UPLOAD_TYPE_MODULE_FILES is used to stream over terraform module files.
+   * These files are located in `.terraform/modules` and are used for dynamic
+   * parameters.
+   */
+  UPLOAD_TYPE_MODULE_FILES = 1,
+  UNRECOGNIZED = -1,
+}
+
 /** Empty indicates a successful request/response. */
 export interface Empty {
 }
@@ -394,6 +405,14 @@ export interface PlanRequest {
   variableValues: VariableValue[];
   externalAuthProviders: ExternalAuthProvider[];
   previousParameterValues: RichParameterValue[];
+  /**
+   * If true, the provisioner can safely assume the caller does not need the
+   * module files downloaded by the `terraform init` command.
+   * Ideally this boolean would be flipped in its truthy value, however for
+   * backwards compatibility reasons, the zero value should be the previous
+   * behavior of downloading the module files.
+   */
+  omitModuleFiles: boolean;
 }
 
 /** PlanComplete indicates a request to plan completed. */
@@ -408,6 +427,7 @@ export interface PlanComplete {
   plan: Uint8Array;
   resourceReplacements: ResourceReplacement[];
   moduleFiles: Uint8Array;
+  moduleFilesHash: Uint8Array;
 }
 
 /**
@@ -455,6 +475,32 @@ export interface Response {
   parse?: ParseComplete | undefined;
   plan?: PlanComplete | undefined;
   apply?: ApplyComplete | undefined;
+  dataUpload?: DataUpload | undefined;
+  chunkPiece?: ChunkPiece | undefined;
+}
+
+export interface DataUpload {
+  uploadType: DataUploadType;
+  /**
+   * data_hash is the sha256 of the payload to be uploaded.
+   * This is also used to uniquely identify the upload.
+   */
+  dataHash: Uint8Array;
+  /** file_size is the total size of the data being uploaded. */
+  fileSize: number;
+  /** Number of chunks to be uploaded. */
+  chunks: number;
+}
+
+/** ChunkPiece is used to stream over large files (over the 4mb limit). */
+export interface ChunkPiece {
+  data: Uint8Array;
+  /**
+   * full_data_hash should match the hash from the original
+   * DataUpload message
+   */
+  fullDataHash: Uint8Array;
+  pieceIndex: number;
 }
 
 export const Empty = {
@@ -1206,6 +1252,9 @@ export const PlanRequest = {
     for (const v of message.previousParameterValues) {
       RichParameterValue.encode(v!, writer.uint32(42).fork()).ldelim();
     }
+    if (message.omitModuleFiles === true) {
+      writer.uint32(48).bool(message.omitModuleFiles);
+    }
     return writer;
   },
 };
@@ -1242,6 +1291,9 @@ export const PlanComplete = {
     if (message.moduleFiles.length !== 0) {
       writer.uint32(90).bytes(message.moduleFiles);
     }
+    if (message.moduleFilesHash.length !== 0) {
+      writer.uint32(98).bytes(message.moduleFilesHash);
+    }
     return writer;
   },
 };
@@ -1347,6 +1399,45 @@ export const Response = {
     if (message.apply !== undefined) {
       ApplyComplete.encode(message.apply, writer.uint32(34).fork()).ldelim();
     }
+    if (message.dataUpload !== undefined) {
+      DataUpload.encode(message.dataUpload, writer.uint32(42).fork()).ldelim();
+    }
+    if (message.chunkPiece !== undefined) {
+      ChunkPiece.encode(message.chunkPiece, writer.uint32(50).fork()).ldelim();
+    }
+    return writer;
+  },
+};
+
+export const DataUpload = {
+  encode(message: DataUpload, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.uploadType !== 0) {
+      writer.uint32(8).int32(message.uploadType);
+    }
+    if (message.dataHash.length !== 0) {
+      writer.uint32(18).bytes(message.dataHash);
+    }
+    if (message.fileSize !== 0) {
+      writer.uint32(24).int64(message.fileSize);
+    }
+    if (message.chunks !== 0) {
+      writer.uint32(32).int32(message.chunks);
+    }
+    return writer;
+  },
+};
+
+export const ChunkPiece = {
+  encode(message: ChunkPiece, writer: _m0.Writer = _m0.Writer.create()): _m0.Writer {
+    if (message.data.length !== 0) {
+      writer.uint32(10).bytes(message.data);
+    }
+    if (message.fullDataHash.length !== 0) {
+      writer.uint32(18).bytes(message.fullDataHash);
+    }
+    if (message.pieceIndex !== 0) {
+      writer.uint32(24).int32(message.pieceIndex);
+    }
     return writer;
   },
 };