feat: use proto streams to increase maximum module files payload (#18268)

This PR implements protobuf streaming to handle large module files by:
1. **Streaming large payloads**: When module files exceed the 4MB limit,
they're streamed in chunks using a new UploadFile RPC method
2. **Database storage**: Streamed files are stored in the database and
referenced by hash for deduplication
3. **Backward compatibility**: Small module files continue using the
existing direct payload method
This commit is contained in:
Steven Masley
2025-06-13 12:46:26 -05:00
committed by GitHub
parent 8e29ee50a3
commit c1341cccdd
22 changed files with 1885 additions and 493 deletions
+1 -1
View File
@@ -171,7 +171,7 @@ var (
DisplayName: "Provisioner Daemon",
Site: rbac.Permissions(map[string][]policy.Action{
rbac.ResourceProvisionerJobs.Type: {policy.ActionRead, policy.ActionUpdate, policy.ActionCreate},
rbac.ResourceFile.Type: {policy.ActionRead},
rbac.ResourceFile.Type: {policy.ActionCreate, policy.ActionRead},
rbac.ResourceSystem.Type: {policy.WildcardSymbol},
rbac.ResourceTemplate.Type: {policy.ActionRead, policy.ActionUpdate},
// Unsure why provisionerd needs update and read personal
+6
View File
@@ -8743,6 +8743,12 @@ func (q *FakeQuerier) InsertFile(_ context.Context, arg database.InsertFileParam
q.mutex.Lock()
defer q.mutex.Unlock()
if slices.ContainsFunc(q.files, func(file database.File) bool {
return file.CreatedBy == arg.CreatedBy && file.Hash == arg.Hash
}) {
return database.File{}, newUniqueConstraintError(database.UniqueFilesHashCreatedByKey)
}
//nolint:gosimple
file := database.File{
ID: arg.ID,