opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-22 05:05:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: reduce size of terraform modules archive (#17749)

Browse Source
This commit is contained in:
ケイラ
2025-05-12 13:50:07 -06:00
committed by GitHub
parent 10b44a5d1d
commit d0ab91c16f
29 changed files with 816 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/dbauthz/dbauthz.go
+5 -6
View File
@@ -12,21 +12,19 @@ import (
"time"
"github.com/google/uuid"
"github.com/open-policy-agent/opa/topdown"
"golang.org/x/xerrors"
"github.com/open-policy-agent/opa/topdown"
"cdr.dev/slog"
"github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/rbac/rolestore"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/httpapi/httpapiconstraints"
"github.com/coder/coder/v2/coderd/httpmw/loggermw"
"github.com/coder/coder/v2/coderd/prebuilds"
"github.com/coder/coder/v2/coderd/rbac"
"github.com/coder/coder/v2/coderd/rbac/policy"
"github.com/coder/coder/v2/coderd/rbac/rolestore"
"github.com/coder/coder/v2/coderd/util/slice"
"github.com/coder/coder/v2/provisionersdk"
)
@@ -347,6 +345,7 @@ var (
rbac.ResourceNotificationPreference.Type: {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
rbac.ResourceNotificationTemplate.Type: {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
rbac.ResourceCryptoKey.Type: {policy.ActionCreate, policy.ActionUpdate, policy.ActionDelete},
rbac.ResourceFile.Type: {policy.ActionCreate, policy.ActionRead},
}),
Org: map[string][]rbac.Permission{},
User: []rbac.Permission{},
coderd/database/dbgen/dbgen.go
+4 -3
View File
@@ -999,9 +999,10 @@ func TemplateVersionTerraformValues(t testing.TB, db database.Store, orig databa
t.Helper()
params := database.InsertTemplateVersionTerraformValuesByJobIDParams{
JobID: takeFirst(orig.JobID, uuid.New()),
CachedPlan: takeFirstSlice(orig.CachedPlan, []byte("{}")),
UpdatedAt: takeFirst(orig.UpdatedAt, dbtime.Now()),
JobID: takeFirst(orig.JobID, uuid.New()),
CachedPlan: takeFirstSlice(orig.CachedPlan, []byte("{}")),
CachedModuleFiles: orig.CachedModuleFiles,
UpdatedAt: takeFirst(orig.UpdatedAt, dbtime.Now()),
}
err := db.InsertTemplateVersionTerraformValuesByJobID(genCtx, params)
coderd/database/dbmem/dbmem.go
+1
View File
@@ -9315,6 +9315,7 @@ func (q *FakeQuerier) InsertTemplateVersionTerraformValuesByJobID(_ context.Cont
row := database.TemplateVersionTerraformValue{
TemplateVersionID: templateVersion.ID,
CachedPlan: arg.CachedPlan,
CachedModuleFiles: arg.CachedModuleFiles,
UpdatedAt: arg.UpdatedAt,
}
q.templateVersionTerraformValues = append(q.templateVersionTerraformValues, row)
coderd/database/dump.sql
Generated
+5 -1
View File
@@ -1440,7 +1440,8 @@ CREATE TABLE template_version_presets (
CREATE TABLE template_version_terraform_values (
template_version_id uuid NOT NULL,
updated_at timestamp with time zone DEFAULT now() NOT NULL,
cached_plan jsonb NOT NULL
cached_plan jsonb NOT NULL,
cached_module_files uuid
);
CREATE TABLE template_version_variables (
@@ -2850,6 +2851,9 @@ ALTER TABLE ONLY template_version_preset_parameters
ALTER TABLE ONLY template_version_presets
ADD CONSTRAINT template_version_presets_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
ALTER TABLE ONLY template_version_terraform_values
ADD CONSTRAINT template_version_terraform_values_cached_module_files_fkey FOREIGN KEY (cached_module_files) REFERENCES files(id);
ALTER TABLE ONLY template_version_terraform_values
ADD CONSTRAINT template_version_terraform_values_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
coderd/database/foreign_key_constraint.go
+1
View File
@@ -46,6 +46,7 @@ const (
ForeignKeyTemplateVersionParametersTemplateVersionID ForeignKeyConstraint = "template_version_parameters_template_version_id_fkey" // ALTER TABLE ONLY template_version_parameters ADD CONSTRAINT template_version_parameters_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
ForeignKeyTemplateVersionPresetParametTemplateVersionPresetID ForeignKeyConstraint = "template_version_preset_paramet_template_version_preset_id_fkey" // ALTER TABLE ONLY template_version_preset_parameters ADD CONSTRAINT template_version_preset_paramet_template_version_preset_id_fkey FOREIGN KEY (template_version_preset_id) REFERENCES template_version_presets(id) ON DELETE CASCADE;
ForeignKeyTemplateVersionPresetsTemplateVersionID ForeignKeyConstraint = "template_version_presets_template_version_id_fkey" // ALTER TABLE ONLY template_version_presets ADD CONSTRAINT template_version_presets_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
ForeignKeyTemplateVersionTerraformValuesCachedModuleFiles ForeignKeyConstraint = "template_version_terraform_values_cached_module_files_fkey" // ALTER TABLE ONLY template_version_terraform_values ADD CONSTRAINT template_version_terraform_values_cached_module_files_fkey FOREIGN KEY (cached_module_files) REFERENCES files(id);
ForeignKeyTemplateVersionTerraformValuesTemplateVersionID ForeignKeyConstraint = "template_version_terraform_values_template_version_id_fkey" // ALTER TABLE ONLY template_version_terraform_values ADD CONSTRAINT template_version_terraform_values_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
ForeignKeyTemplateVersionVariablesTemplateVersionID ForeignKeyConstraint = "template_version_variables_template_version_id_fkey" // ALTER TABLE ONLY template_version_variables ADD CONSTRAINT template_version_variables_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
ForeignKeyTemplateVersionWorkspaceTagsTemplateVersionID ForeignKeyConstraint = "template_version_workspace_tags_template_version_id_fkey" // ALTER TABLE ONLY template_version_workspace_tags ADD CONSTRAINT template_version_workspace_tags_template_version_id_fkey FOREIGN KEY (template_version_id) REFERENCES template_versions(id) ON DELETE CASCADE;
coderd/database/migrations/000320_terraform_cached_modules.down.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE template_version_terraform_values DROP COLUMN cached_module_files;
coderd/database/migrations/000320_terraform_cached_modules.up.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE template_version_terraform_values ADD COLUMN cached_module_files uuid references files(id);
coderd/database/models.go
+1
View File
@@ -3224,6 +3224,7 @@ type TemplateVersionTerraformValue struct {
TemplateVersionID uuid.UUID `db:"template_version_id" json:"template_version_id"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
CachedPlan json.RawMessage `db:"cached_plan" json:"cached_plan"`
CachedModuleFiles uuid.NullUUID `db:"cached_module_files" json:"cached_module_files"`
}
type TemplateVersionVariable struct {
coderd/database/queries.sql.go
+20 -7
View File
@@ -11698,7 +11698,7 @@ func (q *sqlQuerier) UpdateTemplateVersionExternalAuthProvidersByJobID(ctx conte
const getTemplateVersionTerraformValues = `-- name: GetTemplateVersionTerraformValues :one
SELECT
template_version_terraform_values.template_version_id, template_version_terraform_values.updated_at, template_version_terraform_values.cached_plan
template_version_terraform_values.template_version_id, template_version_terraform_values.updated_at, template_version_terraform_values.cached_plan, template_version_terraform_values.cached_module_files
FROM
template_version_terraform_values
WHERE
@@ -11708,7 +11708,12 @@ WHERE
func (q *sqlQuerier) GetTemplateVersionTerraformValues(ctx context.Context, templateVersionID uuid.UUID) (TemplateVersionTerraformValue, error) {
row := q.db.QueryRowContext(ctx, getTemplateVersionTerraformValues, templateVersionID)
var i TemplateVersionTerraformValue
err := row.Scan(&i.TemplateVersionID, &i.UpdatedAt, &i.CachedPlan)
err := row.Scan(
&i.TemplateVersionID,
&i.UpdatedAt,
&i.CachedPlan,
&i.CachedModuleFiles,
)
return i, err
}
@@ -11717,24 +11722,32 @@ INSERT INTO
template_version_terraform_values (
template_version_id,
cached_plan,
cached_module_files,
updated_at
)
VALUES
(
(select id from template_versions where job_id = $1),
$2,
$3
$3,
$4
)
`
type InsertTemplateVersionTerraformValuesByJobIDParams struct {
JobID uuid.UUID `db:"job_id" json:"job_id"`
CachedPlan json.RawMessage `db:"cached_plan" json:"cached_plan"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
JobID uuid.UUID `db:"job_id" json:"job_id"`
CachedPlan json.RawMessage `db:"cached_plan" json:"cached_plan"`
CachedModuleFiles uuid.NullUUID `db:"cached_module_files" json:"cached_module_files"`
UpdatedAt time.Time `db:"updated_at" json:"updated_at"`
}
func (q *sqlQuerier) InsertTemplateVersionTerraformValuesByJobID(ctx context.Context, arg InsertTemplateVersionTerraformValuesByJobIDParams) error {
_, err := q.db.ExecContext(ctx, insertTemplateVersionTerraformValuesByJobID, arg.JobID, arg.CachedPlan, arg.UpdatedAt)
_, err := q.db.ExecContext(ctx, insertTemplateVersionTerraformValuesByJobID,
arg.JobID,
arg.CachedPlan,
arg.CachedModuleFiles,
arg.UpdatedAt,
)
return err
}
coderd/database/queries/templateversionterraformvalues.sql
+2
View File
@@ -11,11 +11,13 @@ INSERT INTO
template_version_terraform_values (
template_version_id,
cached_plan,
cached_module_files,
updated_at
)
VALUES
(
(select id from template_versions where job_id = @job_id),
@cached_plan,
@cached_module_files,
@updated_at
);
coderd/files/cache.go
+5 -1
View File
@@ -63,7 +63,11 @@ func (c *Cache) Acquire(ctx context.Context, fileID uuid.UUID) (fs.FS, error) {
// mutex has been released, or we would continue to hold the lock until the
// entire file has been fetched, which may be slow, and would prevent other
// files from being fetched in parallel.
return c.prepare(ctx, fileID).Load()
it, err := c.prepare(ctx, fileID).Load()
if err != nil {
c.Release(fileID)
}
return it, err
}
func (c *Cache) prepare(ctx context.Context, fileID uuid.UUID) *lazy.ValueWithError[fs.FS] {
coderd/parameters.go
+1 -1
View File
@@ -69,7 +69,6 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
}
fs, err := api.FileCache.Acquire(fileCtx, fileID)
defer api.FileCache.Release(fileID)
if err != nil {
httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
Message: "Internal error fetching template version Terraform.",
@@ -77,6 +76,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
})
return
}
defer api.FileCache.Release(fileID)
// Having the Terraform plan available for the evaluation engine is helpful
// for populating values from data blocks, but isn't strictly required. If
coderd/provisionerdserver/provisionerdserver.go
+59 -5
View File
@@ -2,7 +2,9 @@ package provisionerdserver
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@@ -50,6 +52,10 @@ import (
sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
)
const (
tarMimeType = "application/x-tar"
)
const (
// DefaultAcquireJobLongPollDur is the time the (deprecated) AcquireJob rpc waits to try to obtain a job before
// canceling and returning an empty job.
@@ -1426,11 +1432,59 @@ func (s *server) CompleteJob(ctx context.Context, completed *proto.CompletedJob)
return nil, xerrors.Errorf("update template version external auth providers: %w", err)
}
if len(jobType.TemplateImport.Plan) > 0 {
err := s.Database.InsertTemplateVersionTerraformValuesByJobID(ctx, database.InsertTemplateVersionTerraformValuesByJobIDParams{
JobID: jobID,
CachedPlan: jobType.TemplateImport.Plan,
UpdatedAt: now,
plan := jobType.TemplateImport.Plan
moduleFiles := jobType.TemplateImport.ModuleFiles
// If there is a plan, or a module files archive we need to insert a
// template_version_terraform_values row.
if len(plan) > 0 || len(moduleFiles) > 0 {
// ...but the plan and the module files archive are both optional! So
// we need to fallback to a valid JSON object if the plan was omitted.
if len(plan) == 0 {
plan = []byte("{}")
}
// ...and we only want to insert a files row if an archive was provided.
var fileID uuid.NullUUID
if len(moduleFiles) > 0 {
hashBytes := sha256.Sum256(moduleFiles)
hash := hex.EncodeToString(hashBytes[:])
// nolint:gocritic // Requires reading "system" files
file, err := s.Database.GetFileByHashAndCreator(dbauthz.AsSystemRestricted(ctx), database.GetFileByHashAndCreatorParams{Hash: hash, CreatedBy: uuid.Nil})
switch {
case err == nil:
// This set of modules is already cached, which means we can reuse them
fileID = uuid.NullUUID{
Valid: true,
UUID: file.ID,
}
case !xerrors.Is(err, sql.ErrNoRows):
return nil, xerrors.Errorf("check for cached modules: %w", err)
default:
// nolint:gocritic // Requires creating a "system" file
file, err = s.Database.InsertFile(dbauthz.AsSystemRestricted(ctx), database.InsertFileParams{
ID: uuid.New(),
Hash: hash,
CreatedBy: uuid.Nil,
CreatedAt: dbtime.Now(),
Mimetype: tarMimeType,
Data: moduleFiles,
})
if err != nil {
return nil, xerrors.Errorf("insert template version terraform modules: %w", err)
}
fileID = uuid.NullUUID{
Valid: true,
UUID: file.ID,
}
}
}
err = s.Database.InsertTemplateVersionTerraformValuesByJobID(ctx, database.InsertTemplateVersionTerraformValuesByJobIDParams{
JobID: jobID,
UpdatedAt: now,
CachedPlan: plan,
CachedModuleFiles: fileID,
})
if err != nil {
return nil, xerrors.Errorf("insert template version terraform data: %w", err)