feat: load terraform modules when using dynamic parameters (#17714)

2026-09-24 15:04:27 +08:00 · 2025-05-13 16:07:29 -05:00
parent f9817af11f
commit 60762d4c13
12 changed files with 374 additions and 32 deletions
@@ -0,0 +1,86 @@
+package files
+
+import (
+	"io/fs"
+	"path"
+	"strings"
+
+	"golang.org/x/xerrors"
+)
+
+// overlayFS allows you to "join" together the template files tar file fs.FS
+// with the Terraform modules tar file fs.FS. We could potentially turn this
+// into something more parameterized/configurable, but the requirements here are
+// a _bit_ odd, because every file in the modulesFS includes the
+// .terraform/modules/ folder at the beginning of it's path.
+type overlayFS struct {
+	baseFS   fs.FS
+	overlays []Overlay
+}
+
+type Overlay struct {
+	Path string
+	fs.FS
+}
+
+func NewOverlayFS(baseFS fs.FS, overlays []Overlay) (fs.FS, error) {
+	if err := valid(baseFS); err != nil {
+		return nil, xerrors.Errorf("baseFS: %w", err)
+	}
+
+	for _, overlay := range overlays {
+		if err := valid(overlay.FS); err != nil {
+			return nil, xerrors.Errorf("overlayFS: %w", err)
+		}
+	}
+
+	return overlayFS{
+		baseFS:   baseFS,
+		overlays: overlays,
+	}, nil
+}
+
+func (f overlayFS) Open(p string) (fs.File, error) {
+	for _, overlay := range f.overlays {
+		if strings.HasPrefix(path.Clean(p), overlay.Path) {
+			return overlay.FS.Open(p)
+		}
+	}
+	return f.baseFS.Open(p)
+}
+
+func (f overlayFS) ReadDir(p string) ([]fs.DirEntry, error) {
+	for _, overlay := range f.overlays {
+		if strings.HasPrefix(path.Clean(p), overlay.Path) {
+			//nolint:forcetypeassert
+			return overlay.FS.(fs.ReadDirFS).ReadDir(p)
+		}
+	}
+	//nolint:forcetypeassert
+	return f.baseFS.(fs.ReadDirFS).ReadDir(p)
+}
+
+func (f overlayFS) ReadFile(p string) ([]byte, error) {
+	for _, overlay := range f.overlays {
+		if strings.HasPrefix(path.Clean(p), overlay.Path) {
+			//nolint:forcetypeassert
+			return overlay.FS.(fs.ReadFileFS).ReadFile(p)
+		}
+	}
+	//nolint:forcetypeassert
+	return f.baseFS.(fs.ReadFileFS).ReadFile(p)
+}
+
+// valid checks that the fs.FS implements the required interfaces.
+// The fs.FS interface is not sufficient.
+func valid(fsys fs.FS) error {
+	_, ok := fsys.(fs.ReadDirFS)
+	if !ok {
+		return xerrors.New("overlayFS does not implement ReadDirFS")
+	}
+	_, ok = fsys.(fs.ReadFileFS)
+	if !ok {
+		return xerrors.New("overlayFS does not implement ReadFileFS")
+	}
+	return nil
+}
@@ -0,0 +1,44 @@
+package files_test
+
+import (
+	"io/fs"
+	"testing"
+
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/files"
+)
+
+func TestOverlayFS(t *testing.T) {
+	t.Parallel()
+
+	a := afero.NewMemMapFs()
+	afero.WriteFile(a, "main.tf", []byte("terraform {}"), 0o644)
+	afero.WriteFile(a, ".terraform/modules/example_module/main.tf", []byte("inaccessible"), 0o644)
+	afero.WriteFile(a, ".terraform/modules/other_module/main.tf", []byte("inaccessible"), 0o644)
+	b := afero.NewMemMapFs()
+	afero.WriteFile(b, ".terraform/modules/modules.json", []byte("{}"), 0o644)
+	afero.WriteFile(b, ".terraform/modules/example_module/main.tf", []byte("terraform {}"), 0o644)
+
+	it, err := files.NewOverlayFS(afero.NewIOFS(a), []files.Overlay{{
+		Path: ".terraform/modules",
+		FS:   afero.NewIOFS(b),
+	}})
+	require.NoError(t, err)
+
+	content, err := fs.ReadFile(it, "main.tf")
+	require.NoError(t, err)
+	require.Equal(t, "terraform {}", string(content))
+
+	_, err = fs.ReadFile(it, ".terraform/modules/other_module/main.tf")
+	require.Error(t, err)
+
+	content, err = fs.ReadFile(it, ".terraform/modules/modules.json")
+	require.NoError(t, err)
+	require.Equal(t, "{}", string(content))
+
+	content, err = fs.ReadFile(it, ".terraform/modules/example_module/main.tf")
+	require.NoError(t, err)
+	require.Equal(t, "terraform {}", string(content))
+}
@@ -13,6 +13,7 @@ import (

 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/files"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/coderd/httpmw"
 	"github.com/coder/coder/v2/codersdk"
@@ -68,7 +69,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
 		return
 	}

-	fs, err := api.FileCache.Acquire(fileCtx, fileID)
+	templateFS, err := api.FileCache.Acquire(fileCtx, fileID)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
 			Message: "Internal error fetching template version Terraform.",
@@ -85,6 +86,26 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
 	tf, err := api.Database.GetTemplateVersionTerraformValues(ctx, templateVersion.ID)
 	if err == nil {
 		plan = tf.CachedPlan
+
+		if tf.CachedModuleFiles.Valid {
+			moduleFilesFS, err := api.FileCache.Acquire(fileCtx, tf.CachedModuleFiles.UUID)
+			if err != nil {
+				httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
+					Message: "Internal error fetching Terraform modules.",
+					Detail:  err.Error(),
+				})
+				return
+			}
+			defer api.FileCache.Release(tf.CachedModuleFiles.UUID)
+			templateFS, err = files.NewOverlayFS(templateFS, []files.Overlay{{Path: ".terraform/modules", FS: moduleFilesFS}})
+			if err != nil {
+				httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+					Message: "Internal error creating overlay filesystem.",
+					Detail:  err.Error(),
+				})
+				return
+			}
+		}
 	} else if !xerrors.Is(err, sql.ErrNoRows) {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: "Failed to retrieve Terraform values for template version",
@@ -124,7 +145,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
 	)

 	// Send an initial form state, computed without any user input.
-	result, diagnostics := preview.Preview(ctx, input, fs)
+	result, diagnostics := preview.Preview(ctx, input, templateFS)
 	response := codersdk.DynamicParametersResponse{
 		ID:          -1,
 		Diagnostics: previewtypes.Diagnostics(diagnostics),
@@ -152,7 +173,7 @@ func (api *API) templateVersionDynamicParameters(rw http.ResponseWriter, r *http
 				return
 			}
 			input.ParameterValues = update.Inputs
-			result, diagnostics := preview.Preview(ctx, input, fs)
+			result, diagnostics := preview.Preview(ctx, input, templateFS)
 			response := codersdk.DynamicParametersResponse{
 				ID:          update.ID,
 				Diagnostics: previewtypes.Diagnostics(diagnostics),
@@ -10,6 +10,7 @@ import (
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/provisioner/echo"
+	"github.com/coder/coder/v2/provisioner/terraform"
 	"github.com/coder/coder/v2/provisionersdk/proto"
 	"github.com/coder/coder/v2/testutil"
 	"github.com/coder/websocket"
@@ -132,3 +133,51 @@ func TestDynamicParametersOwnerSSHPublicKey(t *testing.T) {
 	require.True(t, preview.Parameters[0].Value.Valid())
 	require.Equal(t, sshKey.PublicKey, preview.Parameters[0].Value.Value.AsString())
 }
+
+func TestDynamicParametersWithTerraformModules(t *testing.T) {
+	t.Parallel()
+
+	cfg := coderdtest.DeploymentValues(t)
+	cfg.Experiments = []string{string(codersdk.ExperimentDynamicParameters)}
+	ownerClient := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true, DeploymentValues: cfg})
+	owner := coderdtest.CreateFirstUser(t, ownerClient)
+	templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID, rbac.RoleTemplateAdmin())
+
+	dynamicParametersTerraformSource, err := os.ReadFile("testdata/parameters/modules/main.tf")
+	require.NoError(t, err)
+	modulesArchive, err := terraform.GetModulesArchive(os.DirFS("testdata/parameters/modules"))
+	require.NoError(t, err)
+
+	files := echo.WithExtraFiles(map[string][]byte{
+		"main.tf": dynamicParametersTerraformSource,
+	})
+	files.ProvisionPlan = []*proto.Response{{
+		Type: &proto.Response_Plan{
+			Plan: &proto.PlanComplete{
+				Plan:        []byte("{}"),
+				ModuleFiles: modulesArchive,
+			},
+		},
+	}}
+
+	version := coderdtest.CreateTemplateVersion(t, templateAdmin, owner.OrganizationID, files)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, templateAdmin, version.ID)
+	_ = coderdtest.CreateTemplate(t, templateAdmin, owner.OrganizationID, version.ID)
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	stream, err := templateAdmin.TemplateVersionDynamicParameters(ctx, templateAdminUser.ID, version.ID)
+	require.NoError(t, err)
+	defer stream.Close(websocket.StatusGoingAway)
+
+	previews := stream.Chan()
+
+	// Should see the output of the module represented
+	preview := testutil.RequireReceive(ctx, t, previews)
+	require.Equal(t, -1, preview.ID)
+	require.Empty(t, preview.Diagnostics)
+
+	require.Len(t, preview.Parameters, 1)
+	require.Equal(t, "jetbrains_ide", preview.Parameters[0].Name)
+	require.True(t, preview.Parameters[0].Value.Valid())
+	require.Equal(t, "CL", preview.Parameters[0].Value.AsString())
+}
@@ -0,0 +1,94 @@
+terraform {
+  required_version = ">= 1.0"
+
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = ">= 0.17"
+    }
+  }
+}
+
+locals {
+  jetbrains_ides = {
+    "GO" = {
+      icon          = "/icon/goland.svg",
+      name          = "GoLand",
+      identifier    = "GO",
+    },
+    "WS" = {
+      icon          = "/icon/webstorm.svg",
+      name          = "WebStorm",
+      identifier    = "WS",
+    },
+    "IU" = {
+      icon          = "/icon/intellij.svg",
+      name          = "IntelliJ IDEA Ultimate",
+      identifier    = "IU",
+    },
+    "PY" = {
+      icon          = "/icon/pycharm.svg",
+      name          = "PyCharm Professional",
+      identifier    = "PY",
+    },
+    "CL" = {
+      icon          = "/icon/clion.svg",
+      name          = "CLion",
+      identifier    = "CL",
+    },
+    "PS" = {
+      icon          = "/icon/phpstorm.svg",
+      name          = "PhpStorm",
+      identifier    = "PS",
+    },
+    "RM" = {
+      icon          = "/icon/rubymine.svg",
+      name          = "RubyMine",
+      identifier    = "RM",
+    },
+    "RD" = {
+      icon          = "/icon/rider.svg",
+      name          = "Rider",
+      identifier    = "RD",
+    },
+    "RR" = {
+      icon          = "/icon/rustrover.svg",
+      name          = "RustRover",
+      identifier    = "RR"
+    }
+  }
+
+  icon          = local.jetbrains_ides[data.coder_parameter.jetbrains_ide.value].icon
+  display_name  = local.jetbrains_ides[data.coder_parameter.jetbrains_ide.value].name
+  identifier    = data.coder_parameter.jetbrains_ide.value
+}
+
+data "coder_parameter" "jetbrains_ide" {
+  type         = "string"
+  name         = "jetbrains_ide"
+  display_name = "JetBrains IDE"
+  icon         = "/icon/gateway.svg"
+  mutable      = true
+  default      = sort(keys(local.jetbrains_ides))[0]
+
+  dynamic "option" {
+    for_each = local.jetbrains_ides
+    content {
+      icon  = option.value.icon
+      name  = option.value.name
+      value = option.key
+    }
+  }
+}
+
+output "identifier" {
+  value = local.identifier
+}
+
+output "display_name" {
+  value = local.display_name
+}
+
+output "icon" {
+  value = local.icon
+}
@@ -0,0 +1 @@
+{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"jetbrains_gateway","Source":"jetbrains_gateway","Dir":".terraform/modules/jetbrains_gateway"}]}
@@ -0,0 +1,5 @@
+terraform {}
+
+module "jetbrains_gateway" {
+  source = "jetbrains_gateway"
+}
				`@@ -0,0 +1 @@`
				`{"Modules":[{"Key":"","Source":"","Dir":"."},{"Key":"jetbrains_gateway","Source":"jetbrains_gateway","Dir":".terraform/modules/jetbrains_gateway"}]}`