feat: Allow inheriting parameters from previous template_versions when updating a template (#2397)

* WIP: feat: Update templates also updates parameters
* Insert params for template version update
* Working implementation of inherited params
* Add "--always-prompt" flag and logging info

cli/parameterslist.go

@@ -45,16 +45,23 @@ func parameterList() *cobra.Command {
 					return xerrors.Errorf("get workspace template: %w", err)
 				}
 				scopeID = template.ID
-
-			case codersdk.ParameterScopeImportJob, "template_version":
-				scope = string(codersdk.ParameterScopeImportJob)
+			case codersdk.ParameterImportJob, "template_version":
+				scope = string(codersdk.ParameterImportJob)
 				scopeID, err = uuid.Parse(name)
 				if err != nil {
 					return xerrors.Errorf("%q must be a uuid for this scope type", name)
 				}
+
+				// Could be a template_version id or a job id. Check for the
+				// version id.
+				tv, err := client.TemplateVersion(cmd.Context(), scopeID)
+				if err == nil {
+					scopeID = tv.Job.ID
+				}
+
 			default:
 				return xerrors.Errorf("%q is an unsupported scope, use %v", scope, []codersdk.ParameterScope{
-					codersdk.ParameterWorkspace, codersdk.ParameterTemplate, codersdk.ParameterScopeImportJob,
+					codersdk.ParameterWorkspace, codersdk.ParameterTemplate, codersdk.ParameterImportJob,
 				})
 			}
 

cli/templatecreate.go

@@ -82,7 +82,13 @@ func templateCreate() *cobra.Command {
 			}
 			spin.Stop()
 
-			job, parameters, err := createValidTemplateVersion(cmd, client, organization, database.ProvisionerType(provisioner), resp.Hash, parameterFile)
+			job, _, err := createValidTemplateVersion(cmd, createValidTemplateVersionArgs{
+				Client:        client,
+				Organization:  organization,
+				Provisioner:   database.ProvisionerType(provisioner),
+				FileHash:      resp.Hash,
+				ParameterFile: parameterFile,
+			})
 			if err != nil {
 				return err
 			}
@@ -98,7 +104,6 @@ func templateCreate() *cobra.Command {
 			createReq := codersdk.CreateTemplateRequest{
 				Name:                       templateName,
 				VersionID:                  job.ID,
-				ParameterValues:            parameters,
 				MaxTTLMillis:               ptr.Ref(maxTTL.Milliseconds()),
 				MinAutostartIntervalMillis: ptr.Ref(minAutostartInterval.Milliseconds()),
 			}
@@ -133,14 +138,34 @@ func templateCreate() *cobra.Command {
 	return cmd
 }
 
-func createValidTemplateVersion(cmd *cobra.Command, client *codersdk.Client, organization codersdk.Organization, provisioner database.ProvisionerType, hash string, parameterFile string, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) {
+type createValidTemplateVersionArgs struct {
+	Client        *codersdk.Client
+	Organization  codersdk.Organization
+	Provisioner   database.ProvisionerType
+	FileHash      string
+	ParameterFile string
+	// Template is only required if updating a template's active version.
+	Template *codersdk.Template
+	// ReuseParameters will attempt to reuse params from the Template field
+	// before prompting the user. Set to false to always prompt for param
+	// values.
+	ReuseParameters bool
+}
+
+func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVersionArgs, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) {
 	before := time.Now()
-	version, err := client.CreateTemplateVersion(cmd.Context(), organization.ID, codersdk.CreateTemplateVersionRequest{
+	client := args.Client
+
+	req := codersdk.CreateTemplateVersionRequest{
 		StorageMethod:   codersdk.ProvisionerStorageMethodFile,
-		StorageSource:   hash,
-		Provisioner:     codersdk.ProvisionerType(provisioner),
+		StorageSource:   args.FileHash,
+		Provisioner:     codersdk.ProvisionerType(args.Provisioner),
 		ParameterValues: parameters,
-	})
+	}
+	if args.Template != nil {
+		req.TemplateID = args.Template.ID
+	}
+	version, err := client.CreateTemplateVersion(cmd.Context(), args.Organization.ID, req)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -175,33 +200,77 @@ func createValidTemplateVersion(cmd *cobra.Command, client *codersdk.Client, org
 		return nil, nil, err
 	}
 
+	// lastParameterValues are pulled from the current active template version if
+	// templateID is provided. This allows pulling params from the last
+	// version instead of prompting if we are updating template versions.
+	lastParameterValues := make(map[string]codersdk.Parameter)
+	if args.ReuseParameters && args.Template != nil {
+		activeVersion, err := client.TemplateVersion(cmd.Context(), args.Template.ActiveVersionID)
+		if err != nil {
+			return nil, nil, xerrors.Errorf("Fetch current active template version: %w", err)
+		}
+
+		// We don't want to compute the params, we only want to copy from this scope
+		values, err := client.Parameters(cmd.Context(), codersdk.ParameterImportJob, activeVersion.Job.ID)
+		if err != nil {
+			return nil, nil, xerrors.Errorf("Fetch previous version parameters: %w", err)
+		}
+		for _, value := range values {
+			lastParameterValues[value.Name] = value
+		}
+	}
+
 	if provisionerd.IsMissingParameterError(version.Job.Error) {
 		valuesBySchemaID := map[string]codersdk.TemplateVersionParameter{}
 		for _, parameterValue := range parameterValues {
 			valuesBySchemaID[parameterValue.SchemaID.String()] = parameterValue
 		}
+
 		sort.Slice(parameterSchemas, func(i, j int) bool {
 			return parameterSchemas[i].Name < parameterSchemas[j].Name
 		})
+
+		// parameterMapFromFile can be nil if parameter file is not specified
+		var parameterMapFromFile map[string]string
+		if args.ParameterFile != "" {
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("Attempting to read the variables from the parameter file.")+"\r\n")
+			parameterMapFromFile, err = createParameterMapFromFile(args.ParameterFile)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+
+		// pulled params come from the last template version
+		pulled := make([]string, 0)
 		missingSchemas := make([]codersdk.ParameterSchema, 0)
 		for _, parameterSchema := range parameterSchemas {
 			_, ok := valuesBySchemaID[parameterSchema.ID.String()]
 			if ok {
 				continue
 			}
+
+			// The file values are handled below. So don't handle them here,
+			// just check if a value is present in the file.
+			_, fileOk := parameterMapFromFile[parameterSchema.Name]
+			if inherit, ok := lastParameterValues[parameterSchema.Name]; ok && !fileOk {
+				// If the value is not in the param file, and can be pulled from the last template version,
+				// then don't mark it as missing.
+				parameters = append(parameters, codersdk.CreateParameterRequest{
+					CloneID: inherit.ID,
+				})
+				pulled = append(pulled, fmt.Sprintf("%q", parameterSchema.Name))
+				continue
+			}
+
 			missingSchemas = append(missingSchemas, parameterSchema)
 		}
-		_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("This template has required variables! They are scoped to the template, and not viewable after being set.")+"\r\n")
-
-		// parameterMapFromFile can be nil if parameter file is not specified
-		var parameterMapFromFile map[string]string
-		if parameterFile != "" {
-			_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("Attempting to read the variables from the parameter file.")+"\r\n")
-			parameterMapFromFile, err = createParameterMapFromFile(parameterFile)
-			if err != nil {
-				return nil, nil, err
-			}
+		_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("This template has required variables! They are scoped to the template, and not viewable after being set."))
+		if len(pulled) > 0 {
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render(fmt.Sprintf("The following parameter values are being pulled from the latest template version: %s.", strings.Join(pulled, ", "))))
+			_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("Use \"--always-prompt\" flag to change the values."))
 		}
+		_, _ = fmt.Fprint(cmd.OutOrStdout(), "\r\n")
+
 		for _, parameterSchema := range missingSchemas {
 			parameterValue, err := getParameterValueFromMapOrInput(cmd, parameterMapFromFile, parameterSchema)
 			if err != nil {
@@ -218,7 +287,7 @@ func createValidTemplateVersion(cmd *cobra.Command, client *codersdk.Client, org
 
 		// This recursion is only 1 level deep in practice.
 		// The first pass populates the missing parameters, so it does not enter this `if` block again.
-		return createValidTemplateVersion(cmd, client, organization, provisioner, hash, parameterFile, parameters...)
+		return createValidTemplateVersion(cmd, args, parameters...)
 	}
 
 	if version.Job.Status != codersdk.ProvisionerJobSucceeded {

cli/templateupdate.go

@@ -10,14 +10,17 @@ import (
 	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/cli/cliui"
+	"github.com/coder/coder/coderd/database"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/provisionersdk"
 )
 
 func templateUpdate() *cobra.Command {
 	var (
-		directory   string
-		provisioner string
+		directory     string
+		provisioner   string
+		parameterFile string
+		alwaysPrompt  bool
 	)
 
 	cmd := &cobra.Command{
@@ -64,42 +67,30 @@ func templateUpdate() *cobra.Command {
 			}
 			spin.Stop()
 
-			before := time.Now()
-			templateVersion, err := client.CreateTemplateVersion(cmd.Context(), organization.ID, codersdk.CreateTemplateVersionRequest{
-				TemplateID:    template.ID,
-				StorageMethod: codersdk.ProvisionerStorageMethodFile,
-				StorageSource: resp.Hash,
-				Provisioner:   codersdk.ProvisionerType(provisioner),
+			job, _, err := createValidTemplateVersion(cmd, createValidTemplateVersionArgs{
+				Client:          client,
+				Organization:    organization,
+				Provisioner:     database.ProvisionerType(provisioner),
+				FileHash:        resp.Hash,
+				ParameterFile:   parameterFile,
+				Template:        &template,
+				ReuseParameters: !alwaysPrompt,
 			})
 			if err != nil {
 				return err
 			}
-			logs, err := client.TemplateVersionLogsAfter(cmd.Context(), templateVersion.ID, before)
-			if err != nil {
-				return err
-			}
-			for {
-				log, ok := <-logs
-				if !ok {
-					break
-				}
-				_, _ = fmt.Printf("%s (%s): %s\n", provisioner, log.Level, log.Output)
-			}
-			templateVersion, err = client.TemplateVersion(cmd.Context(), templateVersion.ID)
-			if err != nil {
-				return err
-			}
 
-			if templateVersion.Job.Status != codersdk.ProvisionerJobSucceeded {
-				return xerrors.Errorf("job failed: %s", templateVersion.Job.Error)
+			if job.Job.Status != codersdk.ProvisionerJobSucceeded {
+				return xerrors.Errorf("job failed: %s", job.Job.Status)
 			}
 
 			err = client.UpdateActiveTemplateVersion(cmd.Context(), template.ID, codersdk.UpdateActiveTemplateVersion{
-				ID: templateVersion.ID,
+				ID: job.ID,
 			})
 			if err != nil {
 				return err
 			}
+
 			_, _ = fmt.Printf("Updated version!\n")
 			return nil
 		},
@@ -108,6 +99,8 @@ func templateUpdate() *cobra.Command {
 	currentDirectory, _ := os.Getwd()
 	cmd.Flags().StringVarP(&directory, "directory", "d", currentDirectory, "Specify the directory to create from")
 	cmd.Flags().StringVarP(&provisioner, "test.provisioner", "", "terraform", "Customize the provisioner backend")
+	cmd.Flags().StringVarP(&parameterFile, "parameter-file", "", "", "Specify a file path with parameter values.")
+	cmd.Flags().BoolVar(&alwaysPrompt, "always-prompt", false, "Always prompt all parameters. Does not pull parameter values from active template version")
 	cliui.AllowSkipPrompt(cmd)
 	// This is for testing!
 	err := cmd.Flags().MarkHidden("test.provisioner")

cli/templateupdate_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -17,47 +18,152 @@ import (
 
 func TestTemplateUpdate(t *testing.T) {
 	t.Parallel()
+	// NewParameter will:
+	//	1. Create a template version with 0 params
+	//	2. Create a new version with 1 param
+	//		2a. Expects 1 param prompt, fills in value
+	//	3. Assert 1 param value in new version
+	//	4. Creates a new version with same param
+	//		4a. Expects 0 prompts as the param value is carried over
+	//	5. Assert 1 param value in new version
+	//	6. Creates a new version with 0 params
+	//	7. Asset 0 params in new version
+	t.Run("NewParameter", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
+		user := coderdtest.CreateFirstUser(t, client)
+		// Create initial template version to update
+		lastActiveVersion := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+		_ = coderdtest.AwaitTemplateVersionJob(t, client, lastActiveVersion.ID)
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, lastActiveVersion.ID)
 
-	client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
-	user := coderdtest.CreateFirstUser(t, client)
-	version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
-	_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+		// Create new template version with a new parameter
+		source := clitest.CreateTemplateVersionSource(t, &echo.Responses{
+			Parse:     createTestParseResponse(),
+			Provision: echo.ProvisionComplete,
+		})
+		cmd, root := clitest.New(t, "templates", "update", template.Name, "-y", "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho))
+		clitest.SetupConfig(t, client, root)
+		pty := ptytest.New(t)
+		cmd.SetIn(pty.Input())
+		cmd.SetOut(pty.Output())
 
-	// Test the cli command.
-	source := clitest.CreateTemplateVersionSource(t, &echo.Responses{
-		Parse:     echo.ParseComplete,
-		Provision: echo.ProvisionComplete,
+		execDone := make(chan error)
+		go func() {
+			execDone <- cmd.Execute()
+		}()
+
+		matches := []struct {
+			match string
+			write string
+		}{
+			// Expect to be prompted for the new param
+			{match: "Enter a value:", write: "peter-pan"},
+		}
+		for _, m := range matches {
+			pty.ExpectMatch(m.match)
+			pty.WriteLine(m.write)
+		}
+
+		require.NoError(t, <-execDone)
+
+		// Assert template version changed and we have the new param
+		latestTV, latestParams := latestTemplateVersion(t, client, template.ID)
+		assert.NotEqual(t, lastActiveVersion.ID, latestTV.ID)
+		require.Len(t, latestParams, 1, "expect 1 param")
+		lastActiveVersion = latestTV
+
+		// Second update of the same source requires no prompt since the params
+		// are carried over.
+		cmd, root = clitest.New(t, "templates", "update", template.Name, "-y", "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho))
+		clitest.SetupConfig(t, client, root)
+		go func() {
+			execDone <- cmd.Execute()
+		}()
+		require.NoError(t, <-execDone)
+
+		// Assert template version changed and we have the carried over param
+		latestTV, latestParams = latestTemplateVersion(t, client, template.ID)
+		assert.NotEqual(t, lastActiveVersion.ID, latestTV.ID)
+		require.Len(t, latestParams, 1, "expect 1 param")
+		lastActiveVersion = latestTV
+
+		// Remove the param
+		source = clitest.CreateTemplateVersionSource(t, &echo.Responses{
+			Parse:     echo.ParseComplete,
+			Provision: echo.ProvisionComplete,
+		})
+
+		cmd, root = clitest.New(t, "templates", "update", template.Name, "-y", "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho))
+		clitest.SetupConfig(t, client, root)
+		go func() {
+			execDone <- cmd.Execute()
+		}()
+		require.NoError(t, <-execDone)
+		// Assert template version changed and the param was removed
+		latestTV, latestParams = latestTemplateVersion(t, client, template.ID)
+		assert.NotEqual(t, lastActiveVersion.ID, latestTV.ID)
+		require.Len(t, latestParams, 0, "expect 0 param")
+		lastActiveVersion = latestTV
 	})
-	cmd, root := clitest.New(t, "templates", "update", template.Name, "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho))
-	clitest.SetupConfig(t, client, root)
-	pty := ptytest.New(t)
-	cmd.SetIn(pty.Input())
-	cmd.SetOut(pty.Output())
 
-	execDone := make(chan error)
-	go func() {
-		execDone <- cmd.Execute()
-	}()
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+		_ = coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
 
-	matches := []struct {
-		match string
-		write string
-	}{
-		{match: "Upload", write: "yes"},
-	}
-	for _, m := range matches {
-		pty.ExpectMatch(m.match)
-		pty.WriteLine(m.write)
-	}
+		// Test the cli command.
+		source := clitest.CreateTemplateVersionSource(t, &echo.Responses{
+			Parse:     echo.ParseComplete,
+			Provision: echo.ProvisionComplete,
+		})
+		cmd, root := clitest.New(t, "templates", "update", template.Name, "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho))
+		clitest.SetupConfig(t, client, root)
+		pty := ptytest.New(t)
+		cmd.SetIn(pty.Input())
+		cmd.SetOut(pty.Output())
 
-	require.NoError(t, <-execDone)
+		execDone := make(chan error)
+		go func() {
+			execDone <- cmd.Execute()
+		}()
 
-	// Assert that the template version changed.
-	templateVersions, err := client.TemplateVersionsByTemplate(context.Background(), codersdk.TemplateVersionsByTemplateRequest{
-		TemplateID: template.ID,
+		matches := []struct {
+			match string
+			write string
+		}{
+			{match: "Upload", write: "yes"},
+		}
+		for _, m := range matches {
+			pty.ExpectMatch(m.match)
+			pty.WriteLine(m.write)
+		}
+
+		require.NoError(t, <-execDone)
+
+		// Assert that the template version changed.
+		templateVersions, err := client.TemplateVersionsByTemplate(context.Background(), codersdk.TemplateVersionsByTemplateRequest{
+			TemplateID: template.ID,
+		})
+		require.NoError(t, err)
+		assert.Len(t, templateVersions, 2)
+		assert.NotEqual(t, template.ActiveVersionID, templateVersions[1].ID)
 	})
+}
+
+func latestTemplateVersion(t *testing.T, client *codersdk.Client, templateID uuid.UUID) (codersdk.TemplateVersion, []codersdk.Parameter) {
+	t.Helper()
+
+	ctx := context.Background()
+	newTemplate, err := client.Template(ctx, templateID)
 	require.NoError(t, err)
-	assert.Len(t, templateVersions, 2)
-	assert.NotEqual(t, template.ActiveVersionID, templateVersions[1].ID)
+	tv, err := client.TemplateVersion(ctx, newTemplate.ActiveVersionID)
+	require.NoError(t, err)
+	params, err := client.Parameters(ctx, codersdk.ParameterImportJob, tv.Job.ID)
+	require.NoError(t, err)
+
+	return tv, params
 }