From 1cdc3e8921e0188d39aa7b06ecac035fcde6f1b2 Mon Sep 17 00:00:00 2001 From: Cian Johnston Date: Mon, 25 Nov 2024 11:19:14 +0000 Subject: [PATCH] feat!: extract provisioner tags from coder_workspace_tags data source (#15578) Relates to https://github.com/coder/coder/issues/15087 and https://github.com/coder/coder/issues/15427 - Extracts provisioner job tags from `coder_workspace_tags` on template version creation using `provisioner/terraform/tfparse` added in https://github.com/coder/coder/pull/15236 - Drops a WARN log in coderd if no matching provisioners found. - Also drops a warning message in the CLI if no provisioners are found. - To support both CLI and UI warnings, added a `codersdk.MatchedProvisioners` struct to the `TemplateVersion` response containing details of how many provisioners were around at the time of the insert. Co-authored-by: Mathias Fredriksson --- cli/templatepush.go | 24 ++++ cli/templatepush_test.go | 86 +++++++++++ coderd/apidoc/docs.go | 21 +++ coderd/apidoc/swagger.json | 21 +++ coderd/templateversions.go | 139 +++++++++++++++--- coderd/templateversions_test.go | 248 ++++++++++++++++++++++++++++++++ codersdk/provisionerdaemons.go | 16 +++ codersdk/templateversions.go | 3 +- docs/reference/api/schemas.md | 52 +++++-- docs/reference/api/templates.md | 171 +++++++++++++--------- site/src/api/typesGenerated.ts | 8 ++ 11 files changed, 694 insertions(+), 95 deletions(-) diff --git a/cli/templatepush.go b/cli/templatepush.go index 22a77791c5..8516d7f9c1 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -2,6 +2,7 @@ package cli import ( "bufio" + "encoding/json" "errors" "fmt" "io" @@ -415,6 +416,29 @@ func createValidTemplateVersion(inv *serpent.Invocation, args createValidTemplat if err != nil { return nil, err } + var tagsJSON strings.Builder + if err := json.NewEncoder(&tagsJSON).Encode(version.Job.Tags); err != nil { + // Fall back to the less-pretty string representation. + tagsJSON.Reset() + _, _ = tagsJSON.WriteString(fmt.Sprintf("%v", version.Job.Tags)) + } + if version.MatchedProvisioners.Count == 0 { + cliui.Warnf(inv.Stderr, `No provisioners are available to handle the job! +Please contact your deployment administrator for assistance. +Details: + Provisioner job ID : %s + Requested tags : %s +`, version.Job.ID, tagsJSON.String()) + } else if version.MatchedProvisioners.Available == 0 { + cliui.Warnf(inv.Stderr, `All available provisioner daemons have been silent for a while. +Your build will proceed once they become available. +If this persists, please contact your deployment administrator for assistance. +Details: + Provisioner job ID : %s + Requested tags : %s + Most recently seen : %s +`, version.Job.ID, strings.TrimSpace(tagsJSON.String()), version.MatchedProvisioners.MostRecentlySeen.Time) + } err = cliui.ProvisionerJob(inv.Context(), inv.Stdout, cliui.ProvisionerJobOptions{ Fetch: func() (codersdk.ProvisionerJob, error) { diff --git a/cli/templatepush_test.go b/cli/templatepush_test.go index 4e9c861396..a20e307074 100644 --- a/cli/templatepush_test.go +++ b/cli/templatepush_test.go @@ -8,6 +8,7 @@ import ( "runtime" "strings" "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -16,9 +17,12 @@ import ( "github.com/coder/coder/v2/cli/clitest" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/provisioner/echo" + "github.com/coder/coder/v2/provisioner/terraform/tfparse" + "github.com/coder/coder/v2/provisionersdk" "github.com/coder/coder/v2/provisionersdk/proto" "github.com/coder/coder/v2/pty/ptytest" "github.com/coder/coder/v2/testutil" @@ -406,6 +410,88 @@ func TestTemplatePush(t *testing.T) { t.Run("ProvisionerTags", func(t *testing.T) { t.Parallel() + t.Run("WorkspaceTagsTerraform", func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) + + // Start an instance **without** a built-in provisioner. + // We're not actually testing that the Terraform applies. + // What we test is that a provisioner job is created with the expected + // tags based on the __content__ of the Terraform. + store, ps := dbtestutil.NewDB(t) + client := coderdtest.New(t, &coderdtest.Options{ + Database: store, + Pubsub: ps, + }) + + owner := coderdtest.CreateFirstUser(t, client) + templateAdmin, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin()) + + // Create a tar file with some pre-defined content + tarFile := testutil.CreateTar(t, map[string]string{ + "main.tf": ` +variable "a" { + type = string + default = "1" +} +data "coder_parameter" "b" { + type = string + default = "2" +} +resource "null_resource" "test" {} +data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } +}`, + }) + + // Write the tar file to disk. + tempDir := t.TempDir() + err := tfparse.WriteArchive(tarFile, "application/x-tar", tempDir) + require.NoError(t, err) + + // Run `coder templates push` + templateName := strings.ReplaceAll(testutil.GetRandomName(t), "_", "-") + var stdout, stderr strings.Builder + inv, root := clitest.New(t, "templates", "push", templateName, "-d", tempDir, "--yes") + inv.Stdout = &stdout + inv.Stderr = &stderr + clitest.SetupConfig(t, templateAdmin, root) + + // Don't forget to clean up! + cancelCtx, cancel := context.WithCancel(ctx) + t.Cleanup(cancel) + done := make(chan error) + go func() { + done <- inv.WithContext(cancelCtx).Run() + }() + + // Assert that a provisioner job was created with the desired tags. + wantTags := database.StringMap(provisionersdk.MutateTags(uuid.Nil, map[string]string{ + "foo": "bar", + "a": "1", + "b": "2", + })) + require.Eventually(t, func() bool { + jobs, err := store.GetProvisionerJobsCreatedAfter(ctx, time.Time{}) + if !assert.NoError(t, err) { + return false + } + if len(jobs) == 0 { + return false + } + return assert.EqualValues(t, wantTags, jobs[0].Tags) + }, testutil.WaitShort, testutil.IntervalSlow) + + cancel() + <-done + + require.Contains(t, stderr.String(), "No provisioners are available to handle the job!") + }) + t.Run("ChangeTags", func(t *testing.T) { t.Parallel() diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 4c79534ef8..c17ac6d3cf 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -11389,6 +11389,24 @@ const docTemplate = `{ } } }, + "codersdk.MatchedProvisioners": { + "type": "object", + "properties": { + "available": { + "description": "Available is the number of provisioner daemons that are available to\ntake jobs. This may be less than the count if some provisioners are\nbusy or have been stopped.", + "type": "integer" + }, + "count": { + "description": "Count is the number of provisioner daemons that matched the given\ntags. If the count is 0, it means no provisioner daemons matched the\nrequested tags.", + "type": "integer" + }, + "most_recently_seen": { + "description": "MostRecentlySeen is the most recently seen time of the set of matched\nprovisioners. If no provisioners matched, this field will be null.", + "type": "string", + "format": "date-time" + } + } + }, "codersdk.MinimalOrganization": { "type": "object", "required": [ @@ -13582,6 +13600,9 @@ const docTemplate = `{ "job": { "$ref": "#/definitions/codersdk.ProvisionerJob" }, + "matched_provisioners": { + "$ref": "#/definitions/codersdk.MatchedProvisioners" + }, "message": { "type": "string" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 024d247fe5..24933112d9 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -10207,6 +10207,24 @@ } } }, + "codersdk.MatchedProvisioners": { + "type": "object", + "properties": { + "available": { + "description": "Available is the number of provisioner daemons that are available to\ntake jobs. This may be less than the count if some provisioners are\nbusy or have been stopped.", + "type": "integer" + }, + "count": { + "description": "Count is the number of provisioner daemons that matched the given\ntags. If the count is 0, it means no provisioner daemons matched the\nrequested tags.", + "type": "integer" + }, + "most_recently_seen": { + "description": "MostRecentlySeen is the most recently seen time of the set of matched\nprovisioners. If no provisioners matched, this field will be null.", + "type": "string", + "format": "date-time" + } + } + }, "codersdk.MinimalOrganization": { "type": "object", "required": ["id"], @@ -12323,6 +12341,9 @@ "job": { "$ref": "#/definitions/codersdk.ProvisionerJob" }, + "matched_provisioners": { + "$ref": "#/definitions/codersdk.MatchedProvisioners" + }, "message": { "type": "string" }, diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 85e60a1dff..a0609c42c3 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -9,6 +9,8 @@ import ( "errors" "fmt" "net/http" + "os" + "time" "github.com/go-chi/chi/v5" "github.com/google/uuid" @@ -32,6 +34,7 @@ import ( "github.com/coder/coder/v2/coderd/tracing" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/examples" + "github.com/coder/coder/v2/provisioner/terraform/tfparse" "github.com/coder/coder/v2/provisionersdk" sdkproto "github.com/coder/coder/v2/provisionersdk/proto" ) @@ -74,7 +77,7 @@ func (api *API) templateVersion(rw http.ResponseWriter, r *http.Request) { warnings = append(warnings, codersdk.TemplateVersionWarningUnsupportedWorkspaces) } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), warnings)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, warnings)) } // @Summary Patch template version by ID @@ -170,7 +173,7 @@ func (api *API) patchTemplateVersion(rw http.ResponseWriter, r *http.Request) { return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Cancel template version by ID @@ -811,7 +814,7 @@ func (api *API) templateVersionsByTemplate(rw http.ResponseWriter, r *http.Reque return err } - apiVersions = append(apiVersions, convertTemplateVersion(version, convertProvisionerJob(job), nil)) + apiVersions = append(apiVersions, convertTemplateVersion(version, convertProvisionerJob(job), codersdk.MatchedProvisioners{}, nil)) } return nil @@ -866,7 +869,7 @@ func (api *API) templateVersionByName(rw http.ResponseWriter, r *http.Request) { return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Get template version by organization, template, and name @@ -931,7 +934,7 @@ func (api *API) templateVersionByOrganizationTemplateAndName(rw http.ResponseWri return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Get previous template version by organization, template, and name @@ -1017,7 +1020,7 @@ func (api *API) previousTemplateVersionByOrganizationTemplateAndName(rw http.Res return } - httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), nil)) + httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), codersdk.MatchedProvisioners{}, nil)) } // @Summary Archive template unused versions by template id @@ -1341,9 +1344,6 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } } - // Ensures the "owner" is properly applied. - tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags) - if req.ExampleID != "" && req.FileID != uuid.Nil { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ Message: "You cannot specify both an example_id and a file_id.", @@ -1437,8 +1437,58 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht } } + // Try to parse template tags from the given file. + tempDir, err := os.MkdirTemp(api.Options.CacheDir, "tfparse-*") + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "create tempdir: " + err.Error(), + }) + return + } + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + api.Logger.Error(ctx, "failed to remove temporary tfparse dir", slog.Error(err)) + } + }() + + if err := tfparse.WriteArchive(file.Data, file.Mimetype, tempDir); err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "extract archive to tempdir: " + err.Error(), + }) + return + } + + parser, diags := tfparse.New(tempDir, tfparse.WithLogger(api.Logger.Named("tfparse"))) + if diags.HasErrors() { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "parse module: " + diags.Error(), + }) + return + } + + parsedTags, err := parser.WorkspaceTagDefaults(ctx) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error checking workspace tags", + Detail: "evaluate default values of workspace tags: " + err.Error(), + }) + return + } + + // Ensure the "owner" tag is properly applied in addition to request tags and coder_workspace_tags. + // Tag order precedence: + // 1) User-specified tags in the request + // 2) Tags parsed from coder_workspace_tags data source in template file + // 2 may clobber 1. + tags := provisionersdk.MutateTags(apiKey.UserID, req.ProvisionerTags, parsedTags) + var templateVersion database.TemplateVersion var provisionerJob database.ProvisionerJob + var warnings []codersdk.TemplateVersionWarning + var matchedProvisioners codersdk.MatchedProvisioners err = api.Database.InTx(func(tx database.Store) error { jobID := uuid.New() @@ -1463,6 +1513,27 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht return err } + // Check for eligible provisioners. This allows us to log a message warning deployment administrators + // of users submitting jobs for which no provisioners are available. + matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags, api.DeploymentValues.Provisioner.DaemonPollInterval.Value()) + if err != nil { + api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err)) + } else if matchedProvisioners.Count == 0 { + api.Logger.Warn(ctx, "no matching provisioners found for job", + slog.F("user_id", apiKey.UserID), + slog.F("job_id", jobID), + slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport), + slog.F("tags", tags), + ) + } else if matchedProvisioners.Available == 0 { + api.Logger.Warn(ctx, "no active provisioners found for job", + slog.F("user_id", apiKey.UserID), + slog.F("job_id", jobID), + slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport), + slog.F("tags", tags), + ) + } + provisionerJob, err = tx.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{ ID: jobID, CreatedAt: dbtime.Now(), @@ -1552,10 +1623,14 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht api.Logger.Error(ctx, "failed to post provisioner job to pubsub", slog.Error(err)) } - httpapi.Write(ctx, rw, http.StatusCreated, convertTemplateVersion(templateVersion, convertProvisionerJob(database.GetProvisionerJobsByIDsWithQueuePositionRow{ - ProvisionerJob: provisionerJob, - QueuePosition: 0, - }), nil)) + httpapi.Write(ctx, rw, http.StatusCreated, convertTemplateVersion( + templateVersion, + convertProvisionerJob(database.GetProvisionerJobsByIDsWithQueuePositionRow{ + ProvisionerJob: provisionerJob, + QueuePosition: 0, + }), + matchedProvisioners, + warnings)) } // templateVersionResources returns the workspace agent resources associated @@ -1622,7 +1697,7 @@ func (api *API) templateVersionLogs(rw http.ResponseWriter, r *http.Request) { api.provisionerJobLogs(rw, r, job) } -func convertTemplateVersion(version database.TemplateVersion, job codersdk.ProvisionerJob, warnings []codersdk.TemplateVersionWarning) codersdk.TemplateVersion { +func convertTemplateVersion(version database.TemplateVersion, job codersdk.ProvisionerJob, matchedProvisioners codersdk.MatchedProvisioners, warnings []codersdk.TemplateVersionWarning) codersdk.TemplateVersion { return codersdk.TemplateVersion{ ID: version.ID, TemplateID: &version.TemplateID.UUID, @@ -1638,8 +1713,9 @@ func convertTemplateVersion(version database.TemplateVersion, job codersdk.Provi Username: version.CreatedByUsername, AvatarURL: version.CreatedByAvatarURL, }, - Archived: version.Archived, - Warnings: warnings, + Archived: version.Archived, + Warnings: warnings, + MatchedProvisioners: matchedProvisioners, } } @@ -1742,3 +1818,34 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID) slog.F("template_id", templateID), slog.Error(err)) } } + +func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string, pollInterval time.Duration) (codersdk.MatchedProvisioners, error) { + // Check for eligible provisioners. This allows us to return a warning to the user if they + // submit a job for which no provisioner is available. + eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ + OrganizationID: orgID, + WantTags: wantTags, + }) + if err != nil { + // Log the error but do not return any warnings. This is purely advisory and we should not block. + return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err) + } + + threePollsAgo := time.Now().Add(-3 * pollInterval) + mostRecentlySeen := codersdk.NullTime{} + var matched codersdk.MatchedProvisioners + for _, provisioner := range eligibleProvisioners { + if !provisioner.LastSeenAt.Valid { + continue + } + matched.Count++ + if provisioner.LastSeenAt.Time.After(threePollsAgo) { + matched.Available++ + } + if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) { + matched.MostRecentlySeen.Valid = true + matched.MostRecentlySeen.Time = provisioner.LastSeenAt.Time + } + } + return matched, nil +} diff --git a/coderd/templateversions_test.go b/coderd/templateversions_test.go index a03a1c6198..5ebbd0f418 100644 --- a/coderd/templateversions_test.go +++ b/coderd/templateversions_test.go @@ -16,6 +16,7 @@ import ( "github.com/coder/coder/v2/coderd/audit" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/externalauth" "github.com/coder/coder/v2/coderd/rbac" "github.com/coder/coder/v2/coderd/rbac/policy" @@ -221,6 +222,253 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) { }) require.NoError(t, err) }) + + t.Run("WorkspaceTags", func(t *testing.T) { + t.Parallel() + // This test ensures that when creating a template version from an archive continaining a coder_workspace_tags + // data source, we automatically assign some "reasonable" provisioner tag values to the resulting template + // import job. + // TODO(Cian): I'd also like to assert that the correct raw tag values are stored in the database, + // but in order to do this, we need to actually run the job! This isn't straightforward right now. + + store, ps := dbtestutil.NewDB(t) + client := coderdtest.New(t, &coderdtest.Options{ + Database: store, + Pubsub: ps, + }) + owner := coderdtest.CreateFirstUser(t, client) + templateAdmin, templateAdminUser := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin()) + + for _, tt := range []struct { + name string + files map[string]string + reqTags map[string]string + wantTags map[string]string + expectError string + }{ + { + name: "empty", + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + { + name: "main.tf with no tags", + files: map[string]string{ + `main.tf`: ` + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {}`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + { + name: "main.tf with empty workspace tags", + files: map[string]string{ + `main.tf`: ` + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = {} + }`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + { + name: "main.tf with workspace tags", + files: map[string]string{ + `main.tf`: ` + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } + }`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "a": "1", "b": "2"}, + }, + { + name: "main.tf with workspace tags and request tags", + files: map[string]string{ + `main.tf`: ` + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + } + }`, + }, + reqTags: map[string]string{"baz": "zap", "foo": "noclobber"}, + wantTags: map[string]string{"owner": "", "scope": "organization", "foo": "bar", "baz": "zap", "a": "1", "b": "2"}, + }, + { + name: "main.tf with disallowed workspace tag value", + files: map[string]string{ + `main.tf`: ` + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" { + name = "foo" + } + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + "test": null_resource.test.name, + } + }`, + }, + expectError: `Unknown variable; There is no variable named "null_resource".`, + }, + { + name: "main.tf with disallowed function in tag value", + files: map[string]string{ + `main.tf`: ` + variable "a" { + type = string + default = "1" + } + data "coder_parameter" "b" { + type = string + default = "2" + } + resource "null_resource" "test" { + name = "foo" + } + data "coder_workspace_tags" "tags" { + tags = { + "foo": "bar", + "a": var.a, + "b": data.coder_parameter.b.value, + "test": try(null_resource.test.name, "whatever"), + } + }`, + }, + expectError: `Function calls not allowed; Functions may not be called here.`, + }, + // We will allow coder_workspace_tags to set the scope on a template version import job + // BUT the user ID will be ultimately determined by the API key in the scope. + // TODO(Cian): Is this what we want? Or should we just ignore these provisioner + // tags entirely? + { + name: "main.tf with workspace tags that attempts to set user scope", + files: map[string]string{ + `main.tf`: ` + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + "owner": "12345678-1234-1234-1234-1234567890ab", + } + }`, + }, + wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, + }, + { + name: "main.tf with workspace tags that attempt to clobber org ID", + files: map[string]string{ + `main.tf`: ` + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "organization", + "owner": "12345678-1234-1234-1234-1234567890ab", + } + }`, + }, + wantTags: map[string]string{"owner": "", "scope": "organization"}, + }, + { + name: "main.tf with workspace tags that set scope=user", + files: map[string]string{ + `main.tf`: ` + resource "null_resource" "test" {} + data "coder_workspace_tags" "tags" { + tags = { + "scope": "user", + } + }`, + }, + wantTags: map[string]string{"owner": templateAdminUser.ID.String(), "scope": "user"}, + }, + } { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ctx := testutil.Context(t, testutil.WaitShort) + + // Create an archive from the files provided in the test case. + tarFile := testutil.CreateTar(t, tt.files) + + // Post the archive file + fi, err := templateAdmin.Upload(ctx, "application/x-tar", bytes.NewReader(tarFile)) + require.NoError(t, err) + + // Create a template version from the archive + tvName := strings.ReplaceAll(testutil.GetRandomName(t), "_", "-") + tv, err := templateAdmin.CreateTemplateVersion(ctx, owner.OrganizationID, codersdk.CreateTemplateVersionRequest{ + Name: tvName, + StorageMethod: codersdk.ProvisionerStorageMethodFile, + Provisioner: codersdk.ProvisionerTypeTerraform, + FileID: fi.ID, + ProvisionerTags: tt.reqTags, + }) + + if tt.expectError == "" { + require.NoError(t, err) + // Assert the expected provisioner job is created from the template version import + pj, err := store.GetProvisionerJobByID(ctx, tv.Job.ID) + require.NoError(t, err) + require.EqualValues(t, tt.wantTags, pj.Tags) + } else { + require.ErrorContains(t, err, tt.expectError) + } + + // Also assert that we get the expected information back from the API endpoint + require.Zero(t, tv.MatchedProvisioners.Count) + require.Zero(t, tv.MatchedProvisioners.Available) + require.Zero(t, tv.MatchedProvisioners.MostRecentlySeen.Time) + }) + } + }) } func TestPatchCancelTemplateVersion(t *testing.T) { diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index 12b5ac9756..acd0c6955a 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -51,6 +51,22 @@ type ProvisionerDaemon struct { Tags map[string]string `json:"tags"` } +// MatchedProvisioners represents the number of provisioner daemons +// available to take a job at a specific point in time. +type MatchedProvisioners struct { + // Count is the number of provisioner daemons that matched the given + // tags. If the count is 0, it means no provisioner daemons matched the + // requested tags. + Count int `json:"count"` + // Available is the number of provisioner daemons that are available to + // take jobs. This may be less than the count if some provisioners are + // busy or have been stopped. + Available int `json:"available"` + // MostRecentlySeen is the most recently seen time of the set of matched + // provisioners. If no provisioners matched, this field will be null. + MostRecentlySeen NullTime `json:"most_recently_seen,omitempty" format:"date-time"` +} + // ProvisionerJobStatus represents the at-time state of a job. type ProvisionerJobStatus string diff --git a/codersdk/templateversions.go b/codersdk/templateversions.go index a6f9bbe1a2..5bda52daf3 100644 --- a/codersdk/templateversions.go +++ b/codersdk/templateversions.go @@ -31,7 +31,8 @@ type TemplateVersion struct { CreatedBy MinimalUser `json:"created_by"` Archived bool `json:"archived"` - Warnings []TemplateVersionWarning `json:"warnings,omitempty" enums:"DEPRECATED_PARAMETERS"` + Warnings []TemplateVersionWarning `json:"warnings,omitempty" enums:"DEPRECATED_PARAMETERS"` + MatchedProvisioners MatchedProvisioners `json:"matched_provisioners,omitempty"` } type TemplateVersionExternalAuth struct { diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index c0f887a377..bfc923295e 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -3299,6 +3299,24 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o | --------------- | ------ | -------- | ------------ | ----------- | | `session_token` | string | true | | | +## codersdk.MatchedProvisioners + +```json +{ + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +| -------------------- | ------- | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `available` | integer | false | | Available is the number of provisioner daemons that are available to take jobs. This may be less than the count if some provisioners are busy or have been stopped. | +| `count` | integer | false | | Count is the number of provisioner daemons that matched the given tags. If the count is 0, it means no provisioner daemons matched the requested tags. | +| `most_recently_seen` | string | false | | Most recently seen is the most recently seen time of the set of matched provisioners. If no provisioners matched, this field will be null. | + ## codersdk.MinimalOrganization ```json @@ -5573,6 +5591,11 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -5585,20 +5608,21 @@ CreateWorkspaceRequest provides options for creating a new workspace. Only one o ### Properties -| Name | Type | Required | Restrictions | Description | -| ----------------- | --------------------------------------------------------------------------- | -------- | ------------ | ----------- | -| `archived` | boolean | false | | | -| `created_at` | string | false | | | -| `created_by` | [codersdk.MinimalUser](#codersdkminimaluser) | false | | | -| `id` | string | false | | | -| `job` | [codersdk.ProvisionerJob](#codersdkprovisionerjob) | false | | | -| `message` | string | false | | | -| `name` | string | false | | | -| `organization_id` | string | false | | | -| `readme` | string | false | | | -| `template_id` | string | false | | | -| `updated_at` | string | false | | | -| `warnings` | array of [codersdk.TemplateVersionWarning](#codersdktemplateversionwarning) | false | | | +| Name | Type | Required | Restrictions | Description | +| ---------------------- | --------------------------------------------------------------------------- | -------- | ------------ | ----------- | +| `archived` | boolean | false | | | +| `created_at` | string | false | | | +| `created_by` | [codersdk.MinimalUser](#codersdkminimaluser) | false | | | +| `id` | string | false | | | +| `job` | [codersdk.ProvisionerJob](#codersdkprovisionerjob) | false | | | +| `matched_provisioners` | [codersdk.MatchedProvisioners](#codersdkmatchedprovisioners) | false | | | +| `message` | string | false | | | +| `name` | string | false | | | +| `organization_id` | string | false | | | +| `readme` | string | false | | | +| `template_id` | string | false | | | +| `updated_at` | string | false | | | +| `warnings` | array of [codersdk.TemplateVersionWarning](#codersdktemplateversionwarning) | false | | | ## codersdk.TemplateVersionExternalAuth diff --git a/docs/reference/api/templates.md b/docs/reference/api/templates.md index ceda61533e..d7da209e94 100644 --- a/docs/reference/api/templates.md +++ b/docs/reference/api/templates.md @@ -446,6 +446,11 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -517,6 +522,11 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -612,6 +622,11 @@ curl -X POST http://coder-server:8080/api/v2/organizations/{organization}/templa }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1121,6 +1136,11 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1142,38 +1162,42 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ Status Code **200** -| Name | Type | Required | Restrictions | Description | -| -------------------- | ------------------------------------------------------------------------ | -------- | ------------ | ----------- | -| `[array item]` | array | false | | | -| `» archived` | boolean | false | | | -| `» created_at` | string(date-time) | false | | | -| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | -| `»» avatar_url` | string(uri) | false | | | -| `»» id` | string(uuid) | true | | | -| `»» username` | string | true | | | -| `» id` | string(uuid) | false | | | -| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | -| `»» canceled_at` | string(date-time) | false | | | -| `»» completed_at` | string(date-time) | false | | | -| `»» created_at` | string(date-time) | false | | | -| `»» error` | string | false | | | -| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | -| `»» file_id` | string(uuid) | false | | | -| `»» id` | string(uuid) | false | | | -| `»» queue_position` | integer | false | | | -| `»» queue_size` | integer | false | | | -| `»» started_at` | string(date-time) | false | | | -| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | -| `»» tags` | object | false | | | -| `»»» [any property]` | string | false | | | -| `»» worker_id` | string(uuid) | false | | | -| `» message` | string | false | | | -| `» name` | string | false | | | -| `» organization_id` | string(uuid) | false | | | -| `» readme` | string | false | | | -| `» template_id` | string(uuid) | false | | | -| `» updated_at` | string(date-time) | false | | | -| `» warnings` | array | false | | | +| Name | Type | Required | Restrictions | Description | +| ------------------------ | ------------------------------------------------------------------------ | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[array item]` | array | false | | | +| `» archived` | boolean | false | | | +| `» created_at` | string(date-time) | false | | | +| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | +| `»» avatar_url` | string(uri) | false | | | +| `»» id` | string(uuid) | true | | | +| `»» username` | string | true | | | +| `» id` | string(uuid) | false | | | +| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | +| `»» canceled_at` | string(date-time) | false | | | +| `»» completed_at` | string(date-time) | false | | | +| `»» created_at` | string(date-time) | false | | | +| `»» error` | string | false | | | +| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | +| `»» file_id` | string(uuid) | false | | | +| `»» id` | string(uuid) | false | | | +| `»» queue_position` | integer | false | | | +| `»» queue_size` | integer | false | | | +| `»» started_at` | string(date-time) | false | | | +| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | +| `»» tags` | object | false | | | +| `»»» [any property]` | string | false | | | +| `»» worker_id` | string(uuid) | false | | | +| `» matched_provisioners` | [codersdk.MatchedProvisioners](schemas.md#codersdkmatchedprovisioners) | false | | | +| `»» available` | integer | false | | Available is the number of provisioner daemons that are available to take jobs. This may be less than the count if some provisioners are busy or have been stopped. | +| `»» count` | integer | false | | Count is the number of provisioner daemons that matched the given tags. If the count is 0, it means no provisioner daemons matched the requested tags. | +| `»» most_recently_seen` | string(date-time) | false | | Most recently seen is the most recently seen time of the set of matched provisioners. If no provisioners matched, this field will be null. | +| `» message` | string | false | | | +| `» name` | string | false | | | +| `» organization_id` | string(uuid) | false | | | +| `» readme` | string | false | | | +| `» template_id` | string(uuid) | false | | | +| `» updated_at` | string(date-time) | false | | | +| `» warnings` | array | false | | | #### Enumerated Values @@ -1350,6 +1374,11 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1371,38 +1400,42 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ Status Code **200** -| Name | Type | Required | Restrictions | Description | -| -------------------- | ------------------------------------------------------------------------ | -------- | ------------ | ----------- | -| `[array item]` | array | false | | | -| `» archived` | boolean | false | | | -| `» created_at` | string(date-time) | false | | | -| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | -| `»» avatar_url` | string(uri) | false | | | -| `»» id` | string(uuid) | true | | | -| `»» username` | string | true | | | -| `» id` | string(uuid) | false | | | -| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | -| `»» canceled_at` | string(date-time) | false | | | -| `»» completed_at` | string(date-time) | false | | | -| `»» created_at` | string(date-time) | false | | | -| `»» error` | string | false | | | -| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | -| `»» file_id` | string(uuid) | false | | | -| `»» id` | string(uuid) | false | | | -| `»» queue_position` | integer | false | | | -| `»» queue_size` | integer | false | | | -| `»» started_at` | string(date-time) | false | | | -| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | -| `»» tags` | object | false | | | -| `»»» [any property]` | string | false | | | -| `»» worker_id` | string(uuid) | false | | | -| `» message` | string | false | | | -| `» name` | string | false | | | -| `» organization_id` | string(uuid) | false | | | -| `» readme` | string | false | | | -| `» template_id` | string(uuid) | false | | | -| `» updated_at` | string(date-time) | false | | | -| `» warnings` | array | false | | | +| Name | Type | Required | Restrictions | Description | +| ------------------------ | ------------------------------------------------------------------------ | -------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `[array item]` | array | false | | | +| `» archived` | boolean | false | | | +| `» created_at` | string(date-time) | false | | | +| `» created_by` | [codersdk.MinimalUser](schemas.md#codersdkminimaluser) | false | | | +| `»» avatar_url` | string(uri) | false | | | +| `»» id` | string(uuid) | true | | | +| `»» username` | string | true | | | +| `» id` | string(uuid) | false | | | +| `» job` | [codersdk.ProvisionerJob](schemas.md#codersdkprovisionerjob) | false | | | +| `»» canceled_at` | string(date-time) | false | | | +| `»» completed_at` | string(date-time) | false | | | +| `»» created_at` | string(date-time) | false | | | +| `»» error` | string | false | | | +| `»» error_code` | [codersdk.JobErrorCode](schemas.md#codersdkjoberrorcode) | false | | | +| `»» file_id` | string(uuid) | false | | | +| `»» id` | string(uuid) | false | | | +| `»» queue_position` | integer | false | | | +| `»» queue_size` | integer | false | | | +| `»» started_at` | string(date-time) | false | | | +| `»» status` | [codersdk.ProvisionerJobStatus](schemas.md#codersdkprovisionerjobstatus) | false | | | +| `»» tags` | object | false | | | +| `»»» [any property]` | string | false | | | +| `»» worker_id` | string(uuid) | false | | | +| `» matched_provisioners` | [codersdk.MatchedProvisioners](schemas.md#codersdkmatchedprovisioners) | false | | | +| `»» available` | integer | false | | Available is the number of provisioner daemons that are available to take jobs. This may be less than the count if some provisioners are busy or have been stopped. | +| `»» count` | integer | false | | Count is the number of provisioner daemons that matched the given tags. If the count is 0, it means no provisioner daemons matched the requested tags. | +| `»» most_recently_seen` | string(date-time) | false | | Most recently seen is the most recently seen time of the set of matched provisioners. If no provisioners matched, this field will be null. | +| `» message` | string | false | | | +| `» name` | string | false | | | +| `» organization_id` | string(uuid) | false | | | +| `» readme` | string | false | | | +| `» template_id` | string(uuid) | false | | | +| `» updated_at` | string(date-time) | false | | | +| `» warnings` | array | false | | | #### Enumerated Values @@ -1469,6 +1502,11 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion} \ }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", @@ -1549,6 +1587,11 @@ curl -X PATCH http://coder-server:8080/api/v2/templateversions/{templateversion} }, "worker_id": "ae5fa6f7-c55b-40c1-b40a-b36ac467652b" }, + "matched_provisioners": { + "available": 0, + "count": 0, + "most_recently_seen": "2019-08-24T14:15:22Z" + }, "message": "string", "name": "string", "organization_id": "7c60d51f-b44e-4682-87d6-449835ea4de6", diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 5ae5944c51..c1b409013b 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -759,6 +759,13 @@ export interface LoginWithPasswordResponse { readonly session_token: string; } +// From codersdk/provisionerdaemons.go +export interface MatchedProvisioners { + readonly count: number; + readonly available: number; + readonly most_recently_seen?: string; +} + // From codersdk/organizations.go export interface MinimalOrganization { readonly id: string; @@ -1464,6 +1471,7 @@ export interface TemplateVersion { readonly created_by: MinimalUser; readonly archived: boolean; readonly warnings?: Readonly>; + readonly matched_provisioners?: MatchedProvisioners; } // From codersdk/templateversions.go