refactor: Allow provisioner jobs to be disconnected from projects (#194)

* Nest jobs under an organization

* Rename project parameter to parameter schema

* Update references when computing project parameters

* Add files endpoint

* Allow one-off project import jobs

* Allow variables to be injected that are not defined by the schema

* Update API to use jobs first

* Fix CLI tests

* Fix linting

* Fix hex length for files table

* Reduce memory allocation for windows

codersdk/client.go

@@ -51,7 +51,7 @@ func (c *Client) SetSessionToken(token string) error {
 
 // request performs an HTTP request with the body provided.
 // The caller is responsible for closing the response body.
-func (c *Client) request(ctx context.Context, method, path string, body interface{}) (*http.Response, error) {
+func (c *Client) request(ctx context.Context, method, path string, body interface{}, opts ...func(r *http.Request)) (*http.Response, error) {
 	serverURL, err := c.URL.Parse(path)
 	if err != nil {
 		return nil, xerrors.Errorf("parse url: %w", err)
@@ -59,11 +59,16 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac
 
 	var buf bytes.Buffer
 	if body != nil {
-		enc := json.NewEncoder(&buf)
-		enc.SetEscapeHTML(false)
-		err = enc.Encode(body)
-		if err != nil {
-			return nil, xerrors.Errorf("encode body: %w", err)
+		if data, ok := body.([]byte); ok {
+			buf = *bytes.NewBuffer(data)
+		} else {
+			// Assume JSON if not bytes.
+			enc := json.NewEncoder(&buf)
+			enc.SetEscapeHTML(false)
+			err = enc.Encode(body)
+			if err != nil {
+				return nil, xerrors.Errorf("encode body: %w", err)
+			}
 		}
 	}
 
@@ -74,6 +79,9 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac
 	if body != nil {
 		req.Header.Set("Content-Type", "application/json")
 	}
+	for _, opt := range opts {
+		opt(req)
+	}
 
 	resp, err := c.httpClient.Do(req)
 	if err != nil {

codersdk/files.go

@@ -0,0 +1,28 @@
+package codersdk
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	"github.com/coder/coder/coderd"
+)
+
+const (
+	ContentTypeTar = "application/x-tar"
+)
+
+func (c *Client) UploadFile(ctx context.Context, contentType string, content []byte) (coderd.UploadFileResponse, error) {
+	res, err := c.request(ctx, http.MethodPost, "/api/v2/files", content, func(r *http.Request) {
+		r.Header.Set("Content-Type", contentType)
+	})
+	if err != nil {
+		return coderd.UploadFileResponse{}, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusCreated {
+		return coderd.UploadFileResponse{}, readBodyAsError(res)
+	}
+	var resp coderd.UploadFileResponse
+	return resp, json.NewDecoder(res.Body).Decode(&resp)
+}

codersdk/files_test.go

@@ -0,0 +1,28 @@
+package codersdk_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/coderd/coderdtest"
+	"github.com/coder/coder/codersdk"
+)
+
+func TestUpload(t *testing.T) {
+	t.Parallel()
+	t.Run("Error", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t)
+		_, err := client.UploadFile(context.Background(), "wow", []byte{})
+		require.Error(t, err)
+	})
+	t.Run("Upload", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t)
+		_ = coderdtest.CreateInitialUser(t, client)
+		_, err := client.UploadFile(context.Background(), codersdk.ContentTypeTar, []byte{'a'})
+		require.NoError(t, err)
+	})
+}

codersdk/projects_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/coderd"
@@ -42,7 +43,8 @@ func TestProject(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		_, err := client.Project(context.Background(), user.Organization, project.Name)
 		require.NoError(t, err)
 	})
@@ -54,8 +56,8 @@ func TestCreateProject(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		_, err := client.CreateProject(context.Background(), "org", coderd.CreateProjectRequest{
-			Name:        "something",
-			Provisioner: database.ProvisionerTypeEcho,
+			Name:               "something",
+			VersionImportJobID: uuid.New(),
 		})
 		require.Error(t, err)
 	})
@@ -64,7 +66,8 @@ func TestCreateProject(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		_ = coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		_ = coderdtest.CreateProject(t, client, user.Organization, job.ID)
 	})
 }
 
@@ -81,8 +84,8 @@ func TestProjectVersions(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		_, err := client.ProjectVersions(context.Background(), user.Organization, project.Name)
 		require.NoError(t, err)
 	})
@@ -101,9 +104,9 @@ func TestProjectVersion(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
-		_, err := client.ProjectVersion(context.Background(), user.Organization, project.Name, version.Name)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
+		_, err := client.ProjectVersion(context.Background(), user.Organization, project.Name, project.ActiveVersionID.String())
 		require.NoError(t, err)
 	})
 }
@@ -121,8 +124,12 @@ func TestCreateProjectVersion(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		_ = coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
+		_, err := client.CreateProjectVersion(context.Background(), user.Organization, project.Name, coderd.CreateProjectVersionRequest{
+			ImportJobID: job.ID,
+		})
+		require.NoError(t, err)
 	})
 }
 
@@ -140,10 +147,10 @@ func TestProjectVersionParameters(t *testing.T) {
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
 		coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
-		_, err := client.ProjectVersionParameters(context.Background(), user.Organization, project.Name, version.Name)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
+		coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
+		_, err := client.ProjectVersionParameters(context.Background(), user.Organization, project.Name, project.ActiveVersionID.String())
 		require.NoError(t, err)
 	})
 }
@@ -161,8 +168,8 @@ func TestProjectParameters(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		_, err := client.ProjectParameters(context.Background(), user.Organization, project.Name)
 		require.NoError(t, err)
 	})
@@ -181,8 +188,8 @@ func TestCreateProjectParameter(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		_, err := client.CreateProjectParameter(context.Background(), user.Organization, project.Name, coderd.CreateParameterValueRequest{
 			Name:              "example",
 			SourceValue:       "source-value",

codersdk/provisioners.go

@@ -59,14 +59,43 @@ func (c *Client) ProvisionerDaemonClient(ctx context.Context) (proto.DRPCProvisi
 	return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(session)), nil
 }
 
+// CreateProjectVersionImportProvisionerJob creates a job for importing
+// the provided project version.
+func (c *Client) CreateProjectVersionImportProvisionerJob(ctx context.Context, organization string, req coderd.CreateProjectImportJobRequest) (coderd.ProvisionerJob, error) {
+	res, err := c.request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/provisioners/jobs/%s/import", organization), req)
+	if err != nil {
+		return coderd.ProvisionerJob{}, err
+	}
+	if res.StatusCode != http.StatusCreated {
+		defer res.Body.Close()
+		return coderd.ProvisionerJob{}, readBodyAsError(res)
+	}
+	var job coderd.ProvisionerJob
+	return job, json.NewDecoder(res.Body).Decode(&job)
+}
+
+// ProvisionerJob returns a job by ID.
+func (c *Client) ProvisionerJob(ctx context.Context, organization string, job uuid.UUID) (coderd.ProvisionerJob, error) {
+	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/%s", organization, job), nil)
+	if err != nil {
+		return coderd.ProvisionerJob{}, nil
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		return coderd.ProvisionerJob{}, readBodyAsError(res)
+	}
+	var resp coderd.ProvisionerJob
+	return resp, json.NewDecoder(res.Body).Decode(&resp)
+}
+
 // ProvisionerJobLogs returns all logs for workspace history.
 // To stream logs, use the FollowProvisionerJobLogs function.
-func (c *Client) ProvisionerJobLogs(ctx context.Context, jobID uuid.UUID) ([]coderd.ProvisionerJobLog, error) {
-	return c.ProvisionerJobLogsBetween(ctx, jobID, time.Time{}, time.Time{})
+func (c *Client) ProvisionerJobLogs(ctx context.Context, organization string, jobID uuid.UUID) ([]coderd.ProvisionerJobLog, error) {
+	return c.ProvisionerJobLogsBetween(ctx, organization, jobID, time.Time{}, time.Time{})
 }
 
 // ProvisionerJobLogsBetween returns logs between a specific time.
-func (c *Client) ProvisionerJobLogsBetween(ctx context.Context, jobID uuid.UUID, after, before time.Time) ([]coderd.ProvisionerJobLog, error) {
+func (c *Client) ProvisionerJobLogsBetween(ctx context.Context, organization string, jobID uuid.UUID, after, before time.Time) ([]coderd.ProvisionerJobLog, error) {
 	values := url.Values{}
 	if !after.IsZero() {
 		values["after"] = []string{strconv.FormatInt(after.UTC().UnixMilli(), 10)}
@@ -74,7 +103,7 @@ func (c *Client) ProvisionerJobLogsBetween(ctx context.Context, jobID uuid.UUID,
 	if !before.IsZero() {
 		values["before"] = []string{strconv.FormatInt(before.UTC().UnixMilli(), 10)}
 	}
-	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/logs?%s", jobID, values.Encode()), nil)
+	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/%s/logs?%s", organization, jobID, values.Encode()), nil)
 	if err != nil {
 		return nil, err
 	}
@@ -89,12 +118,12 @@ func (c *Client) ProvisionerJobLogsBetween(ctx context.Context, jobID uuid.UUID,
 
 // FollowProvisionerJobLogsAfter returns a stream of workspace history logs.
 // The channel will close when the workspace history job is no longer active.
-func (c *Client) FollowProvisionerJobLogsAfter(ctx context.Context, jobID uuid.UUID, after time.Time) (<-chan coderd.ProvisionerJobLog, error) {
+func (c *Client) FollowProvisionerJobLogsAfter(ctx context.Context, organization string, jobID uuid.UUID, after time.Time) (<-chan coderd.ProvisionerJobLog, error) {
 	afterQuery := ""
 	if !after.IsZero() {
 		afterQuery = fmt.Sprintf("&after=%d", after.UTC().UnixMilli())
 	}
-	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/logs?follow%s", jobID, afterQuery), nil)
+	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/%s/logs?follow%s", organization, jobID, afterQuery), nil)
 	if err != nil {
 		return nil, err
 	}

codersdk/provisioners_test.go

@@ -8,7 +8,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/coderd"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/database"
 	"github.com/coder/coder/provisioner/echo"
@@ -55,7 +54,7 @@ func TestProvisionerJobLogs(t *testing.T) {
 	t.Run("Error", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
-		_, err := client.ProvisionerJobLogs(context.Background(), uuid.New())
+		_, err := client.ProvisionerJobLogs(context.Background(), "nothing", uuid.New())
 		require.Error(t, err)
 	})
 
@@ -64,16 +63,8 @@ func TestProvisionerJobLogs(t *testing.T) {
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
 		_ = coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
-		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
-		history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
-			ProjectVersionID: version.ID,
-			Transition:       database.WorkspaceTransitionStart,
-		})
-		require.NoError(t, err)
-		_, err = client.ProvisionerJobLogs(context.Background(), history.Provision.ID)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		_, err := client.ProvisionerJobLogs(context.Background(), user.Organization, job.ID)
 		require.NoError(t, err)
 	})
 }
@@ -83,7 +74,7 @@ func TestFollowProvisionerJobLogsAfter(t *testing.T) {
 	t.Run("Error", func(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
-		_, err := client.FollowProvisionerJobLogsAfter(context.Background(), uuid.New(), time.Time{})
+		_, err := client.FollowProvisionerJobLogsAfter(context.Background(), "nothing", uuid.New(), time.Time{})
 		require.Error(t, err)
 	})
 
@@ -92,30 +83,22 @@ func TestFollowProvisionerJobLogsAfter(t *testing.T) {
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
 		_ = coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, &echo.Responses{
-			Parse: echo.ParseComplete,
-			Provision: []*sdkproto.Provision_Response{{
-				Type: &sdkproto.Provision_Response_Log{
+		before := database.Now()
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, &echo.Responses{
+			Parse: []*sdkproto.Parse_Response{{
+				Type: &sdkproto.Parse_Response_Log{
 					Log: &sdkproto.Log{
 						Output: "hello",
 					},
 				},
 			}, {
-				Type: &sdkproto.Provision_Response_Complete{
-					Complete: &sdkproto.Provision_Complete{},
+				Type: &sdkproto.Parse_Response_Complete{
+					Complete: &sdkproto.Parse_Complete{},
 				},
 			}},
+			Provision: echo.ProvisionComplete,
 		})
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
-		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
-		after := database.Now()
-		history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
-			ProjectVersionID: version.ID,
-			Transition:       database.WorkspaceTransitionStart,
-		})
-		require.NoError(t, err)
-		logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, after)
+		logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), user.Organization, job.ID, before)
 		require.NoError(t, err)
 		_, ok := <-logs
 		require.True(t, ok)

codersdk/workspaces_test.go

@@ -42,7 +42,8 @@ func TestWorkspacesByProject(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		_, err := client.WorkspacesByProject(context.Background(), user.Organization, project.Name)
 		require.NoError(t, err)
 	})
@@ -61,7 +62,8 @@ func TestWorkspace(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
 		_, err := client.Workspace(context.Background(), "", workspace.Name)
 		require.NoError(t, err)
@@ -81,7 +83,8 @@ func TestListWorkspaceHistory(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
 		_, err := client.ListWorkspaceHistory(context.Background(), "", workspace.Name)
 		require.NoError(t, err)
@@ -102,12 +105,12 @@ func TestWorkspaceHistory(t *testing.T) {
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
 		_ = coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
+		coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
 		_, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
-			ProjectVersionID: version.ID,
+			ProjectVersionID: project.ActiveVersionID,
 			Transition:       database.WorkspaceTransitionStart,
 		})
 		require.NoError(t, err)
@@ -127,7 +130,8 @@ func TestCreateWorkspace(t *testing.T) {
 		t.Parallel()
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
 		_ = coderdtest.CreateWorkspace(t, client, "", project.ID)
 	})
 }
@@ -146,12 +150,12 @@ func TestCreateWorkspaceHistory(t *testing.T) {
 		client := coderdtest.New(t)
 		user := coderdtest.CreateInitialUser(t, client)
 		_ = coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
+		job := coderdtest.CreateProjectImportProvisionerJob(t, client, user.Organization, nil)
+		project := coderdtest.CreateProject(t, client, user.Organization, job.ID)
+		coderdtest.AwaitProvisionerJob(t, client, user.Organization, job.ID)
 		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
 		_, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
-			ProjectVersionID: version.ID,
+			ProjectVersionID: project.ActiveVersionID,
 			Transition:       database.WorkspaceTransitionStart,
 		})
 		require.NoError(t, err)