refactor: Generalize log ownership to allow for scratch jobs (#182)

* refactor: Generalize log ownership to allow for scratch jobs

Importing may fail when creating a project. We don't want to lose this output,
but we don't want to allow users to create a failing project.

This generalizes logs to soon enable one-off situations where a user can upload
their archive, create a project, and watch the output parse to completion.

* Improve file table schema by using hash

* Fix racey test by allowing logs before

* Add debug logging for PostgreSQL insert

codersdk/provisioners.go

@@ -3,9 +3,14 @@ package codersdk
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
+	"net/url"
+	"strconv"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/hashicorp/yamux"
 	"golang.org/x/xerrors"
 	"nhooyr.io/websocket"
@@ -53,3 +58,67 @@ func (c *Client) ProvisionerDaemonClient(ctx context.Context) (proto.DRPCProvisi
 	}
 	return proto.NewDRPCProvisionerDaemonClient(provisionersdk.Conn(session)), nil
 }
+
+// ProvisionerJobLogs returns all logs for workspace history.
+// To stream logs, use the FollowProvisionerJobLogs function.
+func (c *Client) ProvisionerJobLogs(ctx context.Context, jobID uuid.UUID) ([]coderd.ProvisionerJobLog, error) {
+	return c.ProvisionerJobLogsBetween(ctx, jobID, time.Time{}, time.Time{})
+}
+
+// ProvisionerJobLogsBetween returns logs between a specific time.
+func (c *Client) ProvisionerJobLogsBetween(ctx context.Context, jobID uuid.UUID, after, before time.Time) ([]coderd.ProvisionerJobLog, error) {
+	values := url.Values{}
+	if !after.IsZero() {
+		values["after"] = []string{strconv.FormatInt(after.UTC().UnixMilli(), 10)}
+	}
+	if !before.IsZero() {
+		values["before"] = []string{strconv.FormatInt(before.UTC().UnixMilli(), 10)}
+	}
+	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/logs?%s", jobID, values.Encode()), nil)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		defer res.Body.Close()
+		return nil, readBodyAsError(res)
+	}
+
+	var logs []coderd.ProvisionerJobLog
+	return logs, json.NewDecoder(res.Body).Decode(&logs)
+}
+
+// FollowProvisionerJobLogsAfter returns a stream of workspace history logs.
+// The channel will close when the workspace history job is no longer active.
+func (c *Client) FollowProvisionerJobLogsAfter(ctx context.Context, jobID uuid.UUID, after time.Time) (<-chan coderd.ProvisionerJobLog, error) {
+	afterQuery := ""
+	if !after.IsZero() {
+		afterQuery = fmt.Sprintf("&after=%d", after.UTC().UnixMilli())
+	}
+	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/provisioners/jobs/%s/logs?follow%s", jobID, afterQuery), nil)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		defer res.Body.Close()
+		return nil, readBodyAsError(res)
+	}
+
+	logs := make(chan coderd.ProvisionerJobLog)
+	decoder := json.NewDecoder(res.Body)
+	go func() {
+		defer close(logs)
+		var log coderd.ProvisionerJobLog
+		for {
+			err = decoder.Decode(&log)
+			if err != nil {
+				return
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case logs <- log:
+			}
+		}
+	}()
+	return logs, nil
+}

codersdk/provisioners_test.go

@@ -3,11 +3,17 @@ package codersdk_test
 import (
 	"context"
 	"testing"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
 
+	"github.com/coder/coder/coderd"
 	"github.com/coder/coder/coderd/coderdtest"
+	"github.com/coder/coder/database"
+	"github.com/coder/coder/provisioner/echo"
 	"github.com/coder/coder/provisionerd/proto"
+	sdkproto "github.com/coder/coder/provisionersdk/proto"
 )
 
 func TestProvisionerDaemons(t *testing.T) {
@@ -44,3 +50,76 @@ func TestProvisionerDaemonClient(t *testing.T) {
 		require.NoError(t, err)
 	})
 }
+func TestProvisionerJobLogs(t *testing.T) {
+	t.Parallel()
+	t.Run("Error", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t)
+		_, err := client.ProvisionerJobLogs(context.Background(), uuid.New())
+		require.Error(t, err)
+	})
+
+	t.Run("List", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t)
+		user := coderdtest.CreateInitialUser(t, client)
+		_ = coderdtest.NewProvisionerDaemon(t, client)
+		project := coderdtest.CreateProject(t, client, user.Organization)
+		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
+		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
+		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
+		history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
+			ProjectVersionID: version.ID,
+			Transition:       database.WorkspaceTransitionCreate,
+		})
+		require.NoError(t, err)
+		_, err = client.ProvisionerJobLogs(context.Background(), history.Provision.ID)
+		require.NoError(t, err)
+	})
+}
+
+func TestFollowProvisionerJobLogsAfter(t *testing.T) {
+	t.Parallel()
+	t.Run("Error", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t)
+		_, err := client.FollowProvisionerJobLogsAfter(context.Background(), uuid.New(), time.Time{})
+		require.Error(t, err)
+	})
+
+	t.Run("Stream", func(t *testing.T) {
+		t.Parallel()
+		client := coderdtest.New(t)
+		user := coderdtest.CreateInitialUser(t, client)
+		_ = coderdtest.NewProvisionerDaemon(t, client)
+		project := coderdtest.CreateProject(t, client, user.Organization)
+		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, &echo.Responses{
+			Parse: echo.ParseComplete,
+			Provision: []*sdkproto.Provision_Response{{
+				Type: &sdkproto.Provision_Response_Log{
+					Log: &sdkproto.Log{
+						Output: "hello",
+					},
+				},
+			}, {
+				Type: &sdkproto.Provision_Response_Complete{
+					Complete: &sdkproto.Provision_Complete{},
+				},
+			}},
+		})
+		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
+		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
+		after := database.Now()
+		history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
+			ProjectVersionID: version.ID,
+			Transition:       database.WorkspaceTransitionCreate,
+		})
+		require.NoError(t, err)
+		logs, err := client.FollowProvisionerJobLogsAfter(context.Background(), history.Provision.ID, after)
+		require.NoError(t, err)
+		_, ok := <-logs
+		require.True(t, ok)
+		_, ok = <-logs
+		require.False(t, ok)
+	})
+}

codersdk/workspaces.go

@@ -5,9 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"net/url"
-	"strconv"
-	"time"
 
 	"github.com/coder/coder/coderd"
 )
@@ -134,74 +131,3 @@ func (c *Client) CreateWorkspaceHistory(ctx context.Context, owner, workspace st
 	var workspaceHistory coderd.WorkspaceHistory
 	return workspaceHistory, json.NewDecoder(res.Body).Decode(&workspaceHistory)
 }
-
-// WorkspaceHistoryLogs returns all logs for workspace history.
-// To stream logs, use the FollowWorkspaceHistoryLogs function.
-func (c *Client) WorkspaceHistoryLogs(ctx context.Context, owner, workspace, history string) ([]coderd.WorkspaceHistoryLog, error) {
-	return c.WorkspaceHistoryLogsBetween(ctx, owner, workspace, history, time.Time{}, time.Time{})
-}
-
-// WorkspaceHistoryLogsBetween returns logs between a specific time.
-func (c *Client) WorkspaceHistoryLogsBetween(ctx context.Context, owner, workspace, history string, after, before time.Time) ([]coderd.WorkspaceHistoryLog, error) {
-	if owner == "" {
-		owner = "me"
-	}
-	values := url.Values{}
-	if !after.IsZero() {
-		values["after"] = []string{strconv.FormatInt(after.UTC().UnixMilli(), 10)}
-	}
-	if !before.IsZero() {
-		values["before"] = []string{strconv.FormatInt(before.UTC().UnixMilli(), 10)}
-	}
-	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaces/%s/%s/version/%s/logs?%s", owner, workspace, history, values.Encode()), nil)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != http.StatusOK {
-		defer res.Body.Close()
-		return nil, readBodyAsError(res)
-	}
-
-	var logs []coderd.WorkspaceHistoryLog
-	return logs, json.NewDecoder(res.Body).Decode(&logs)
-}
-
-// FollowWorkspaceHistoryLogsAfter returns a stream of workspace history logs.
-// The channel will close when the workspace history job is no longer active.
-func (c *Client) FollowWorkspaceHistoryLogsAfter(ctx context.Context, owner, workspace, history string, after time.Time) (<-chan coderd.WorkspaceHistoryLog, error) {
-	afterQuery := ""
-	if !after.IsZero() {
-		afterQuery = fmt.Sprintf("&after=%d", after.UTC().UnixMilli())
-	}
-	if owner == "" {
-		owner = "me"
-	}
-
-	res, err := c.request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/workspaces/%s/%s/version/%s/logs?follow%s", owner, workspace, history, afterQuery), nil)
-	if err != nil {
-		return nil, err
-	}
-	if res.StatusCode != http.StatusOK {
-		defer res.Body.Close()
-		return nil, readBodyAsError(res)
-	}
-
-	logs := make(chan coderd.WorkspaceHistoryLog)
-	decoder := json.NewDecoder(res.Body)
-	go func() {
-		defer close(logs)
-		var log coderd.WorkspaceHistoryLog
-		for {
-			err = decoder.Decode(&log)
-			if err != nil {
-				return
-			}
-			select {
-			case <-ctx.Done():
-				return
-			case logs <- log:
-			}
-		}
-	}()
-	return logs, nil
-}

codersdk/workspaces_test.go

@@ -3,15 +3,12 @@ package codersdk_test
 import (
 	"context"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/require"
 
 	"github.com/coder/coder/coderd"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/database"
-	"github.com/coder/coder/provisioner/echo"
-	"github.com/coder/coder/provisionersdk/proto"
 )
 
 func TestWorkspaces(t *testing.T) {
@@ -160,77 +157,3 @@ func TestCreateWorkspaceHistory(t *testing.T) {
 		require.NoError(t, err)
 	})
 }
-
-func TestWorkspaceHistoryLogs(t *testing.T) {
-	t.Parallel()
-	t.Run("Error", func(t *testing.T) {
-		t.Parallel()
-		client := coderdtest.New(t)
-		_, err := client.WorkspaceHistoryLogs(context.Background(), "", "", "")
-		require.Error(t, err)
-	})
-
-	t.Run("List", func(t *testing.T) {
-		t.Parallel()
-		client := coderdtest.New(t)
-		user := coderdtest.CreateInitialUser(t, client)
-		_ = coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, nil)
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
-		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
-		history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
-			ProjectVersionID: version.ID,
-			Transition:       database.WorkspaceTransitionCreate,
-		})
-		require.NoError(t, err)
-		_, err = client.WorkspaceHistoryLogs(context.Background(), "", workspace.Name, history.Name)
-		require.NoError(t, err)
-	})
-}
-
-func TestFollowWorkspaceHistoryLogsAfter(t *testing.T) {
-	t.Parallel()
-	t.Run("Error", func(t *testing.T) {
-		t.Parallel()
-		client := coderdtest.New(t)
-		_, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", "", "", time.Time{})
-		require.Error(t, err)
-	})
-
-	t.Run("Stream", func(t *testing.T) {
-		t.Parallel()
-		client := coderdtest.New(t)
-		user := coderdtest.CreateInitialUser(t, client)
-		_ = coderdtest.NewProvisionerDaemon(t, client)
-		project := coderdtest.CreateProject(t, client, user.Organization)
-		version := coderdtest.CreateProjectVersion(t, client, user.Organization, project.Name, &echo.Responses{
-			Parse: echo.ParseComplete,
-			Provision: []*proto.Provision_Response{{
-				Type: &proto.Provision_Response_Log{
-					Log: &proto.Log{
-						Output: "hello",
-					},
-				},
-			}, {
-				Type: &proto.Provision_Response_Complete{
-					Complete: &proto.Provision_Complete{},
-				},
-			}},
-		})
-		coderdtest.AwaitProjectVersionImported(t, client, user.Organization, project.Name, version.Name)
-		workspace := coderdtest.CreateWorkspace(t, client, "", project.ID)
-		after := database.Now()
-		history, err := client.CreateWorkspaceHistory(context.Background(), "", workspace.Name, coderd.CreateWorkspaceHistoryRequest{
-			ProjectVersionID: version.ID,
-			Transition:       database.WorkspaceTransitionCreate,
-		})
-		require.NoError(t, err)
-		logs, err := client.FollowWorkspaceHistoryLogsAfter(context.Background(), "", workspace.Name, history.Name, after)
-		require.NoError(t, err)
-		_, ok := <-logs
-		require.True(t, ok)
-		_, ok = <-logs
-		require.False(t, ok)
-	})
-}