opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add buffering to provisioner job logs (#4918)

Browse Source 
* feat: Add bufferring to provisioner job logs

This should improve overall build performance, and especially under load.

It removes the old `id` column on the `provisioner_job_logs` table
and replaces it with an auto-incrementing big integer to preserve order.

Funny enough, we never had to care about order before because inserts
would at minimum be 1ms different. Now they aren't, so the order needs
to be preserved.

* Fix log bufferring

* Fix frontend log streaming

* Fix JS test
This commit is contained in:
Kyle Carberry
2022-11-06 20:50:34 -06:00
committed by GitHub
parent 531f7cd489
commit 30281852d6
34 changed files with 300 additions and 476 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/cliui/provisionerjob.go
+2 -2
View File
@@ -16,14 +16,14 @@ import (
"github.com/coder/coder/codersdk"
)
func WorkspaceBuild(ctx context.Context, writer io.Writer, client *codersdk.Client, build uuid.UUID, before time.Time) error {
func WorkspaceBuild(ctx context.Context, writer io.Writer, client *codersdk.Client, build uuid.UUID) error {
return ProvisionerJob(ctx, writer, ProvisionerJobOptions{
Fetch: func() (codersdk.ProvisionerJob, error) {
build, err := client.WorkspaceBuild(ctx, build)
return build.Job, err
},
Logs: func() (<-chan codersdk.ProvisionerJobLog, io.Closer, error) {
return client.WorkspaceBuildLogsAfter(ctx, build, before)
return client.WorkspaceBuildLogsAfter(ctx, build, 0)
},
})
}
cli/create.go
+2 -4
View File
@@ -139,7 +139,6 @@ func create() *cobra.Command {
return err
}
after := time.Now()
workspace, err := client.CreateWorkspace(cmd.Context(), organization.ID, codersdk.Me, codersdk.CreateWorkspaceRequest{
TemplateID: template.ID,
Name: workspaceName,
@@ -151,7 +150,7 @@ func create() *cobra.Command {
return err
}
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, workspace.LatestBuild.ID, after)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, workspace.LatestBuild.ID)
if err != nil {
return err
}
@@ -238,7 +237,6 @@ PromptParamLoop:
_, _ = fmt.Fprintln(cmd.OutOrStdout())
// Run a dry-run with the given parameters to check correctness
after := time.Now()
dryRun, err := client.CreateTemplateVersionDryRun(cmd.Context(), templateVersion.ID, codersdk.CreateTemplateVersionDryRunRequest{
WorkspaceName: args.NewWorkspaceName,
ParameterValues: parameters,
@@ -255,7 +253,7 @@ PromptParamLoop:
return client.CancelTemplateVersionDryRun(cmd.Context(), templateVersion.ID, dryRun.ID)
},
Logs: func() (<-chan codersdk.ProvisionerJobLog, io.Closer, error) {
return client.TemplateVersionDryRunLogsAfter(cmd.Context(), templateVersion.ID, dryRun.ID, after)
return client.TemplateVersionDryRunLogsAfter(cmd.Context(), templateVersion.ID, dryRun.ID, 0)
},
// Don't show log output for the dry-run unless there's an error.
Silent: true,
cli/delete.go
+1 -2
View File
@@ -47,7 +47,6 @@ func deleteWorkspace() *cobra.Command {
)
}
before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
Transition: codersdk.WorkspaceTransitionDelete,
ProvisionerState: state,
@@ -57,7 +56,7 @@ func deleteWorkspace() *cobra.Command {
return err
}
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID, before)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID)
if err != nil {
return err
}
cli/portforward.go
+1 -1
View File
@@ -79,7 +79,7 @@ func portForward() *cobra.Command {
return xerrors.New("workspace must be in start transition to port-forward")
}
if workspace.LatestBuild.Job.CompletedAt == nil {
err = cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID, workspace.CreatedAt)
err = cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID)
if err != nil {
return err
}
cli/ssh.go
+1 -1
View File
@@ -250,7 +250,7 @@ func getWorkspaceAndAgent(ctx context.Context, cmd *cobra.Command, client *coder
return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, xerrors.New("workspace must be in start transition to ssh")
}
if workspace.LatestBuild.Job.CompletedAt == nil {
err := cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID, workspace.CreatedAt)
err := cliui.WorkspaceBuild(ctx, cmd.ErrOrStderr(), client, workspace.LatestBuild.ID)
if err != nil {
return codersdk.Workspace{}, codersdk.WorkspaceAgent{}, err
}
cli/start.go
+1 -2
View File
@@ -25,7 +25,6 @@ func start() *cobra.Command {
if err != nil {
return err
}
before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
Transition: codersdk.WorkspaceTransitionStart,
})
@@ -33,7 +32,7 @@ func start() *cobra.Command {
return err
}
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID, before)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID)
if err != nil {
return err
}
cli/state.go
+1 -3
View File
@@ -5,7 +5,6 @@ import (
"io"
"os"
"strconv"
"time"
"github.com/spf13/cobra"
@@ -100,7 +99,6 @@ func statePush() *cobra.Command {
return err
}
before := time.Now()
build, err = client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: build.TemplateVersionID,
Transition: build.Transition,
@@ -109,7 +107,7 @@ func statePush() *cobra.Command {
if err != nil {
return err
}
return cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStderr(), client, build.ID, before)
return cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStderr(), client, build.ID)
},
}
cmd.Flags().IntVarP(&buildNumber, "build", "b", 0, "Specify a workspace build to target by name.")
cli/stop.go
+1 -2
View File
@@ -33,7 +33,6 @@ func stop() *cobra.Command {
if err != nil {
return err
}
before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
Transition: codersdk.WorkspaceTransitionStop,
})
@@ -41,7 +40,7 @@ func stop() *cobra.Command {
return err
}
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID, before)
err = cliui.WorkspaceBuild(cmd.Context(), cmd.OutOrStdout(), client, build.ID)
if err != nil {
return err
}
cli/templatecreate.go
+1 -2
View File
@@ -160,7 +160,6 @@ type createValidTemplateVersionArgs struct {
}
func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVersionArgs, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) {
before := time.Now()
client := args.Client
req := codersdk.CreateTemplateVersionRequest{
@@ -187,7 +186,7 @@ func createValidTemplateVersion(cmd *cobra.Command, args createValidTemplateVers
return client.CancelTemplateVersion(cmd.Context(), version.ID)
},
Logs: func() (<-chan codersdk.ProvisionerJobLog, io.Closer, error) {
return client.TemplateVersionLogsAfter(cmd.Context(), version.ID, before)
return client.TemplateVersionLogsAfter(cmd.Context(), version.ID, 0)
},
})
if err != nil {
cli/update.go
+1 -3
View File
@@ -2,7 +2,6 @@ package cli
import (
"fmt"
"time"
"github.com/spf13/cobra"
@@ -57,7 +56,6 @@ func update() *cobra.Command {
return nil
}
before := time.Now()
build, err := client.CreateWorkspaceBuild(cmd.Context(), workspace.ID, codersdk.CreateWorkspaceBuildRequest{
TemplateVersionID: template.ActiveVersionID,
Transition: workspace.LatestBuild.Transition,
@@ -66,7 +64,7 @@ func update() *cobra.Command {
if err != nil {
return err
}
logs, closer, err := client.WorkspaceBuildLogsAfter(cmd.Context(), build.ID, before)
logs, closer, err := client.WorkspaceBuildLogsAfter(cmd.Context(), build.ID, 0)
if err != nil {
return err
}