feat: loadtest output formats (#4928)

loadtest/harness/harness.go

@@ -3,6 +3,7 @@ package harness
 import (
 	"context"
 	"sync"
+	"time"
 
 	"github.com/hashicorp/go-multierror"
 	"golang.org/x/xerrors"
@@ -27,6 +28,7 @@ type TestHarness struct {
 	runs    []*TestRun
 	started bool
 	done    chan struct{}
+	elapsed time.Duration
 }
 
 // NewTestHarness creates a new TestHarness with the given ExecutionStrategy.
@@ -63,6 +65,13 @@ func (h *TestHarness) Run(ctx context.Context) (err error) {
 		}
 	}()
 
+	start := time.Now()
+	defer func() {
+		h.mut.Lock()
+		defer h.mut.Unlock()
+		h.elapsed = time.Since(start)
+	}()
+
 	err = h.strategy.Execute(ctx, h.runs)
 	//nolint:revive // we use named returns because we mutate it in a defer
 	return

loadtest/harness/results.go

@@ -1,25 +1,36 @@
 package harness
 
-import "time"
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/coder/coder/coderd/httpapi"
+)
 
 // Results is the full compiled results for a set of test runs.
 type Results struct {
-	TotalRuns int
-	TotalPass int
-	TotalFail int
+	TotalRuns int              `json:"total_runs"`
+	TotalPass int              `json:"total_pass"`
+	TotalFail int              `json:"total_fail"`
+	Elapsed   httpapi.Duration `json:"elapsed"`
+	ElapsedMS int64            `json:"elapsed_ms"`
 
-	Runs map[string]RunResult
+	Runs map[string]RunResult `json:"runs"`
 }
 
 // RunResult is the result of a single test run.
 type RunResult struct {
-	FullID    string
-	TestName  string
-	ID        string
-	Logs      []byte
-	Error     error
-	StartedAt time.Time
-	Duration  time.Duration
+	FullID     string           `json:"full_id"`
+	TestName   string           `json:"test_name"`
+	ID         string           `json:"id"`
+	Logs       string           `json:"logs"`
+	Error      error            `json:"error"`
+	StartedAt  time.Time        `json:"started_at"`
+	Duration   httpapi.Duration `json:"duration"`
+	DurationMS int64            `json:"duration_ms"`
 }
 
 // Results returns the results of the test run. Panics if the test run is not
@@ -32,13 +43,14 @@ func (r *TestRun) Result() RunResult {
 	}
 
 	return RunResult{
-		FullID:    r.FullID(),
-		TestName:  r.testName,
-		ID:        r.id,
-		Logs:      r.logs.Bytes(),
-		Error:     r.err,
-		StartedAt: r.started,
-		Duration:  r.duration,
+		FullID:     r.FullID(),
+		TestName:   r.testName,
+		ID:         r.id,
+		Logs:       r.logs.String(),
+		Error:      r.err,
+		StartedAt:  r.started,
+		Duration:   httpapi.Duration(r.duration),
+		DurationMS: r.duration.Milliseconds(),
 	}
 }
 
@@ -56,6 +68,8 @@ func (h *TestHarness) Results() Results {
 	results := Results{
 		TotalRuns: len(h.runs),
 		Runs:      make(map[string]RunResult, len(h.runs)),
+		Elapsed:   httpapi.Duration(h.elapsed),
+		ElapsedMS: h.elapsed.Milliseconds(),
 	}
 	for _, run := range h.runs {
 		runRes := run.Result()
@@ -70,3 +84,40 @@ func (h *TestHarness) Results() Results {
 
 	return results
 }
+
+// PrintText prints the results as human-readable text to the given writer.
+func (r *Results) PrintText(w io.Writer) {
+	var totalDuration time.Duration
+	for _, run := range r.Runs {
+		totalDuration += time.Duration(run.Duration)
+		if run.Error == nil {
+			continue
+		}
+
+		_, _ = fmt.Fprintf(w, "\n== FAIL: %s\n\n", run.FullID)
+		_, _ = fmt.Fprintf(w, "\tError: %s\n\n", run.Error)
+
+		// Print log lines indented.
+		_, _ = fmt.Fprintf(w, "\tLog:\n")
+		rd := bufio.NewReader(strings.NewReader(run.Logs))
+		for {
+			line, err := rd.ReadBytes('\n')
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				_, _ = fmt.Fprintf(w, "\n\tLOG PRINT ERROR: %+v\n", err)
+			}
+
+			_, _ = fmt.Fprintf(w, "\t\t%s", line)
+		}
+	}
+
+	_, _ = fmt.Fprintln(w, "\n\nTest results:")
+	_, _ = fmt.Fprintf(w, "\tPass:  %d\n", r.TotalPass)
+	_, _ = fmt.Fprintf(w, "\tFail:  %d\n", r.TotalFail)
+	_, _ = fmt.Fprintf(w, "\tTotal: %d\n", r.TotalRuns)
+	_, _ = fmt.Fprintln(w, "")
+	_, _ = fmt.Fprintf(w, "\tTotal duration: %s\n", time.Duration(r.Elapsed))
+	_, _ = fmt.Fprintf(w, "\tAvg. duration:  %s\n", totalDuration/time.Duration(r.TotalRuns))
+}