mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: loadtest output formats (#4928)
This commit is contained in:
Dean Sheather
+156
-62
@@ -1,14 +1,13 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -21,44 +20,46 @@ import (
|
||||
|
||||
func loadtest() *cobra.Command {
|
||||
var (
|
||||
configPath string
|
||||
configPath string
|
||||
outputSpecs []string
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "loadtest --config <path>",
|
||||
Use: "loadtest --config <path> [--output json[:path]] [--output text[:path]]]",
|
||||
Short: "Load test the Coder API",
|
||||
// TODO: documentation and a JSON scheme file
|
||||
Long: "Perform load tests against the Coder server. The load tests " +
|
||||
"configurable via a JSON file.",
|
||||
// TODO: documentation and a JSON schema file
|
||||
Long: "Perform load tests against the Coder server. The load tests are configurable via a JSON file.",
|
||||
Example: formatExamples(
|
||||
example{
|
||||
Description: "Run a loadtest with the given configuration file",
|
||||
Command: "coder loadtest --config path/to/config.json",
|
||||
},
|
||||
example{
|
||||
Description: "Run a loadtest, reading the configuration from stdin",
|
||||
Command: "cat path/to/config.json | coder loadtest --config -",
|
||||
},
|
||||
example{
|
||||
Description: "Run a loadtest outputting JSON results instead",
|
||||
Command: "coder loadtest --config path/to/config.json --output json",
|
||||
},
|
||||
example{
|
||||
Description: "Run a loadtest outputting JSON results to a file",
|
||||
Command: "coder loadtest --config path/to/config.json --output json:path/to/results.json",
|
||||
},
|
||||
example{
|
||||
Description: "Run a loadtest outputting text results to stdout and JSON results to a file",
|
||||
Command: "coder loadtest --config path/to/config.json --output text --output json:path/to/results.json",
|
||||
},
|
||||
),
|
||||
Hidden: true,
|
||||
Args: cobra.ExactArgs(0),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if configPath == "" {
|
||||
return xerrors.New("config is required")
|
||||
}
|
||||
|
||||
var (
|
||||
configReader io.ReadCloser
|
||||
)
|
||||
if configPath == "-" {
|
||||
configReader = io.NopCloser(cmd.InOrStdin())
|
||||
} else {
|
||||
f, err := os.Open(configPath)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("open config file %q: %w", configPath, err)
|
||||
}
|
||||
configReader = f
|
||||
}
|
||||
|
||||
var config LoadTestConfig
|
||||
err := json.NewDecoder(configReader).Decode(&config)
|
||||
_ = configReader.Close()
|
||||
config, err := loadLoadTestConfigFile(configPath, cmd.InOrStdin())
|
||||
if err != nil {
|
||||
return xerrors.Errorf("read config file %q: %w", configPath, err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = config.Validate()
|
||||
outputs, err := parseLoadTestOutputs(outputSpecs)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("validate config: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
client, err := CreateClient(cmd)
|
||||
@@ -117,52 +118,44 @@ func loadtest() *cobra.Command {
|
||||
}
|
||||
|
||||
// TODO: live progress output
|
||||
start := time.Now()
|
||||
err = th.Run(testCtx)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
|
||||
// Print the results.
|
||||
// TODO: better result printing
|
||||
// TODO: move result printing to the loadtest package, add multiple
|
||||
// output formats (like HTML, JSON)
|
||||
res := th.Results()
|
||||
var totalDuration time.Duration
|
||||
for _, run := range res.Runs {
|
||||
totalDuration += run.Duration
|
||||
if run.Error == nil {
|
||||
continue
|
||||
for _, output := range outputs {
|
||||
var (
|
||||
w = cmd.OutOrStdout()
|
||||
c io.Closer
|
||||
)
|
||||
if output.path != "-" {
|
||||
f, err := os.Create(output.path)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("create output file: %w", err)
|
||||
}
|
||||
w, c = f, f
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n== FAIL: %s\n\n", run.FullID)
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tError: %s\n\n", run.Error)
|
||||
|
||||
// Print log lines indented.
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tLog:\n")
|
||||
rd := bufio.NewReader(bytes.NewBuffer(run.Logs))
|
||||
for {
|
||||
line, err := rd.ReadBytes('\n')
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
switch output.format {
|
||||
case loadTestOutputFormatText:
|
||||
res.PrintText(w)
|
||||
case loadTestOutputFormatJSON:
|
||||
err = json.NewEncoder(w).Encode(res)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\n\tLOG PRINT ERROR: %+v\n", err)
|
||||
return xerrors.Errorf("encode JSON: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\t\t%s", line)
|
||||
if c != nil {
|
||||
err = c.Close()
|
||||
if err != nil {
|
||||
return xerrors.Errorf("close output file: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\n\nTest results:")
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tPass: %d\n", res.TotalPass)
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tFail: %d\n", res.TotalFail)
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal: %d\n", res.TotalRuns)
|
||||
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "")
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tTotal duration: %s\n", elapsed)
|
||||
_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "\tAvg. duration: %s\n", totalDuration/time.Duration(res.TotalRuns))
|
||||
|
||||
// Cleanup.
|
||||
_, _ = fmt.Fprintln(cmd.ErrOrStderr(), "\nCleaning up...")
|
||||
err = th.Cleanup(cmd.Context())
|
||||
@@ -170,10 +163,111 @@ func loadtest() *cobra.Command {
|
||||
return xerrors.Errorf("cleanup tests: %w", err)
|
||||
}
|
||||
|
||||
if res.TotalFail > 0 {
|
||||
return xerrors.New("load test failed, see above for more details")
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
cliflag.StringVarP(cmd.Flags(), &configPath, "config", "", "CODER_LOADTEST_CONFIG_PATH", "", "Path to the load test configuration file, or - to read from stdin.")
|
||||
cliflag.StringArrayVarP(cmd.Flags(), &outputSpecs, "output", "", "CODER_LOADTEST_OUTPUTS", []string{"text"}, "Output formats, see usage for more information.")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func loadLoadTestConfigFile(configPath string, stdin io.Reader) (LoadTestConfig, error) {
|
||||
if configPath == "" {
|
||||
return LoadTestConfig{}, xerrors.New("config is required")
|
||||
}
|
||||
|
||||
var (
|
||||
configReader io.ReadCloser
|
||||
)
|
||||
if configPath == "-" {
|
||||
configReader = io.NopCloser(stdin)
|
||||
} else {
|
||||
f, err := os.Open(configPath)
|
||||
if err != nil {
|
||||
return LoadTestConfig{}, xerrors.Errorf("open config file %q: %w", configPath, err)
|
||||
}
|
||||
configReader = f
|
||||
}
|
||||
|
||||
var config LoadTestConfig
|
||||
err := json.NewDecoder(configReader).Decode(&config)
|
||||
_ = configReader.Close()
|
||||
if err != nil {
|
||||
return LoadTestConfig{}, xerrors.Errorf("read config file %q: %w", configPath, err)
|
||||
}
|
||||
|
||||
err = config.Validate()
|
||||
if err != nil {
|
||||
return LoadTestConfig{}, xerrors.Errorf("validate config: %w", err)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
type loadTestOutputFormat string
|
||||
|
||||
const (
|
||||
loadTestOutputFormatText loadTestOutputFormat = "text"
|
||||
loadTestOutputFormatJSON loadTestOutputFormat = "json"
|
||||
// TODO: html format
|
||||
)
|
||||
|
||||
type loadTestOutput struct {
|
||||
format loadTestOutputFormat
|
||||
// Up to one path (the first path) will have the value "-" which signifies
|
||||
// stdout.
|
||||
path string
|
||||
}
|
||||
|
||||
func parseLoadTestOutputs(outputs []string) ([]loadTestOutput, error) {
|
||||
var stdoutFormat loadTestOutputFormat
|
||||
|
||||
validFormats := map[loadTestOutputFormat]struct{}{
|
||||
loadTestOutputFormatText: {},
|
||||
loadTestOutputFormatJSON: {},
|
||||
}
|
||||
|
||||
var out []loadTestOutput
|
||||
for i, o := range outputs {
|
||||
parts := strings.SplitN(o, ":", 2)
|
||||
format := loadTestOutputFormat(parts[0])
|
||||
if _, ok := validFormats[format]; !ok {
|
||||
return nil, xerrors.Errorf("invalid output format %q in output flag %d", parts[0], i)
|
||||
}
|
||||
|
||||
if len(parts) == 1 {
|
||||
if stdoutFormat != "" {
|
||||
return nil, xerrors.Errorf("multiple output flags specified for stdout")
|
||||
}
|
||||
stdoutFormat = format
|
||||
continue
|
||||
}
|
||||
if len(parts) != 2 {
|
||||
return nil, xerrors.Errorf("invalid output flag %d: %q", i, o)
|
||||
}
|
||||
|
||||
out = append(out, loadTestOutput{
|
||||
format: format,
|
||||
path: parts[1],
|
||||
})
|
||||
}
|
||||
|
||||
// Default to --output text
|
||||
if stdoutFormat == "" && len(out) == 0 {
|
||||
stdoutFormat = loadTestOutputFormatText
|
||||
}
|
||||
|
||||
if stdoutFormat != "" {
|
||||
out = append([]loadTestOutput{{
|
||||
format: stdoutFormat,
|
||||
path: "-",
|
||||
}}, out...)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/coder/coder/coderd/coderdtest"
|
||||
"github.com/coder/coder/coderd/httpapi"
|
||||
"github.com/coder/coder/codersdk"
|
||||
"github.com/coder/coder/loadtest/harness"
|
||||
"github.com/coder/coder/loadtest/placebo"
|
||||
"github.com/coder/coder/loadtest/workspacebuild"
|
||||
"github.com/coder/coder/pty/ptytest"
|
||||
@@ -133,4 +135,161 @@ func TestLoadTest(t *testing.T) {
|
||||
<-done
|
||||
cancelFunc()
|
||||
})
|
||||
|
||||
t.Run("OutputFormats", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
type outputFlag struct {
|
||||
format string
|
||||
path string
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
outputs []outputFlag
|
||||
errContains string
|
||||
}{
|
||||
{
|
||||
name: "Default",
|
||||
outputs: []outputFlag{},
|
||||
},
|
||||
{
|
||||
name: "ExplicitText",
|
||||
outputs: []outputFlag{{format: "text"}},
|
||||
},
|
||||
{
|
||||
name: "JSON",
|
||||
outputs: []outputFlag{
|
||||
{
|
||||
format: "json",
|
||||
path: filepath.Join(dir, "results.json"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "TextAndJSON",
|
||||
outputs: []outputFlag{
|
||||
{
|
||||
format: "text",
|
||||
},
|
||||
{
|
||||
format: "json",
|
||||
path: filepath.Join(dir, "results.json"),
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "TextAndJSON2",
|
||||
outputs: []outputFlag{
|
||||
{
|
||||
format: "text",
|
||||
},
|
||||
{
|
||||
format: "text",
|
||||
path: filepath.Join(dir, "results.txt"),
|
||||
},
|
||||
{
|
||||
format: "json",
|
||||
path: filepath.Join(dir, "results.json"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
client := coderdtest.New(t, nil)
|
||||
_ = coderdtest.CreateFirstUser(t, client)
|
||||
|
||||
config := cli.LoadTestConfig{
|
||||
Strategy: cli.LoadTestStrategy{
|
||||
Type: cli.LoadTestStrategyTypeLinear,
|
||||
},
|
||||
Tests: []cli.LoadTest{
|
||||
{
|
||||
Type: cli.LoadTestTypePlacebo,
|
||||
Count: 10,
|
||||
Placebo: &placebo.Config{
|
||||
Sleep: httpapi.Duration(10 * time.Millisecond),
|
||||
},
|
||||
},
|
||||
},
|
||||
Timeout: httpapi.Duration(testutil.WaitShort),
|
||||
}
|
||||
|
||||
configBytes, err := json.Marshal(config)
|
||||
require.NoError(t, err)
|
||||
|
||||
args := []string{"loadtest", "--config", "-"}
|
||||
for _, output := range c.outputs {
|
||||
flag := output.format
|
||||
if output.path != "" {
|
||||
flag += ":" + output.path
|
||||
}
|
||||
args = append(args, "--output", flag)
|
||||
}
|
||||
|
||||
cmd, root := clitest.New(t, args...)
|
||||
clitest.SetupConfig(t, client, root)
|
||||
cmd.SetIn(bytes.NewReader(configBytes))
|
||||
out := bytes.NewBuffer(nil)
|
||||
cmd.SetOut(out)
|
||||
pty := ptytest.New(t)
|
||||
cmd.SetErr(pty.Output())
|
||||
|
||||
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitLong)
|
||||
defer cancelFunc()
|
||||
|
||||
done := make(chan any)
|
||||
go func() {
|
||||
errC := cmd.ExecuteContext(ctx)
|
||||
if c.errContains != "" {
|
||||
assert.Error(t, errC)
|
||||
assert.Contains(t, errC.Error(), c.errContains)
|
||||
} else {
|
||||
assert.NoError(t, errC)
|
||||
}
|
||||
close(done)
|
||||
}()
|
||||
|
||||
<-done
|
||||
|
||||
if c.errContains != "" {
|
||||
return
|
||||
}
|
||||
if len(c.outputs) == 0 {
|
||||
// This is the default output format when no flags are
|
||||
// specified.
|
||||
c.outputs = []outputFlag{{format: "text"}}
|
||||
}
|
||||
for i, output := range c.outputs {
|
||||
msg := fmt.Sprintf("flag %d", i)
|
||||
var b []byte
|
||||
if output.path == "" {
|
||||
b = out.Bytes()
|
||||
} else {
|
||||
b, err = os.ReadFile(output.path)
|
||||
require.NoError(t, err, msg)
|
||||
}
|
||||
|
||||
switch output.format {
|
||||
case "text":
|
||||
require.Contains(t, string(b), "Test results:", msg)
|
||||
require.Contains(t, string(b), "Pass: 10", msg)
|
||||
case "json":
|
||||
var res harness.Results
|
||||
err = json.Unmarshal(b, &res)
|
||||
require.NoError(t, err, msg)
|
||||
require.Equal(t, 10, res.TotalRuns, msg)
|
||||
require.Equal(t, 10, res.TotalPass, msg)
|
||||
require.Len(t, res.Runs, 10, msg)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user