opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): add dashboard load test command (#8723)

Browse Source
This commit is contained in:
Cian Johnston
2023-07-27 09:40:13 +01:00
committed by GitHub
parent c3aface285
commit 32829080ac
10 changed files with 995 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/exp_scaletest.go
+160 -18
View File
@@ -30,6 +30,7 @@ import (
"github.com/coder/coder/cryptorand"
"github.com/coder/coder/scaletest/agentconn"
"github.com/coder/coder/scaletest/createworkspaces"
"github.com/coder/coder/scaletest/dashboard"
"github.com/coder/coder/scaletest/harness"
"github.com/coder/coder/scaletest/reconnectingpty"
"github.com/coder/coder/scaletest/workspacebuild"
@@ -47,6 +48,7 @@ func (r *RootCmd) scaletestCmd() *clibase.Cmd {
},
Children: []*clibase.Cmd{
r.scaletestCleanup(),
r.scaletestDashboard(),
r.scaletestCreateWorkspaces(),
r.scaletestWorkspaceTraffic(),
},
@@ -317,6 +319,30 @@ func (s *scaletestOutputFlags) parse() ([]scaleTestOutput, error) {
return out, nil
}
type scaletestPrometheusFlags struct {
Address string
Wait time.Duration
}
func (s *scaletestPrometheusFlags) attach(opts *clibase.OptionSet) {
*opts = append(*opts,
clibase.Option{
Flag: "scaletest-prometheus-address",
Env: "CODER_SCALETEST_PROMETHEUS_ADDRESS",
Default: "0.0.0.0:21112",
Description: "Address on which to expose scaletest Prometheus metrics.",
Value: clibase.StringOf(&s.Address),
},
clibase.Option{
Flag: "scaletest-prometheus-wait",
Env: "CODER_SCALETEST_PROMETHEUS_WAIT",
Default: "15s",
Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.",
Value: clibase.DurationOf(&s.Wait),
},
)
}
func requireAdmin(ctx context.Context, client *codersdk.Client) (codersdk.User, error) {
me, err := client.User(ctx, codersdk.Me)
if err != nil {
@@ -846,17 +872,16 @@ func (r *RootCmd) scaletestCreateWorkspaces() *clibase.Cmd {
func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
var (
tickInterval time.Duration
bytesPerTick int64
ssh bool
scaletestPrometheusAddress string
scaletestPrometheusWait time.Duration
tickInterval time.Duration
bytesPerTick int64
ssh bool
client = &codersdk.Client{}
tracingFlags = &scaletestTracingFlags{}
strategy = &scaletestStrategyFlags{}
cleanupStrategy = &scaletestStrategyFlags{cleanup: true}
output = &scaletestOutputFlags{}
prometheusFlags = &scaletestPrometheusFlags{}
)
cmd := &clibase.Cmd{
@@ -871,7 +896,7 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
metrics := workspacetraffic.NewMetrics(reg, "username", "workspace_name", "agent_name")
logger := slog.Make(sloghuman.Sink(io.Discard))
prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), scaletestPrometheusAddress, "prometheus")
prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
defer prometheusSrvClose()
// Bypass rate limiting
@@ -905,8 +930,8 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
}
// Wait for prometheus metrics to be scraped
_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", scaletestPrometheusWait)
<-time.After(scaletestPrometheusWait)
_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
<-time.After(prometheusFlags.Wait)
}()
tracer := tracerProvider.Tracer(scaletestTracerName)
@@ -1009,19 +1034,135 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
Description: "Send traffic over SSH.",
Value: clibase.BoolOf(&ssh),
},
}
tracingFlags.attach(&cmd.Options)
strategy.attach(&cmd.Options)
cleanupStrategy.attach(&cmd.Options)
output.attach(&cmd.Options)
prometheusFlags.attach(&cmd.Options)
return cmd
}
func (r *RootCmd) scaletestDashboard() *clibase.Cmd {
var (
count int64
minWait time.Duration
maxWait time.Duration
client = &codersdk.Client{}
tracingFlags = &scaletestTracingFlags{}
strategy = &scaletestStrategyFlags{}
cleanupStrategy = &scaletestStrategyFlags{cleanup: true}
output = &scaletestOutputFlags{}
prometheusFlags = &scaletestPrometheusFlags{}
)
cmd := &clibase.Cmd{
Use: "dashboard",
Short: "Generate traffic to the HTTP API to simulate use of the dashboard.",
Middleware: clibase.Chain(
r.InitClient(client),
),
Handler: func(inv *clibase.Invocation) error {
ctx := inv.Context()
logger := slog.Make(sloghuman.Sink(inv.Stdout)).Leveled(slog.LevelInfo)
tracerProvider, closeTracing, tracingEnabled, err := tracingFlags.provider(ctx)
if err != nil {
return xerrors.Errorf("create tracer provider: %w", err)
}
defer func() {
// Allow time for traces to flush even if command context is
// canceled. This is a no-op if tracing is not enabled.
_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
if err := closeTracing(ctx); err != nil {
_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
}
// Wait for prometheus metrics to be scraped
_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
<-time.After(prometheusFlags.Wait)
}()
tracer := tracerProvider.Tracer(scaletestTracerName)
outputs, err := output.parse()
if err != nil {
return xerrors.Errorf("could not parse --output flags")
}
reg := prometheus.NewRegistry()
prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
defer prometheusSrvClose()
metrics := dashboard.NewMetrics(reg)
th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy())
for i := int64(0); i < count; i++ {
name := fmt.Sprintf("dashboard-%d", i)
config := dashboard.Config{
MinWait: minWait,
MaxWait: maxWait,
Trace: tracingEnabled,
Logger: logger.Named(name),
RollTable: dashboard.DefaultActions,
}
if err := config.Validate(); err != nil {
return err
}
var runner harness.Runnable = dashboard.NewRunner(client, metrics, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
spanName: name,
runner: runner,
}
}
th.AddRun("dashboard", name, runner)
}
_, _ = fmt.Fprintln(inv.Stderr, "Running load test...")
testCtx, testCancel := strategy.toContext(ctx)
defer testCancel()
err = th.Run(testCtx)
if err != nil {
return xerrors.Errorf("run test harness (harness failure, not a test failure): %w", err)
}
res := th.Results()
for _, o := range outputs {
err = o.write(res, inv.Stdout)
if err != nil {
return xerrors.Errorf("write output %q to %q: %w", o.format, o.path, err)
}
}
if res.TotalFail > 0 {
return xerrors.New("load test failed, see above for more details")
}
return nil
},
}
cmd.Options = []clibase.Option{
{
Flag: "scaletest-prometheus-address",
Env: "CODER_SCALETEST_PROMETHEUS_ADDRESS",
Default: "0.0.0.0:21112",
Description: "Address on which to expose scaletest Prometheus metrics.",
Value: clibase.StringOf(&scaletestPrometheusAddress),
Flag: "count",
Env: "CODER_SCALETEST_DASHBOARD_COUNT",
Default: "1",
Description: "Number of concurrent workers.",
Value: clibase.Int64Of(&count),
},
{
Flag: "scaletest-prometheus-wait",
Env: "CODER_SCALETEST_PROMETHEUS_WAIT",
Default: "5s",
Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.",
Value: clibase.DurationOf(&scaletestPrometheusWait),
Flag: "min-wait",
Env: "CODER_SCALETEST_DASHBOARD_MIN_WAIT",
Default: "100ms",
Description: "Minimum wait between fetches.",
Value: clibase.DurationOf(&minWait),
},
{
Flag: "max-wait",
Env: "CODER_SCALETEST_DASHBOARD_MAX_WAIT",
Default: "1s",
Description: "Maximum wait between fetches.",
Value: clibase.DurationOf(&maxWait),
},
}
@@ -1029,6 +1170,7 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
strategy.attach(&cmd.Options)
cleanupStrategy.attach(&cmd.Options)
output.attach(&cmd.Options)
prometheusFlags.attach(&cmd.Options)
return cmd
}
cli/exp_scaletest_test.go
+26
View File
@@ -78,3 +78,29 @@ func TestScaleTestWorkspaceTraffic(t *testing.T) {
err := inv.WithContext(ctx).Run()
require.ErrorContains(t, err, "no scaletest workspaces exist")
}
// This test just validates that the CLI command accepts its known arguments.
func TestScaleTestDashboard(t *testing.T) {
t.Parallel()
ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitMedium)
defer cancelFunc()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
inv, root := clitest.New(t, "exp", "scaletest", "dashboard",
"--count", "1",
"--min-wait", "100ms",
"--max-wait", "1s",
"--timeout", "1s",
"--scaletest-prometheus-address", "127.0.0.1:0",
"--scaletest-prometheus-wait", "0s",
)
clitest.SetupConfig(t, client, root)
var stdout, stderr bytes.Buffer
inv.Stdout = &stdout
inv.Stderr = &stderr
err := inv.WithContext(ctx).Run()
require.NoError(t, err, "")
}