From eb4a1e2568748d9c189564d067540d3cc89c4eb5 Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 28 Feb 2024 09:21:56 +0100 Subject: [PATCH] feat: enable Prometheus endpoint for external provisioner (#12320) --- docs/cli/provisionerd_start.md | 10 +++ enterprise/cli/provisionerdaemons.go | 31 +++++++ enterprise/cli/provisionerdaemons_test.go | 89 +++++++++++++++++++ .../coder_provisionerd_start_--help.golden | 3 + 4 files changed, 133 insertions(+) diff --git a/docs/cli/provisionerd_start.md b/docs/cli/provisionerd_start.md index 643707f621..4decf76915 100644 --- a/docs/cli/provisionerd_start.md +++ b/docs/cli/provisionerd_start.md @@ -88,6 +88,16 @@ Deprecated and ignored. Deprecated and ignored. +### --prometheus-address + +| | | +| ----------- | -------------------------------------- | +| Type | string | +| Environment | $CODER_PROMETHEUS_ADDRESS | +| Default | 127.0.0.1:2112 | + +The bind address to serve prometheus metrics. + ### --psk | | | diff --git a/enterprise/cli/provisionerdaemons.go b/enterprise/cli/provisionerdaemons.go index f48b82e239..37b9a629ea 100644 --- a/enterprise/cli/provisionerdaemons.go +++ b/enterprise/cli/provisionerdaemons.go @@ -10,6 +10,9 @@ import ( "time" "github.com/google/uuid" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/xerrors" "cdr.dev/slog" @@ -67,6 +70,8 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd { pollJitter time.Duration preSharedKey string verbose bool + + prometheusAddress string ) client := new(codersdk.Client) cmd := &clibase.Cmd{ @@ -165,6 +170,24 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd { } }() + var metrics *provisionerd.Metrics + if prometheusAddress != "" { + logger.Info(ctx, "starting Prometheus endpoint", slog.F("address", prometheusAddress)) + + prometheusRegistry := prometheus.NewRegistry() + prometheusRegistry.MustRegister(collectors.NewGoCollector()) + prometheusRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + + m := provisionerd.NewMetrics(prometheusRegistry) + m.Runner.NumDaemons.Set(float64(1)) // Set numDaemons to 1 as this is standalone mode. + metrics = &m + + closeFunc := agpl.ServeHandler(ctx, logger, promhttp.InstrumentMetricHandler( + prometheusRegistry, promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{}), + ), prometheusAddress, "prometheus") + defer closeFunc() + } + logger.Info(ctx, "starting provisioner daemon", slog.F("tags", tags), slog.F("name", name)) connector := provisionerd.LocalProvisioners{ @@ -185,6 +208,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd { Logger: logger, UpdateInterval: 500 * time.Millisecond, Connector: connector, + Metrics: metrics, }) var exitErr error @@ -291,6 +315,13 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd { Value: clibase.StringArrayOf(&logFilter), Default: "", }, + { + Flag: "prometheus-address", + Env: "CODER_PROMETHEUS_ADDRESS", + Description: "The bind address to serve prometheus metrics.", + Value: clibase.StringOf(&prometheusAddress), + Default: "127.0.0.1:2112", + }, } return cmd diff --git a/enterprise/cli/provisionerdaemons_test.go b/enterprise/cli/provisionerdaemons_test.go index 98526d1014..e0999a4ef8 100644 --- a/enterprise/cli/provisionerdaemons_test.go +++ b/enterprise/cli/provisionerdaemons_test.go @@ -1,7 +1,12 @@ package cli_test import ( + "bufio" "context" + "fmt" + "net" + "net/http" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -161,4 +166,88 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) { assert.Equal(t, buildinfo.Version(), daemons[0].Version) assert.Equal(t, proto.CurrentVersion.String(), daemons[0].APIVersion) }) + + t.Run("PrometheusEnabled", func(t *testing.T) { + t.Parallel() + + // Helper function to find a free random port + randomPort := func(t *testing.T) int { + random, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + _ = random.Close() + tcpAddr, valid := random.Addr().(*net.TCPAddr) + require.True(t, valid) + return tcpAddr.Port + } + prometheusPort := randomPort(t) + + // Configure CLI client + client, admin := coderdenttest.New(t, &coderdenttest.Options{ + ProvisionerDaemonPSK: "provisionersftw", + LicenseOptions: &coderdenttest.LicenseOptions{ + Features: license.Features{ + codersdk.FeatureExternalProvisionerDaemons: 1, + }, + }, + }) + anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID, rbac.RoleTemplateAdmin()) + inv, conf := newCLI(t, "provisionerd", "start", "--name", "daemon-with-prometheus", "--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort)) + clitest.SetupConfig(t, anotherClient, conf) + pty := ptytest.New(t).Attach(inv) + ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong) + defer cancel() + + // Start "provisionerd" command + clitest.Start(t, inv) + pty.ExpectMatchContext(ctx, "starting provisioner daemon") + + var daemons []codersdk.ProvisionerDaemon + var err error + require.Eventually(t, func() bool { + daemons, err = client.ProvisionerDaemons(ctx) + if err != nil { + return false + } + return len(daemons) == 1 + }, testutil.WaitShort, testutil.IntervalSlow) + require.Equal(t, "daemon-with-prometheus", daemons[0].Name) + + // Fetch metrics from Prometheus endpoint + var res *http.Response + require.Eventually(t, func() bool { + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", prometheusPort), nil) + assert.NoError(t, err) + // nolint:bodyclose + res, err = http.DefaultClient.Do(req) + return err == nil + }, testutil.WaitShort, testutil.IntervalFast) + defer res.Body.Close() + + // Scan for metric patterns + scanner := bufio.NewScanner(res.Body) + hasOneDaemon := false + hasGoStats := false + hasPromHTTP := false + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "coderd_provisionerd_num_daemons 1") { + hasOneDaemon = true + continue + } + if strings.HasPrefix(scanner.Text(), "go_goroutines") { + hasGoStats = true + continue + } + if strings.HasPrefix(scanner.Text(), "promhttp_metric_handler_requests_total") { + hasPromHTTP = true + continue + } + t.Logf("scanned %s", scanner.Text()) + } + require.NoError(t, scanner.Err()) + + // Verify patterns + require.True(t, hasOneDaemon, "should be one daemon running") + require.True(t, hasGoStats, "Go stats are missing") + require.True(t, hasPromHTTP, "Prometheus HTTP metrics are missing") + }) } diff --git a/enterprise/cli/testdata/coder_provisionerd_start_--help.golden b/enterprise/cli/testdata/coder_provisionerd_start_--help.golden index 042e788c12..fcf2e5ef2f 100644 --- a/enterprise/cli/testdata/coder_provisionerd_start_--help.golden +++ b/enterprise/cli/testdata/coder_provisionerd_start_--help.golden @@ -32,6 +32,9 @@ OPTIONS: --poll-jitter duration, $CODER_PROVISIONERD_POLL_JITTER (default: 100ms) Deprecated and ignored. + --prometheus-address string, $CODER_PROMETHEUS_ADDRESS (default: 127.0.0.1:2112) + The bind address to serve prometheus metrics. + --psk string, $CODER_PROVISIONER_DAEMON_PSK Pre-shared key to authenticate with Coder server.