feat: provisionerd tracing, add public trace ingestion (#4070)

coderd/coderd.go

@@ -15,7 +15,7 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"github.com/pion/webrtc/v3"
 	"github.com/prometheus/client_golang/prometheus"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 	"google.golang.org/api/idtoken"
 	"tailscale.com/derp"
@@ -70,7 +70,7 @@ type Options struct {
 	SSHKeygenAlgorithm   gitsshkey.Algorithm
 	Telemetry            telemetry.Reporter
 	TURNServer           *turnconn.Server
-	TracerProvider       *sdktrace.TracerProvider
+	TracerProvider       trace.TracerProvider
 	AutoImportTemplates  []AutoImportTemplate
 	LicenseHandler       http.Handler
 	FeaturesService      features.Service

coderd/tracing/exporter.go

@@ -5,37 +5,74 @@ import (
 
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
 	"golang.org/x/xerrors"
 )
 
+// TracerOpts specifies which telemetry exporters should be configured.
+type TracerOpts struct {
+	// Default exports to a backend configured by environment variables. See:
+	// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
+	Default bool
+	// Coder exports traces to Coder's public tracing ingest service and is used
+	// to improve the product. It is disabled when opting out of telemetry.
+	Coder bool
+}
+
 // TracerProvider creates a grpc otlp exporter and configures a trace provider.
 // Caller is responsible for calling TracerProvider.Shutdown to ensure all data is flushed.
-func TracerProvider(ctx context.Context, service string) (*sdktrace.TracerProvider, error) {
-	res, err := resource.New(ctx,
-		resource.WithAttributes(
-			// the service name used to display traces in backends
-			semconv.ServiceNameKey.String(service),
-		),
+func TracerProvider(ctx context.Context, service string, opts TracerOpts) (*sdktrace.TracerProvider, error) {
+	res := resource.NewWithAttributes(
+		semconv.SchemaURL,
+		// the service name used to display traces in backends
+		semconv.ServiceNameKey.String(service),
 	)
-	if err != nil {
-		return nil, xerrors.Errorf("creating otlp resource: %w", err)
-	}
 
-	// By default we send span data to a local otel collector.
-	// The endpoint we push to can be configured with env vars.
-	// See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
-	exporter, err := otlptrace.New(ctx, otlptracegrpc.NewClient(otlptracegrpc.WithInsecure()))
-	if err != nil {
-		return nil, xerrors.Errorf("creating otlp exporter: %w", err)
-	}
-
-	tracerProvider := sdktrace.NewTracerProvider(
-		sdktrace.WithBatcher(exporter),
+	tracerOpts := []sdktrace.TracerProviderOption{
 		sdktrace.WithResource(res),
-	)
+	}
+	if opts.Default {
+		exporter, err := DefaultExporter(ctx)
+		if err != nil {
+			return nil, xerrors.Errorf("default exporter: %w", err)
+		}
+		tracerOpts = append(tracerOpts, sdktrace.WithBatcher(exporter))
+	}
+	if opts.Coder {
+		exporter, err := CoderExporter(ctx)
+		if err != nil {
+			return nil, xerrors.Errorf("coder exporter: %w", err)
+		}
+		tracerOpts = append(tracerOpts, sdktrace.WithBatcher(exporter))
+	}
+
+	tracerProvider := sdktrace.NewTracerProvider(tracerOpts...)
 
 	return tracerProvider, nil
 }
+
+func DefaultExporter(ctx context.Context) (*otlptrace.Exporter, error) {
+	exporter, err := otlptrace.New(ctx, otlptracegrpc.NewClient(otlptracegrpc.WithInsecure()))
+	if err != nil {
+		return nil, xerrors.Errorf("create otlp exporter: %w", err)
+	}
+
+	return exporter, nil
+}
+
+func CoderExporter(ctx context.Context) (*otlptrace.Exporter, error) {
+	opts := []otlptracehttp.Option{
+		otlptracehttp.WithEndpoint("oss-otel-ingest-http.coder.app:443"),
+		otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
+	}
+
+	exporter, err := otlptrace.New(ctx, otlptracehttp.NewClient(opts...))
+	if err != nil {
+		return nil, xerrors.Errorf("create otlp exporter: %w", err)
+	}
+
+	return exporter, nil
+}

coderd/tracing/httpmw.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 
 	"github.com/go-chi/chi/v5"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
 	"go.opentelemetry.io/otel/trace"
 
@@ -13,7 +12,7 @@ import (
 )
 
 // HTTPMW adds tracing to http routes.
-func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Handler) http.Handler {
+func HTTPMW(tracerProvider trace.TracerProvider, name string) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 			if tracerProvider == nil {
@@ -34,20 +33,15 @@ func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Hand
 			// pass the span through the request context and serve the request to the next middleware
 			next.ServeHTTP(sw, r)
 			// capture response data
-			EndHTTPSpan(r, sw.Status)
+			EndHTTPSpan(r, sw.Status, span)
 		})
 	}
 }
 
 // EndHTTPSpan captures request and response data after the handler is done.
-func EndHTTPSpan(r *http.Request, status int) {
-	span := trace.SpanFromContext(r.Context())
-
+func EndHTTPSpan(r *http.Request, status int, span trace.Span) {
 	// set the resource name as we get it only once the handler is executed
 	route := chi.RouteContext(r.Context()).RoutePattern()
-	if route != "" {
-		span.SetName(fmt.Sprintf("%s %s", r.Method, route))
-	}
 	span.SetName(fmt.Sprintf("%s %s", r.Method, route))
 	span.SetAttributes(semconv.NetAttributesFromHTTPRequest("tcp", r)...)
 	span.SetAttributes(semconv.EndUserAttributesFromHTTPRequest(r)...)

coderd/tracing/postgres.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/nhatthm/otelsql"
-	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 )

coderd/tracing/util.go

@@ -0,0 +1,19 @@
+package tracing
+
+import (
+	"runtime"
+	"strings"
+)
+
+func FuncName() string {
+	fnpc, _, _, ok := runtime.Caller(1)
+	if !ok {
+		return ""
+	}
+	fn := runtime.FuncForPC(fnpc)
+	name := fn.Name()
+	if i := strings.LastIndex(name, "/"); i > 0 {
+		name = name[i+1:]
+	}
+	return name
+}

coderd/tracing/util_test.go

@@ -0,0 +1,39 @@
+package tracing_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/coderd/tracing"
+)
+
+// t.Parallel affects the result of these tests.
+
+//nolint:paralleltest
+func TestFuncName(t *testing.T) {
+	fn := tracing.FuncName()
+	assert.Equal(t, "tracing_test.TestFuncName", fn)
+}
+
+type foo struct{}
+
+func (foo) bar() string {
+	return tracing.FuncName()
+}
+
+//nolint:paralleltest
+func TestFuncNameMethod(t *testing.T) {
+	fn := foo{}.bar()
+	assert.Equal(t, "tracing_test.foo.bar", fn)
+}
+
+func (*foo) baz() string {
+	return tracing.FuncName()
+}
+
+//nolint:paralleltest
+func TestFuncNameMethodPointer(t *testing.T) {
+	fn := (&foo{}).baz()
+	assert.Equal(t, "tracing_test.(*foo).baz", fn)
+}

coderd/workspaceagents.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/hashicorp/yamux"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/mod/semver"
 	"golang.org/x/xerrors"
 	"nhooyr.io/websocket"
@@ -113,7 +114,7 @@ func (api *API) workspaceAgentDial(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	// end span so we don't get long lived trace data
-	tracing.EndHTTPSpan(r, 200)
+	tracing.EndHTTPSpan(r, http.StatusOK, trace.SpanFromContext(ctx))
 
 	err = peerbroker.ProxyListen(ctx, session, peerbroker.ProxyOptions{
 		ChannelID: workspaceAgent.ID.String(),
@@ -309,7 +310,7 @@ func (api *API) workspaceAgentListen(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	// end span so we don't get long lived trace data
-	tracing.EndHTTPSpan(r, 200)
+	tracing.EndHTTPSpan(r, http.StatusOK, trace.SpanFromContext(ctx))
 
 	api.Logger.Info(ctx, "accepting agent", slog.F("resource", resource), slog.F("agent", workspaceAgent))
 
@@ -398,8 +399,9 @@ func (api *API) workspaceAgentTurn(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	ctx, wsNetConn := websocketNetConn(r.Context(), wsConn, websocket.MessageBinary)
-	defer wsNetConn.Close()     // Also closes conn.
-	tracing.EndHTTPSpan(r, 200) // end span so we don't get long lived trace data
+	defer wsNetConn.Close() // Also closes conn.
+	// end span so we don't get long lived trace data
+	tracing.EndHTTPSpan(r, http.StatusOK, trace.SpanFromContext(ctx))
 
 	api.Logger.Debug(ctx, "accepting turn connection", slog.F("remote-address", r.RemoteAddr), slog.F("local-address", localAddress))
 	select {

coderd/workspaceapps.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/go-chi/chi/v5"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/coder/coder/coderd/database"
 	"github.com/coder/coder/coderd/httpapi"
@@ -125,6 +126,7 @@ type proxyApplication struct {
 }
 
 func (api *API) proxyWorkspaceApplication(proxyApp proxyApplication, rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
 	if !api.Authorize(r, rbac.ActionCreate, proxyApp.Workspace.ExecutionRBAC()) {
 		httpapi.ResourceNotFound(rw)
 		return
@@ -138,7 +140,7 @@ func (api *API) proxyWorkspaceApplication(proxyApp proxyApplication, rw http.Res
 	// If the app name was used instead, fetch the app from the database so we
 	// can get the internal URL.
 	if proxyApp.AppName != "" {
-		app, err := api.Database.GetWorkspaceAppByAgentIDAndName(r.Context(), database.GetWorkspaceAppByAgentIDAndNameParams{
+		app, err := api.Database.GetWorkspaceAppByAgentIDAndName(ctx, database.GetWorkspaceAppByAgentIDAndNameParams{
 			AgentID: proxyApp.Agent.ID,
 			Name:    proxyApp.AppName,
 		})
@@ -195,7 +197,7 @@ func (api *API) proxyWorkspaceApplication(proxyApp proxyApplication, rw http.Res
 		if proxyApp.DashboardOnError {
 			// To pass friendly errors to the frontend, special meta tags are
 			// overridden in the index.html with the content passed here.
-			r = r.WithContext(site.WithAPIResponse(r.Context(), site.APIResponse{
+			r = r.WithContext(site.WithAPIResponse(ctx, site.APIResponse{
 				StatusCode: http.StatusBadGateway,
 				Message:    err.Error(),
 			}))
@@ -228,7 +230,7 @@ func (api *API) proxyWorkspaceApplication(proxyApp proxyApplication, rw http.Res
 	proxy.Transport = conn.HTTPTransport()
 
 	// end span so we don't get long lived trace data
-	tracing.EndHTTPSpan(r, 200)
+	tracing.EndHTTPSpan(r, http.StatusOK, trace.SpanFromContext(ctx))
 
 	proxy.ServeHTTP(rw, r)
 }