feat: provisionerd tracing, add public trace ingestion (#4070)

coderd/tracing/exporter.go

@@ -5,37 +5,74 @@ import (
 
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
 	"golang.org/x/xerrors"
 )
 
+// TracerOpts specifies which telemetry exporters should be configured.
+type TracerOpts struct {
+	// Default exports to a backend configured by environment variables. See:
+	// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
+	Default bool
+	// Coder exports traces to Coder's public tracing ingest service and is used
+	// to improve the product. It is disabled when opting out of telemetry.
+	Coder bool
+}
+
 // TracerProvider creates a grpc otlp exporter and configures a trace provider.
 // Caller is responsible for calling TracerProvider.Shutdown to ensure all data is flushed.
-func TracerProvider(ctx context.Context, service string) (*sdktrace.TracerProvider, error) {
-	res, err := resource.New(ctx,
-		resource.WithAttributes(
-			// the service name used to display traces in backends
-			semconv.ServiceNameKey.String(service),
-		),
+func TracerProvider(ctx context.Context, service string, opts TracerOpts) (*sdktrace.TracerProvider, error) {
+	res := resource.NewWithAttributes(
+		semconv.SchemaURL,
+		// the service name used to display traces in backends
+		semconv.ServiceNameKey.String(service),
 	)
-	if err != nil {
-		return nil, xerrors.Errorf("creating otlp resource: %w", err)
-	}
 
-	// By default we send span data to a local otel collector.
-	// The endpoint we push to can be configured with env vars.
-	// See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md
-	exporter, err := otlptrace.New(ctx, otlptracegrpc.NewClient(otlptracegrpc.WithInsecure()))
-	if err != nil {
-		return nil, xerrors.Errorf("creating otlp exporter: %w", err)
-	}
-
-	tracerProvider := sdktrace.NewTracerProvider(
-		sdktrace.WithBatcher(exporter),
+	tracerOpts := []sdktrace.TracerProviderOption{
 		sdktrace.WithResource(res),
-	)
+	}
+	if opts.Default {
+		exporter, err := DefaultExporter(ctx)
+		if err != nil {
+			return nil, xerrors.Errorf("default exporter: %w", err)
+		}
+		tracerOpts = append(tracerOpts, sdktrace.WithBatcher(exporter))
+	}
+	if opts.Coder {
+		exporter, err := CoderExporter(ctx)
+		if err != nil {
+			return nil, xerrors.Errorf("coder exporter: %w", err)
+		}
+		tracerOpts = append(tracerOpts, sdktrace.WithBatcher(exporter))
+	}
+
+	tracerProvider := sdktrace.NewTracerProvider(tracerOpts...)
 
 	return tracerProvider, nil
 }
+
+func DefaultExporter(ctx context.Context) (*otlptrace.Exporter, error) {
+	exporter, err := otlptrace.New(ctx, otlptracegrpc.NewClient(otlptracegrpc.WithInsecure()))
+	if err != nil {
+		return nil, xerrors.Errorf("create otlp exporter: %w", err)
+	}
+
+	return exporter, nil
+}
+
+func CoderExporter(ctx context.Context) (*otlptrace.Exporter, error) {
+	opts := []otlptracehttp.Option{
+		otlptracehttp.WithEndpoint("oss-otel-ingest-http.coder.app:443"),
+		otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
+	}
+
+	exporter, err := otlptrace.New(ctx, otlptracehttp.NewClient(opts...))
+	if err != nil {
+		return nil, xerrors.Errorf("create otlp exporter: %w", err)
+	}
+
+	return exporter, nil
+}

coderd/tracing/httpmw.go

@@ -5,7 +5,6 @@ import (
 	"net/http"
 
 	"github.com/go-chi/chi/v5"
-	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
 	"go.opentelemetry.io/otel/trace"
 
@@ -13,7 +12,7 @@ import (
 )
 
 // HTTPMW adds tracing to http routes.
-func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Handler) http.Handler {
+func HTTPMW(tracerProvider trace.TracerProvider, name string) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
 			if tracerProvider == nil {
@@ -34,20 +33,15 @@ func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Hand
 			// pass the span through the request context and serve the request to the next middleware
 			next.ServeHTTP(sw, r)
 			// capture response data
-			EndHTTPSpan(r, sw.Status)
+			EndHTTPSpan(r, sw.Status, span)
 		})
 	}
 }
 
 // EndHTTPSpan captures request and response data after the handler is done.
-func EndHTTPSpan(r *http.Request, status int) {
-	span := trace.SpanFromContext(r.Context())
-
+func EndHTTPSpan(r *http.Request, status int, span trace.Span) {
 	// set the resource name as we get it only once the handler is executed
 	route := chi.RouteContext(r.Context()).RoutePattern()
-	if route != "" {
-		span.SetName(fmt.Sprintf("%s %s", r.Method, route))
-	}
 	span.SetName(fmt.Sprintf("%s %s", r.Method, route))
 	span.SetAttributes(semconv.NetAttributesFromHTTPRequest("tcp", r)...)
 	span.SetAttributes(semconv.EndUserAttributesFromHTTPRequest(r)...)

coderd/tracing/postgres.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 
 	"github.com/nhatthm/otelsql"
-	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 )

coderd/tracing/util.go

@@ -0,0 +1,19 @@
+package tracing
+
+import (
+	"runtime"
+	"strings"
+)
+
+func FuncName() string {
+	fnpc, _, _, ok := runtime.Caller(1)
+	if !ok {
+		return ""
+	}
+	fn := runtime.FuncForPC(fnpc)
+	name := fn.Name()
+	if i := strings.LastIndex(name, "/"); i > 0 {
+		name = name[i+1:]
+	}
+	return name
+}

coderd/tracing/util_test.go

@@ -0,0 +1,39 @@
+package tracing_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/coderd/tracing"
+)
+
+// t.Parallel affects the result of these tests.
+
+//nolint:paralleltest
+func TestFuncName(t *testing.T) {
+	fn := tracing.FuncName()
+	assert.Equal(t, "tracing_test.TestFuncName", fn)
+}
+
+type foo struct{}
+
+func (foo) bar() string {
+	return tracing.FuncName()
+}
+
+//nolint:paralleltest
+func TestFuncNameMethod(t *testing.T) {
+	fn := foo{}.bar()
+	assert.Equal(t, "tracing_test.foo.bar", fn)
+}
+
+func (*foo) baz() string {
+	return tracing.FuncName()
+}
+
+//nolint:paralleltest
+func TestFuncNameMethodPointer(t *testing.T) {
+	fn := (&foo{}).baz()
+	assert.Equal(t, "tracing_test.(*foo).baz", fn)
+}