diff --git a/coderd/telemetry/telemetry.go b/coderd/telemetry/telemetry.go index e581c8b473..05af3f2e37 100644 --- a/coderd/telemetry/telemetry.go +++ b/coderd/telemetry/telemetry.go @@ -345,28 +345,41 @@ func (r *remoteReporter) deployment() error { scimEnabled := r.options.SCIMEnabled scimUseLegacy := r.options.SCIMUseLegacy + agentsExperimentValues := make(map[string]json.RawMessage, len(agentsExperiments)) + for _, exp := range agentsExperiments { + agentsExperimentValues[exp.name] = exp.collect(r.ctx, r.options) + } + agentsExperimentsJSON, err := json.Marshal(agentsExperimentValues) + if err != nil { + // Best-effort: the field is omitempty, so the deployment report + // proceeds without it. + r.options.Logger.Warn(r.ctx, "marshal agent experiments telemetry", slog.Error(err)) + agentsExperimentsJSON = nil + } + data, err := json.Marshal(&Deployment{ - ID: r.options.DeploymentID, - Architecture: sysInfo.Architecture, - BuiltinPostgres: r.options.BuiltinPostgres, - Containerized: containerized, - Config: r.options.DeploymentConfig, - Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", - InstallSource: installSource, - Tunnel: r.options.Tunnel, - OSType: sysInfo.OS.Type, - OSFamily: sysInfo.OS.Family, - OSPlatform: sysInfo.OS.Platform, - OSName: sysInfo.OS.Name, - OSVersion: sysInfo.OS.Version, - CPUCores: runtime.NumCPU(), - MemoryTotal: mem.Total, - MachineID: sysInfo.UniqueID, - StartedAt: r.startedAt, - ShutdownAt: r.shutdownAt, - IDPOrgSync: &idpOrgSync, - SCIMEnabled: &scimEnabled, - SCIMUseLegacy: &scimUseLegacy, + ID: r.options.DeploymentID, + Architecture: sysInfo.Architecture, + BuiltinPostgres: r.options.BuiltinPostgres, + Containerized: containerized, + Config: r.options.DeploymentConfig, + Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "", + InstallSource: installSource, + Tunnel: r.options.Tunnel, + OSType: sysInfo.OS.Type, + OSFamily: sysInfo.OS.Family, + OSPlatform: sysInfo.OS.Platform, + OSName: sysInfo.OS.Name, + OSVersion: sysInfo.OS.Version, + CPUCores: runtime.NumCPU(), + MemoryTotal: mem.Total, + MachineID: sysInfo.UniqueID, + StartedAt: r.startedAt, + ShutdownAt: r.shutdownAt, + IDPOrgSync: &idpOrgSync, + SCIMEnabled: &scimEnabled, + SCIMUseLegacy: &scimUseLegacy, + AgentsExperiments: agentsExperimentsJSON, }) if err != nil { return xerrors.Errorf("marshal deployment: %w", err) @@ -1660,6 +1673,10 @@ type Deployment struct { // enterprise/coderd/scim. Nullable for the same backward compatibility // reason as SCIMEnabled. SCIMUseLegacy *bool `json:"scim_use_legacy"` + // AgentsExperiments reports the state of the Coder Agents experiments as + // opaque per-experiment JSON, so rotating the reported set is a code-only + // change. Omitted by older Coder versions, so it decodes as nil there. + AgentsExperiments json.RawMessage `json:"agents_experiments,omitempty"` } type APIKey struct { @@ -2322,6 +2339,129 @@ const ( TelemetryItemKeyTelemetryEnabled telemetryItemKey = "telemetry_enabled" ) +// agentsExperiment is one entry in the Deployment.AgentsExperiments field. +// Edit agentsExperiments to rotate the reported set without schema or +// telemetry-server changes. Collectors are best-effort: they log and return +// a degraded payload instead of erroring, so they can never fail a report. +type agentsExperiment struct { + name string + collect func(ctx context.Context, opts Options) json.RawMessage +} + +var agentsExperiments = []agentsExperiment{ + {name: "virtual_desktop", collect: CollectAgentsVirtualDesktop}, + {name: "advisor", collect: CollectAgentsAdvisor}, +} + +const ( + // AgentsExperimentAdvisorReuseChatModel reports that the advisor has no active + // dedicated model override and reuses the chat model at runtime. + AgentsExperimentAdvisorReuseChatModel = "advisor_reuse_chat_model" + // AgentsExperimentUnknown reports a value that could not be determined, + // e.g. after a transient DB error. + AgentsExperimentUnknown = "unknown" +) + +// AgentsVirtualDesktopTelemetry is the value shape for the virtual_desktop +// entry in Deployment.AgentsExperiments. +type AgentsVirtualDesktopTelemetry struct { + Enabled bool `json:"enabled"` + ComputerUse AgentsComputerUseTelemetry `json:"computer_use"` +} + +type AgentsComputerUseTelemetry struct { + Provider string `json:"provider"` + ProviderSource string `json:"provider_source"` +} + +// AgentsAdvisorTelemetry is the value shape for the advisor entry in +// Deployment.AgentsExperiments. +type AgentsAdvisorTelemetry struct { + Enabled bool `json:"enabled"` + MaxUsesPerRun int `json:"max_uses_per_run"` + MaxOutputTokens int64 `json:"max_output_tokens"` + Provider string `json:"provider"` + Model string `json:"model"` +} + +// CollectAgentsVirtualDesktop collects the virtual_desktop entry in +// Deployment.AgentsExperiments. The chat-virtual-desktop experiment gates both +// the desktop and computer use. +func CollectAgentsVirtualDesktop(ctx context.Context, opts Options) json.RawMessage { + provider, err := opts.Database.GetChatComputerUseProvider(ctx) + providerSource := "configured" + switch { + case err != nil: + opts.Logger.Warn(ctx, "get chat computer use provider for telemetry", slog.Error(err)) + provider = AgentsExperimentUnknown + providerSource = AgentsExperimentUnknown + case provider == "": + provider = string(codersdk.ChatComputerUseProviderAnthropic) + providerSource = "default" + } + val, err := json.Marshal(AgentsVirtualDesktopTelemetry{ + Enabled: opts.Experiments.Enabled(codersdk.ExperimentChatVirtualDesktop), + ComputerUse: AgentsComputerUseTelemetry{ + Provider: provider, + ProviderSource: providerSource, + }, + }) + if err != nil { + opts.Logger.Warn(ctx, "marshal agent virtual desktop telemetry", slog.Error(err)) + return nil + } + return val +} + +// CollectAgentsAdvisor collects the advisor entry in +// Deployment.AgentsExperiments. +func CollectAgentsAdvisor(ctx context.Context, opts Options) json.RawMessage { + payload := AgentsAdvisorTelemetry{ + Enabled: opts.Experiments.Enabled(codersdk.ExperimentChatAdvisor), + Provider: AgentsExperimentUnknown, + Model: AgentsExperimentUnknown, + } + var cfg codersdk.AdvisorConfig + raw, err := opts.Database.GetChatAdvisorConfig(ctx) + if err != nil { + opts.Logger.Warn(ctx, "get chat advisor config for telemetry", slog.Error(err)) + } else if err := json.Unmarshal([]byte(raw), &cfg); err != nil { + opts.Logger.Warn(ctx, "parse chat advisor config for telemetry", slog.Error(err)) + } else { + payload.MaxUsesPerRun = max(cfg.MaxUsesPerRun, 0) + payload.MaxOutputTokens = max(cfg.MaxOutputTokens, 0) + payload.Provider, payload.Model = advisorModelTelemetry(ctx, opts.Database, opts.Logger, cfg.ModelConfigID) + } + val, err := json.Marshal(payload) + if err != nil { + opts.Logger.Warn(ctx, "marshal agent advisor telemetry", slog.Error(err)) + return nil + } + return val +} + +func advisorModelTelemetry(ctx context.Context, db database.Store, log slog.Logger, id uuid.UUID) (provider string, model string) { + if id == uuid.Nil { + return AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel + } + + cfg, err := db.GetEnabledChatModelConfigByID(ctx, id) + if errors.Is(err, sql.ErrNoRows) { + // An inactive override; the runtime falls back to the chat model. + return AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel + } + if err != nil { + log.Warn(ctx, "resolve chat advisor model config for telemetry", slog.Error(err)) + return AgentsExperimentUnknown, AgentsExperimentUnknown + } + providerRow, err := db.GetAIProviderByID(ctx, cfg.AIProviderID.UUID) + if err != nil { + log.Warn(ctx, "resolve chat advisor model provider for telemetry", slog.Error(err)) + return AgentsExperimentUnknown, cfg.Model + } + return string(providerRow.Type), cfg.Model +} + type TelemetryItem struct { Key string `json:"key"` Value string `json:"value"` diff --git a/coderd/telemetry/telemetry_test.go b/coderd/telemetry/telemetry_test.go index e6f05e9f8e..305c149a6b 100644 --- a/coderd/telemetry/telemetry_test.go +++ b/coderd/telemetry/telemetry_test.go @@ -20,18 +20,21 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" + "go.uber.org/mock/gomock" "github.com/coder/coder/v2/buildinfo" "github.com/coder/coder/v2/coderd/boundaryusage" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbfake" "github.com/coder/coder/v2/coderd/database/dbgen" + "github.com/coder/coder/v2/coderd/database/dbmock" "github.com/coder/coder/v2/coderd/database/dbtestutil" "github.com/coder/coder/v2/coderd/database/dbtime" "github.com/coder/coder/v2/coderd/idpsync" "github.com/coder/coder/v2/coderd/runtimeconfig" "github.com/coder/coder/v2/coderd/telemetry" "github.com/coder/coder/v2/coderd/util/ptr" + "github.com/coder/coder/v2/coderd/util/slice" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" @@ -292,10 +295,14 @@ func TestTelemetry(t *testing.T) { clock := quartz.NewMock(t) clock.Set(now) - _, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options { + deployment, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options { opts.Clock = clock return opts }) + var agentsExperiments map[string]json.RawMessage + require.NoError(t, json.Unmarshal(deployment.AgentsExperiments, &agentsExperiments)) + require.Contains(t, agentsExperiments, "virtual_desktop") + require.Contains(t, agentsExperiments, "advisor") require.Len(t, snapshot.ProvisionerJobs, 2) require.Len(t, snapshot.Licenses, 1) require.Len(t, snapshot.Templates, 2) @@ -313,8 +320,13 @@ func TestTelemetry(t *testing.T) { require.Len(t, snapshot.WorkspaceProxies, 1) require.Len(t, snapshot.WorkspaceModules, 1) require.Len(t, snapshot.Organizations, 1) - // We create one item manually above. The other is TelemetryEnabled, created by the snapshotter. - require.Len(t, snapshot.TelemetryItems, 2) + telemetryItemKeys := slice.Convert(snapshot.TelemetryItems, func(item telemetry.TelemetryItem) string { + return item.Key + }) + require.ElementsMatch(t, []string{ + string(telemetry.TelemetryItemKeyHTMLFirstServedAt), + string(telemetry.TelemetryItemKeyTelemetryEnabled), + }, telemetryItemKeys) require.Len(t, snapshot.WorkspaceAgentMemoryResourceMonitors, 1) require.Len(t, snapshot.WorkspaceAgentVolumeResourceMonitors, 1) wsa := snapshot.WorkspaceAgents[1] @@ -2283,3 +2295,228 @@ func TestUserSecretsTelemetry(t *testing.T) { require.Nil(t, snap2.UserSecretsSummary) }) } + +func TestCollectAgentsVirtualDesktop(t *testing.T) { + t.Parallel() + + collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsVirtualDesktopTelemetry { + t.Helper() + var payload telemetry.AgentsVirtualDesktopTelemetry + require.NoError(t, json.Unmarshal(telemetry.CollectAgentsVirtualDesktop(context.Background(), opts), &payload)) + return payload + } + + t.Run("Default", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("", nil) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.False(t, payload.Enabled) + require.EqualValues(t, codersdk.ChatComputerUseProviderAnthropic, payload.ComputerUse.Provider) + require.Equal(t, "default", payload.ComputerUse.ProviderSource) + }) + + t.Run("Configured", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("openai", nil) + + payload := collect(t, telemetry.Options{ + Database: db, + Logger: testutil.Logger(t), + Experiments: codersdk.Experiments{codersdk.ExperimentChatVirtualDesktop}, + }) + require.True(t, payload.Enabled) + require.Equal(t, "openai", payload.ComputerUse.Provider) + require.Equal(t, "configured", payload.ComputerUse.ProviderSource) + }) + + t.Run("QueryError", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("", sql.ErrConnDone) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.ComputerUse.Provider) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.ComputerUse.ProviderSource) + }) +} + +func TestCollectAgentsAdvisor(t *testing.T) { + t.Parallel() + + collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsAdvisorTelemetry { + t.Helper() + var payload telemetry.AgentsAdvisorTelemetry + require.NoError(t, json.Unmarshal(telemetry.CollectAgentsAdvisor(context.Background(), opts), &payload)) + return payload + } + marshalConfig := func(t *testing.T, cfg codersdk.AdvisorConfig) string { + t.Helper() + raw, err := json.Marshal(cfg) + require.NoError(t, err) + return string(raw) + } + + t.Run("ReuseChatModel", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.False(t, payload.Enabled) + require.Zero(t, payload.MaxUsesPerRun) + require.Zero(t, payload.MaxOutputTokens) + require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Provider) + require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Model) + }) + + t.Run("ModelOverride", func(t *testing.T) { + t.Parallel() + + modelID := uuid.New() + providerID := uuid.New() + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{ + Enabled: true, + MaxUsesPerRun: 7, + MaxOutputTokens: 2048, + ModelConfigID: modelID, + }), nil) + db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).Return(database.ChatModelConfig{ + Model: "gpt-6-preview", + AIProviderID: uuid.NullUUID{UUID: providerID, Valid: true}, + }, nil) + db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(database.AIProvider{ + Type: database.AIProviderTypeOpenai, + }, nil) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + // Stored enabled is ignored; the chat-advisor experiment gates it. + require.False(t, payload.Enabled) + require.Equal(t, 7, payload.MaxUsesPerRun) + require.Equal(t, int64(2048), payload.MaxOutputTokens) + require.Equal(t, string(database.AIProviderTypeOpenai), payload.Provider) + require.Equal(t, "gpt-6-preview", payload.Model) + }) + + t.Run("ExperimentEnabled", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil) + + payload := collect(t, telemetry.Options{ + Database: db, + Logger: testutil.Logger(t), + Experiments: codersdk.Experiments{codersdk.ExperimentChatAdvisor}, + }) + require.True(t, payload.Enabled) + }) + + t.Run("MalformedJSON", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("not-json", nil) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model) + }) + + t.Run("PartialParse", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(`{"max_uses_per_run": 42, "model_config_id": "not-a-uuid"}`, nil) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Zero(t, payload.MaxUsesPerRun) + require.Zero(t, payload.MaxOutputTokens) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model) + }) + + t.Run("ClampsNegativeLimits", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(`{"max_uses_per_run": -3, "max_output_tokens": -99}`, nil) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Zero(t, payload.MaxUsesPerRun) + require.Zero(t, payload.MaxOutputTokens) + }) + + t.Run("InactiveModelConfig", func(t *testing.T) { + t.Parallel() + + modelID := uuid.New() + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil) + db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID). + Return(database.ChatModelConfig{}, sql.ErrNoRows) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Provider) + require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Model) + }) + + t.Run("ConfigFetchError", func(t *testing.T) { + t.Parallel() + + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("", sql.ErrConnDone) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model) + }) + + t.Run("ModelResolveError", func(t *testing.T) { + t.Parallel() + + modelID := uuid.New() + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil) + db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID). + Return(database.ChatModelConfig{}, sql.ErrConnDone) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider) + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model) + }) + + t.Run("ProviderResolveError", func(t *testing.T) { + t.Parallel() + + modelID := uuid.New() + providerID := uuid.New() + db := dbmock.NewMockStore(gomock.NewController(t)) + db.EXPECT().GetChatAdvisorConfig(gomock.Any()). + Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil) + db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).Return(database.ChatModelConfig{ + Model: "gpt-6-preview", + AIProviderID: uuid.NullUUID{UUID: providerID, Valid: true}, + }, nil) + db.EXPECT().GetAIProviderByID(gomock.Any(), providerID). + Return(database.AIProvider{}, sql.ErrConnDone) + + payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)}) + // The provider is unknown, but the already-resolved model still ships. + require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider) + require.Equal(t, "gpt-6-preview", payload.Model) + }) +}