mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
chore: report Coder Agents experiments in telemetry (#27042)
Closes CODAGT-352
This adds the Coder Agents experiments (virtual desktop with computer
use, and the advisor) to telemetry, so they finally show up in each
deployment snapshot. Everything stays inside `coderd/telemetry/`.
## Shape received by the telemetry server
The experiments are reported as a single `agents_experiments` field on
the deployment record, alongside the other config-derived deployment
fields. Its value is one JSON blob with one top-level key per
experiment:
```json
{
"virtual_desktop": {
"enabled": false,
"computer_use": {"provider": "anthropic", "provider_source": "default"}
},
"advisor": {"enabled": true, "max_uses_per_run": 5, "max_output_tokens": 4096, "provider": "openai", "model": "gpt-5.2"}
}
```
When the advisor falls back to the chat model, either because no
override is set or because the configured override is inactive (its
config or provider was deleted or disabled), the provider and model
carry a sentinel instead:
```json
"advisor": {"enabled": true, "max_uses_per_run": 5, "max_output_tokens": 4096, "provider": "advisor_reuse_chat_model", "model": "advisor_reuse_chat_model"}
```
- `virtual_desktop.enabled` and `advisor.enabled` track the
`chat-virtual-desktop` and `chat-advisor` deployment experiments, not
the stored config. We ignore the stored advisor `enabled` flag on
purpose: since #26809 the runtime gates on the experiment, and the
stored flag ends up permanently true for any deployment that ever opened
the settings form.
- Computer use sits under `virtual_desktop` rather than as its own
top-level key because it isn't a separate experiment; the same
`chat-virtual-desktop` flag gates both the desktop and the computer-use
provider. `provider_source` says whether an admin picked the provider
(`configured`) or we fell back to the default (`default`).
- `advisor.provider` is the `ai_providers` type (e.g. `openai`,
`anthropic`, `azure`) and `advisor.model` is the configured model
string. Two sentinels stand in when there's no concrete value:
`advisor_reuse_chat_model` when the advisor has no active override and
falls back to the chat model (matching the runtime), and `unknown` when
we genuinely couldn't tell, e.g. a query failed or the stored config
wouldn't parse.
- `advisor.max_uses_per_run` and `advisor.max_output_tokens` are clamped
to 0 before reporting, matching how the API normalizes these values on
read.
## Why this shape
Putting the data on the deployment record keeps it next to the other
config-derived fields, and leaves `telemetry_items` as a faithful mirror
of the `telemetry_items` table rather than a place we inject synthetic
rows. Adding or removing an experiment is a one-line edit to the
`agentsExperiments` registry. The `agents_experiments` field itself
never changes; only the JSON inside it does. The field is `omitempty`,
so older Coder versions that don't emit it are distinguishable from a
real absence, and when an experiment isn't reported in a snapshot its
JSON path is simply missing, so queries can tell "not reported" apart
from a real `false`.
One key holding one JSON blob is also easier to query than many separate
fields. Because everything lives in one blob, a question like "of the
deployments running the desktop, how many changed the computer-use
provider?" is one query with no join:
```sql
SELECT
JSON_VALUE(agents_experiments, '$.virtual_desktop.computer_use.provider_source') AS src,
COUNT(*) AS deployments
FROM deployments
WHERE JSON_VALUE(agents_experiments, '$.virtual_desktop.enabled') = 'true'
GROUP BY src
```
This commit is contained in:
+161
-21
@@ -345,28 +345,41 @@ func (r *remoteReporter) deployment() error {
|
||||
scimEnabled := r.options.SCIMEnabled
|
||||
scimUseLegacy := r.options.SCIMUseLegacy
|
||||
|
||||
agentsExperimentValues := make(map[string]json.RawMessage, len(agentsExperiments))
|
||||
for _, exp := range agentsExperiments {
|
||||
agentsExperimentValues[exp.name] = exp.collect(r.ctx, r.options)
|
||||
}
|
||||
agentsExperimentsJSON, err := json.Marshal(agentsExperimentValues)
|
||||
if err != nil {
|
||||
// Best-effort: the field is omitempty, so the deployment report
|
||||
// proceeds without it.
|
||||
r.options.Logger.Warn(r.ctx, "marshal agent experiments telemetry", slog.Error(err))
|
||||
agentsExperimentsJSON = nil
|
||||
}
|
||||
|
||||
data, err := json.Marshal(&Deployment{
|
||||
ID: r.options.DeploymentID,
|
||||
Architecture: sysInfo.Architecture,
|
||||
BuiltinPostgres: r.options.BuiltinPostgres,
|
||||
Containerized: containerized,
|
||||
Config: r.options.DeploymentConfig,
|
||||
Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "",
|
||||
InstallSource: installSource,
|
||||
Tunnel: r.options.Tunnel,
|
||||
OSType: sysInfo.OS.Type,
|
||||
OSFamily: sysInfo.OS.Family,
|
||||
OSPlatform: sysInfo.OS.Platform,
|
||||
OSName: sysInfo.OS.Name,
|
||||
OSVersion: sysInfo.OS.Version,
|
||||
CPUCores: runtime.NumCPU(),
|
||||
MemoryTotal: mem.Total,
|
||||
MachineID: sysInfo.UniqueID,
|
||||
StartedAt: r.startedAt,
|
||||
ShutdownAt: r.shutdownAt,
|
||||
IDPOrgSync: &idpOrgSync,
|
||||
SCIMEnabled: &scimEnabled,
|
||||
SCIMUseLegacy: &scimUseLegacy,
|
||||
ID: r.options.DeploymentID,
|
||||
Architecture: sysInfo.Architecture,
|
||||
BuiltinPostgres: r.options.BuiltinPostgres,
|
||||
Containerized: containerized,
|
||||
Config: r.options.DeploymentConfig,
|
||||
Kubernetes: os.Getenv("KUBERNETES_SERVICE_HOST") != "",
|
||||
InstallSource: installSource,
|
||||
Tunnel: r.options.Tunnel,
|
||||
OSType: sysInfo.OS.Type,
|
||||
OSFamily: sysInfo.OS.Family,
|
||||
OSPlatform: sysInfo.OS.Platform,
|
||||
OSName: sysInfo.OS.Name,
|
||||
OSVersion: sysInfo.OS.Version,
|
||||
CPUCores: runtime.NumCPU(),
|
||||
MemoryTotal: mem.Total,
|
||||
MachineID: sysInfo.UniqueID,
|
||||
StartedAt: r.startedAt,
|
||||
ShutdownAt: r.shutdownAt,
|
||||
IDPOrgSync: &idpOrgSync,
|
||||
SCIMEnabled: &scimEnabled,
|
||||
SCIMUseLegacy: &scimUseLegacy,
|
||||
AgentsExperiments: agentsExperimentsJSON,
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("marshal deployment: %w", err)
|
||||
@@ -1660,6 +1673,10 @@ type Deployment struct {
|
||||
// enterprise/coderd/scim. Nullable for the same backward compatibility
|
||||
// reason as SCIMEnabled.
|
||||
SCIMUseLegacy *bool `json:"scim_use_legacy"`
|
||||
// AgentsExperiments reports the state of the Coder Agents experiments as
|
||||
// opaque per-experiment JSON, so rotating the reported set is a code-only
|
||||
// change. Omitted by older Coder versions, so it decodes as nil there.
|
||||
AgentsExperiments json.RawMessage `json:"agents_experiments,omitempty"`
|
||||
}
|
||||
|
||||
type APIKey struct {
|
||||
@@ -2322,6 +2339,129 @@ const (
|
||||
TelemetryItemKeyTelemetryEnabled telemetryItemKey = "telemetry_enabled"
|
||||
)
|
||||
|
||||
// agentsExperiment is one entry in the Deployment.AgentsExperiments field.
|
||||
// Edit agentsExperiments to rotate the reported set without schema or
|
||||
// telemetry-server changes. Collectors are best-effort: they log and return
|
||||
// a degraded payload instead of erroring, so they can never fail a report.
|
||||
type agentsExperiment struct {
|
||||
name string
|
||||
collect func(ctx context.Context, opts Options) json.RawMessage
|
||||
}
|
||||
|
||||
var agentsExperiments = []agentsExperiment{
|
||||
{name: "virtual_desktop", collect: CollectAgentsVirtualDesktop},
|
||||
{name: "advisor", collect: CollectAgentsAdvisor},
|
||||
}
|
||||
|
||||
const (
|
||||
// AgentsExperimentAdvisorReuseChatModel reports that the advisor has no active
|
||||
// dedicated model override and reuses the chat model at runtime.
|
||||
AgentsExperimentAdvisorReuseChatModel = "advisor_reuse_chat_model"
|
||||
// AgentsExperimentUnknown reports a value that could not be determined,
|
||||
// e.g. after a transient DB error.
|
||||
AgentsExperimentUnknown = "unknown"
|
||||
)
|
||||
|
||||
// AgentsVirtualDesktopTelemetry is the value shape for the virtual_desktop
|
||||
// entry in Deployment.AgentsExperiments.
|
||||
type AgentsVirtualDesktopTelemetry struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
ComputerUse AgentsComputerUseTelemetry `json:"computer_use"`
|
||||
}
|
||||
|
||||
type AgentsComputerUseTelemetry struct {
|
||||
Provider string `json:"provider"`
|
||||
ProviderSource string `json:"provider_source"`
|
||||
}
|
||||
|
||||
// AgentsAdvisorTelemetry is the value shape for the advisor entry in
|
||||
// Deployment.AgentsExperiments.
|
||||
type AgentsAdvisorTelemetry struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
MaxUsesPerRun int `json:"max_uses_per_run"`
|
||||
MaxOutputTokens int64 `json:"max_output_tokens"`
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// CollectAgentsVirtualDesktop collects the virtual_desktop entry in
|
||||
// Deployment.AgentsExperiments. The chat-virtual-desktop experiment gates both
|
||||
// the desktop and computer use.
|
||||
func CollectAgentsVirtualDesktop(ctx context.Context, opts Options) json.RawMessage {
|
||||
provider, err := opts.Database.GetChatComputerUseProvider(ctx)
|
||||
providerSource := "configured"
|
||||
switch {
|
||||
case err != nil:
|
||||
opts.Logger.Warn(ctx, "get chat computer use provider for telemetry", slog.Error(err))
|
||||
provider = AgentsExperimentUnknown
|
||||
providerSource = AgentsExperimentUnknown
|
||||
case provider == "":
|
||||
provider = string(codersdk.ChatComputerUseProviderAnthropic)
|
||||
providerSource = "default"
|
||||
}
|
||||
val, err := json.Marshal(AgentsVirtualDesktopTelemetry{
|
||||
Enabled: opts.Experiments.Enabled(codersdk.ExperimentChatVirtualDesktop),
|
||||
ComputerUse: AgentsComputerUseTelemetry{
|
||||
Provider: provider,
|
||||
ProviderSource: providerSource,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
opts.Logger.Warn(ctx, "marshal agent virtual desktop telemetry", slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
// CollectAgentsAdvisor collects the advisor entry in
|
||||
// Deployment.AgentsExperiments.
|
||||
func CollectAgentsAdvisor(ctx context.Context, opts Options) json.RawMessage {
|
||||
payload := AgentsAdvisorTelemetry{
|
||||
Enabled: opts.Experiments.Enabled(codersdk.ExperimentChatAdvisor),
|
||||
Provider: AgentsExperimentUnknown,
|
||||
Model: AgentsExperimentUnknown,
|
||||
}
|
||||
var cfg codersdk.AdvisorConfig
|
||||
raw, err := opts.Database.GetChatAdvisorConfig(ctx)
|
||||
if err != nil {
|
||||
opts.Logger.Warn(ctx, "get chat advisor config for telemetry", slog.Error(err))
|
||||
} else if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
|
||||
opts.Logger.Warn(ctx, "parse chat advisor config for telemetry", slog.Error(err))
|
||||
} else {
|
||||
payload.MaxUsesPerRun = max(cfg.MaxUsesPerRun, 0)
|
||||
payload.MaxOutputTokens = max(cfg.MaxOutputTokens, 0)
|
||||
payload.Provider, payload.Model = advisorModelTelemetry(ctx, opts.Database, opts.Logger, cfg.ModelConfigID)
|
||||
}
|
||||
val, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
opts.Logger.Warn(ctx, "marshal agent advisor telemetry", slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
return val
|
||||
}
|
||||
|
||||
func advisorModelTelemetry(ctx context.Context, db database.Store, log slog.Logger, id uuid.UUID) (provider string, model string) {
|
||||
if id == uuid.Nil {
|
||||
return AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel
|
||||
}
|
||||
|
||||
cfg, err := db.GetEnabledChatModelConfigByID(ctx, id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
// An inactive override; the runtime falls back to the chat model.
|
||||
return AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel
|
||||
}
|
||||
if err != nil {
|
||||
log.Warn(ctx, "resolve chat advisor model config for telemetry", slog.Error(err))
|
||||
return AgentsExperimentUnknown, AgentsExperimentUnknown
|
||||
}
|
||||
providerRow, err := db.GetAIProviderByID(ctx, cfg.AIProviderID.UUID)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "resolve chat advisor model provider for telemetry", slog.Error(err))
|
||||
return AgentsExperimentUnknown, cfg.Model
|
||||
}
|
||||
return string(providerRow.Type), cfg.Model
|
||||
}
|
||||
|
||||
type TelemetryItem struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
|
||||
@@ -20,18 +20,21 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
"go.uber.org/mock/gomock"
|
||||
|
||||
"github.com/coder/coder/v2/buildinfo"
|
||||
"github.com/coder/coder/v2/coderd/boundaryusage"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbfake"
|
||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||
"github.com/coder/coder/v2/coderd/database/dbmock"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||
"github.com/coder/coder/v2/coderd/idpsync"
|
||||
"github.com/coder/coder/v2/coderd/runtimeconfig"
|
||||
"github.com/coder/coder/v2/coderd/telemetry"
|
||||
"github.com/coder/coder/v2/coderd/util/ptr"
|
||||
"github.com/coder/coder/v2/coderd/util/slice"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
"github.com/coder/quartz"
|
||||
@@ -292,10 +295,14 @@ func TestTelemetry(t *testing.T) {
|
||||
clock := quartz.NewMock(t)
|
||||
clock.Set(now)
|
||||
|
||||
_, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
||||
deployment, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
||||
opts.Clock = clock
|
||||
return opts
|
||||
})
|
||||
var agentsExperiments map[string]json.RawMessage
|
||||
require.NoError(t, json.Unmarshal(deployment.AgentsExperiments, &agentsExperiments))
|
||||
require.Contains(t, agentsExperiments, "virtual_desktop")
|
||||
require.Contains(t, agentsExperiments, "advisor")
|
||||
require.Len(t, snapshot.ProvisionerJobs, 2)
|
||||
require.Len(t, snapshot.Licenses, 1)
|
||||
require.Len(t, snapshot.Templates, 2)
|
||||
@@ -313,8 +320,13 @@ func TestTelemetry(t *testing.T) {
|
||||
require.Len(t, snapshot.WorkspaceProxies, 1)
|
||||
require.Len(t, snapshot.WorkspaceModules, 1)
|
||||
require.Len(t, snapshot.Organizations, 1)
|
||||
// We create one item manually above. The other is TelemetryEnabled, created by the snapshotter.
|
||||
require.Len(t, snapshot.TelemetryItems, 2)
|
||||
telemetryItemKeys := slice.Convert(snapshot.TelemetryItems, func(item telemetry.TelemetryItem) string {
|
||||
return item.Key
|
||||
})
|
||||
require.ElementsMatch(t, []string{
|
||||
string(telemetry.TelemetryItemKeyHTMLFirstServedAt),
|
||||
string(telemetry.TelemetryItemKeyTelemetryEnabled),
|
||||
}, telemetryItemKeys)
|
||||
require.Len(t, snapshot.WorkspaceAgentMemoryResourceMonitors, 1)
|
||||
require.Len(t, snapshot.WorkspaceAgentVolumeResourceMonitors, 1)
|
||||
wsa := snapshot.WorkspaceAgents[1]
|
||||
@@ -2283,3 +2295,228 @@ func TestUserSecretsTelemetry(t *testing.T) {
|
||||
require.Nil(t, snap2.UserSecretsSummary)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollectAgentsVirtualDesktop(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsVirtualDesktopTelemetry {
|
||||
t.Helper()
|
||||
var payload telemetry.AgentsVirtualDesktopTelemetry
|
||||
require.NoError(t, json.Unmarshal(telemetry.CollectAgentsVirtualDesktop(context.Background(), opts), &payload))
|
||||
return payload
|
||||
}
|
||||
|
||||
t.Run("Default", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("", nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.False(t, payload.Enabled)
|
||||
require.EqualValues(t, codersdk.ChatComputerUseProviderAnthropic, payload.ComputerUse.Provider)
|
||||
require.Equal(t, "default", payload.ComputerUse.ProviderSource)
|
||||
})
|
||||
|
||||
t.Run("Configured", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("openai", nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{
|
||||
Database: db,
|
||||
Logger: testutil.Logger(t),
|
||||
Experiments: codersdk.Experiments{codersdk.ExperimentChatVirtualDesktop},
|
||||
})
|
||||
require.True(t, payload.Enabled)
|
||||
require.Equal(t, "openai", payload.ComputerUse.Provider)
|
||||
require.Equal(t, "configured", payload.ComputerUse.ProviderSource)
|
||||
})
|
||||
|
||||
t.Run("QueryError", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("", sql.ErrConnDone)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.ComputerUse.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.ComputerUse.ProviderSource)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCollectAgentsAdvisor(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsAdvisorTelemetry {
|
||||
t.Helper()
|
||||
var payload telemetry.AgentsAdvisorTelemetry
|
||||
require.NoError(t, json.Unmarshal(telemetry.CollectAgentsAdvisor(context.Background(), opts), &payload))
|
||||
return payload
|
||||
}
|
||||
marshalConfig := func(t *testing.T, cfg codersdk.AdvisorConfig) string {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(cfg)
|
||||
require.NoError(t, err)
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
t.Run("ReuseChatModel", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.False(t, payload.Enabled)
|
||||
require.Zero(t, payload.MaxUsesPerRun)
|
||||
require.Zero(t, payload.MaxOutputTokens)
|
||||
require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Model)
|
||||
})
|
||||
|
||||
t.Run("ModelOverride", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
modelID := uuid.New()
|
||||
providerID := uuid.New()
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{
|
||||
Enabled: true,
|
||||
MaxUsesPerRun: 7,
|
||||
MaxOutputTokens: 2048,
|
||||
ModelConfigID: modelID,
|
||||
}), nil)
|
||||
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).Return(database.ChatModelConfig{
|
||||
Model: "gpt-6-preview",
|
||||
AIProviderID: uuid.NullUUID{UUID: providerID, Valid: true},
|
||||
}, nil)
|
||||
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(database.AIProvider{
|
||||
Type: database.AIProviderTypeOpenai,
|
||||
}, nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
// Stored enabled is ignored; the chat-advisor experiment gates it.
|
||||
require.False(t, payload.Enabled)
|
||||
require.Equal(t, 7, payload.MaxUsesPerRun)
|
||||
require.Equal(t, int64(2048), payload.MaxOutputTokens)
|
||||
require.Equal(t, string(database.AIProviderTypeOpenai), payload.Provider)
|
||||
require.Equal(t, "gpt-6-preview", payload.Model)
|
||||
})
|
||||
|
||||
t.Run("ExperimentEnabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{
|
||||
Database: db,
|
||||
Logger: testutil.Logger(t),
|
||||
Experiments: codersdk.Experiments{codersdk.ExperimentChatAdvisor},
|
||||
})
|
||||
require.True(t, payload.Enabled)
|
||||
})
|
||||
|
||||
t.Run("MalformedJSON", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("not-json", nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
|
||||
})
|
||||
|
||||
t.Run("PartialParse", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(`{"max_uses_per_run": 42, "model_config_id": "not-a-uuid"}`, nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Zero(t, payload.MaxUsesPerRun)
|
||||
require.Zero(t, payload.MaxOutputTokens)
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
|
||||
})
|
||||
|
||||
t.Run("ClampsNegativeLimits", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(`{"max_uses_per_run": -3, "max_output_tokens": -99}`, nil)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Zero(t, payload.MaxUsesPerRun)
|
||||
require.Zero(t, payload.MaxOutputTokens)
|
||||
})
|
||||
|
||||
t.Run("InactiveModelConfig", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
modelID := uuid.New()
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil)
|
||||
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).
|
||||
Return(database.ChatModelConfig{}, sql.ErrNoRows)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Model)
|
||||
})
|
||||
|
||||
t.Run("ConfigFetchError", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("", sql.ErrConnDone)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
|
||||
})
|
||||
|
||||
t.Run("ModelResolveError", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
modelID := uuid.New()
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil)
|
||||
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).
|
||||
Return(database.ChatModelConfig{}, sql.ErrConnDone)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
|
||||
})
|
||||
|
||||
t.Run("ProviderResolveError", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
modelID := uuid.New()
|
||||
providerID := uuid.New()
|
||||
db := dbmock.NewMockStore(gomock.NewController(t))
|
||||
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
|
||||
Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil)
|
||||
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).Return(database.ChatModelConfig{
|
||||
Model: "gpt-6-preview",
|
||||
AIProviderID: uuid.NullUUID{UUID: providerID, Valid: true},
|
||||
}, nil)
|
||||
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).
|
||||
Return(database.AIProvider{}, sql.ErrConnDone)
|
||||
|
||||
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
||||
// The provider is unknown, but the already-resolved model still ships.
|
||||
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
|
||||
require.Equal(t, "gpt-6-preview", payload.Model)
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user