feat!: scope chat model override settings to organizations (#28442)

Moves chat model override settings from deployment scope to organization
scope, following the org-scoping of chat model configs in #28440.

Depends on #28440 (based on
`ethan/chat-models-availability-consolidation`).

Remote dogfood UAT passed on this change, including multi-organization
scenarios on a licensed deployment with real models.

## Problem

Chat model configs are now owned by organizations, but the override
settings that reference them stayed deployment-wide: admin
subagent/title/compaction overrides in `site_configs`, personal
overrides in `user_configs`, and the advisor model embedded in the
advisor runtime JSON. A single deployment-level override can only point
at one organization's model config, so overrides effectively worked for
the default organization only, and title generation had to soften its
hard-failure contract to tolerate unusable overrides.

## Fix

Admin overrides (general, explore, title generation, compaction,
advisor) and personal overrides (root, general, explore) become
organization-scoped rows in two new typed tables,
`chat_organization_model_overrides` and `chat_user_model_overrides`.
Composite foreign keys on `(organization_id, model_config_id)` make
cross-organization model references unrepresentable. The migration
deletes the legacy serialized `site_configs`/`user_configs` override
keys instead of migrating them; admins and users re-select models in the
new organization-scoped settings.

New endpoints live under
`/api/experimental/organizations/{organization}/chats/model-overrides`
(admin defaults) and
`/api/experimental/organizations/{organization}/members/{user}/chats/model-overrides`
(personal overrides); the legacy deployment-level routes are removed.
chatd resolves overrides via the chat's organization, restoring the
hard-failure contract for configured-but-unusable title generation
overrides. The deployment-level `AllowUsers` switch and advisor runtime
limits stay deployment-scoped.

The Coder Agents settings UI gains a per-organization "Defaults &
overrides" page under AI Settings > Models, and the personal overrides
page gains an organization selector for users in multiple organizations.
Advisor telemetry now reports per-organization override rows.

**BREAKING CHANGE**: Existing deployment-level overrides are dropped,
not migrated. Every organization starts with no overrides after upgrade,
and model selections must be re-applied in the new settings. The legacy
deployment-level override endpoints are removed.

Closes https://linear.app/coder/issue/CODAGT-872

> [!NOTE]
> Xum (AI agent) authored this PR on behalf of @ibetitsmike.

<!-- xum-attribution: model=claude-opus-4-6 thinking=high -->

---------

Co-authored-by: Ethan Dickson <ethan@coder.com>

coderd/telemetry/telemetry.go

@@ -2378,14 +2378,20 @@ type AgentsComputerUseTelemetry struct {
 	ProviderSource string `json:"provider_source"`
 }
 
+// AgentsAdvisorOverrideTelemetry describes one organization's advisor model override.
+type AgentsAdvisorOverrideTelemetry struct {
+	OrganizationID string `json:"organization_id"`
+	Provider       string `json:"provider"`
+	Model          string `json:"model"`
+}
+
 // AgentsAdvisorTelemetry is the value shape for the advisor entry in
 // Deployment.AgentsExperiments.
 type AgentsAdvisorTelemetry struct {
-	Enabled         bool   `json:"enabled"`
-	MaxUsesPerRun   int    `json:"max_uses_per_run"`
-	MaxOutputTokens int64  `json:"max_output_tokens"`
-	Provider        string `json:"provider"`
-	Model           string `json:"model"`
+	Enabled         bool                             `json:"enabled"`
+	MaxUsesPerRun   int                              `json:"max_uses_per_run"`
+	MaxOutputTokens int64                            `json:"max_output_tokens"`
+	Overrides       []AgentsAdvisorOverrideTelemetry `json:"overrides"`
 }
 
 // CollectAgentsVirtualDesktop collects the virtual_desktop entry in
@@ -2421,9 +2427,8 @@ func CollectAgentsVirtualDesktop(ctx context.Context, opts Options) json.RawMess
 // Deployment.AgentsExperiments.
 func CollectAgentsAdvisor(ctx context.Context, opts Options) json.RawMessage {
 	payload := AgentsAdvisorTelemetry{
-		Enabled:  opts.Experiments.Enabled(codersdk.ExperimentChatAdvisor),
-		Provider: AgentsExperimentUnknown,
-		Model:    AgentsExperimentUnknown,
+		Enabled:   opts.Experiments.Enabled(codersdk.ExperimentChatAdvisor),
+		Overrides: []AgentsAdvisorOverrideTelemetry{},
 	}
 	var cfg codersdk.AdvisorConfig
 	raw, err := opts.Database.GetChatAdvisorConfig(ctx)
@@ -2434,8 +2439,27 @@ func CollectAgentsAdvisor(ctx context.Context, opts Options) json.RawMessage {
 	} else {
 		payload.MaxUsesPerRun = max(cfg.MaxUsesPerRun, 0)
 		payload.MaxOutputTokens = max(cfg.MaxOutputTokens, 0)
-		payload.Provider, payload.Model = advisorModelTelemetry(ctx, opts.Database, opts.Logger, cfg.ModelConfigID)
 	}
+
+	overrides, err := opts.Database.GetChatOrganizationModelOverridesByContext(ctx, string(codersdk.ChatModelOverrideContextAdvisor))
+	if err != nil {
+		opts.Logger.Warn(ctx, "get chat advisor model overrides for telemetry", slog.Error(err))
+	} else {
+		for _, override := range overrides {
+			// When the override's model is disabled or deleted, the runtime
+			// falls back to the chat model.
+			provider, model := AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel
+			if override.ModelAvailable {
+				provider, model = override.ProviderType, override.Model
+			}
+			payload.Overrides = append(payload.Overrides, AgentsAdvisorOverrideTelemetry{
+				OrganizationID: override.OrganizationID.String(),
+				Provider:       provider,
+				Model:          model,
+			})
+		}
+	}
+
 	val, err := json.Marshal(payload)
 	if err != nil {
 		opts.Logger.Warn(ctx, "marshal agent advisor telemetry", slog.Error(err))
@@ -2444,28 +2468,6 @@ func CollectAgentsAdvisor(ctx context.Context, opts Options) json.RawMessage {
 	return val
 }
 
-func advisorModelTelemetry(ctx context.Context, db database.Store, log slog.Logger, id uuid.UUID) (provider string, model string) {
-	if id == uuid.Nil {
-		return AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel
-	}
-
-	cfg, err := db.GetEnabledChatModelConfigByID(ctx, id)
-	if errors.Is(err, sql.ErrNoRows) {
-		// An inactive override; the runtime falls back to the chat model.
-		return AgentsExperimentAdvisorReuseChatModel, AgentsExperimentAdvisorReuseChatModel
-	}
-	if err != nil {
-		log.Warn(ctx, "resolve chat advisor model config for telemetry", slog.Error(err))
-		return AgentsExperimentUnknown, AgentsExperimentUnknown
-	}
-	providerRow, err := db.GetAIProviderByID(ctx, cfg.AIProviderID.UUID)
-	if err != nil {
-		log.Warn(ctx, "resolve chat advisor model provider for telemetry", slog.Error(err))
-		return AgentsExperimentUnknown, cfg.Model
-	}
-	return string(providerRow.Type), cfg.Model
-}
-
 type TelemetryItem struct {
 	Key       string    `json:"key"`
 	Value     string    `json:"value"`

coderd/telemetry/telemetry_test.go

@@ -2382,162 +2382,105 @@ func TestCollectAgentsAdvisor(t *testing.T) {
 		require.NoError(t, err)
 		return string(raw)
 	}
+	expectOverrides := func(db *dbmock.MockStore, rows ...database.GetChatOrganizationModelOverridesByContextRow) {
+		db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), string(codersdk.ChatModelOverrideContextAdvisor)).Return(rows, nil)
+	}
 
-	t.Run("ReuseChatModel", func(t *testing.T) {
+	t.Run("Defaults", func(t *testing.T) {
 		t.Parallel()
-
 		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
+		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
+		expectOverrides(db)
 
 		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
 		require.False(t, payload.Enabled)
 		require.Zero(t, payload.MaxUsesPerRun)
 		require.Zero(t, payload.MaxOutputTokens)
-		require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Provider)
-		require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Model)
+		require.Empty(t, payload.Overrides)
 	})
 
-	t.Run("ModelOverride", func(t *testing.T) {
+	t.Run("ModelOverrides", func(t *testing.T) {
 		t.Parallel()
-
-		modelID := uuid.New()
-		providerID := uuid.New()
+		org := database.Organization{ID: uuid.New()}
 		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{
-			Enabled:         true,
-			MaxUsesPerRun:   7,
-			MaxOutputTokens: 2048,
-			ModelConfigID:   modelID,
-		}), nil)
-		db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).Return(database.ChatModelConfig{
-			Model:        "gpt-6-preview",
-			AIProviderID: uuid.NullUUID{UUID: providerID, Valid: true},
-		}, nil)
-		db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(database.AIProvider{
-			Type: database.AIProviderTypeOpenai,
-		}, nil)
+		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{MaxUsesPerRun: 7, MaxOutputTokens: 2048}), nil)
+		expectOverrides(db, database.GetChatOrganizationModelOverridesByContextRow{
+			OrganizationID: org.ID,
+			ModelAvailable: true,
+			Model:          "gpt-6-preview",
+			ProviderType:   string(database.AIProviderTypeOpenai),
+		})
 
-		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
-		// Stored enabled is ignored; the chat-advisor experiment gates it.
-		require.False(t, payload.Enabled)
+		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t), Experiments: codersdk.Experiments{codersdk.ExperimentChatAdvisor}})
+		require.True(t, payload.Enabled)
 		require.Equal(t, 7, payload.MaxUsesPerRun)
 		require.Equal(t, int64(2048), payload.MaxOutputTokens)
-		require.Equal(t, string(database.AIProviderTypeOpenai), payload.Provider)
-		require.Equal(t, "gpt-6-preview", payload.Model)
+		require.Equal(t, []telemetry.AgentsAdvisorOverrideTelemetry{{
+			OrganizationID: org.ID.String(), Provider: string(database.AIProviderTypeOpenai), Model: "gpt-6-preview",
+		}}, payload.Overrides)
 	})
 
-	t.Run("ExperimentEnabled", func(t *testing.T) {
+	t.Run("InactiveModel", func(t *testing.T) {
 		t.Parallel()
-
+		org := database.Organization{ID: uuid.New()}
 		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
-
-		payload := collect(t, telemetry.Options{
-			Database:    db,
-			Logger:      testutil.Logger(t),
-			Experiments: codersdk.Experiments{codersdk.ExperimentChatAdvisor},
+		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
+		expectOverrides(db, database.GetChatOrganizationModelOverridesByContextRow{
+			OrganizationID: org.ID,
+			ModelAvailable: false,
 		})
-		require.True(t, payload.Enabled)
+		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
+		require.Equal(t, []telemetry.AgentsAdvisorOverrideTelemetry{{
+			OrganizationID: org.ID.String(),
+			Provider:       telemetry.AgentsExperimentAdvisorReuseChatModel,
+			Model:          telemetry.AgentsExperimentAdvisorReuseChatModel,
+		}}, payload.Overrides)
 	})
 
-	t.Run("MalformedJSON", func(t *testing.T) {
+	t.Run("NoOverride", func(t *testing.T) {
 		t.Parallel()
+		db := dbmock.NewMockStore(gomock.NewController(t))
+		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
+		expectOverrides(db)
+		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
+		require.Empty(t, payload.Overrides)
+	})
 
+	t.Run("MalformedConfig", func(t *testing.T) {
+		t.Parallel()
 		db := dbmock.NewMockStore(gomock.NewController(t))
 		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("not-json", nil)
-
-		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
-	})
-
-	t.Run("PartialParse", func(t *testing.T) {
-		t.Parallel()
-
-		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(`{"max_uses_per_run": 42, "model_config_id": "not-a-uuid"}`, nil)
-
+		expectOverrides(db)
 		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
 		require.Zero(t, payload.MaxUsesPerRun)
 		require.Zero(t, payload.MaxOutputTokens)
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
 	})
 
 	t.Run("ClampsNegativeLimits", func(t *testing.T) {
 		t.Parallel()
-
 		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(`{"max_uses_per_run": -3, "max_output_tokens": -99}`, nil)
-
+		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(`{"max_uses_per_run":-3,"max_output_tokens":-99}`, nil)
+		expectOverrides(db)
 		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
 		require.Zero(t, payload.MaxUsesPerRun)
 		require.Zero(t, payload.MaxOutputTokens)
 	})
 
-	t.Run("InactiveModelConfig", func(t *testing.T) {
-		t.Parallel()
-
-		modelID := uuid.New()
-		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil)
-		db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).
-			Return(database.ChatModelConfig{}, sql.ErrNoRows)
-
-		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
-		require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Provider)
-		require.Equal(t, telemetry.AgentsExperimentAdvisorReuseChatModel, payload.Model)
-	})
-
 	t.Run("ConfigFetchError", func(t *testing.T) {
 		t.Parallel()
-
 		db := dbmock.NewMockStore(gomock.NewController(t))
 		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("", sql.ErrConnDone)
-
+		expectOverrides(db)
 		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
+		require.Empty(t, payload.Overrides)
 	})
 
-	t.Run("ModelResolveError", func(t *testing.T) {
+	t.Run("OverridesFetchError", func(t *testing.T) {
 		t.Parallel()
-
-		modelID := uuid.New()
 		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil)
-		db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).
-			Return(database.ChatModelConfig{}, sql.ErrConnDone)
-
+		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
+		db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), gomock.Any()).Return(nil, sql.ErrConnDone)
 		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Model)
-	})
-
-	t.Run("ProviderResolveError", func(t *testing.T) {
-		t.Parallel()
-
-		modelID := uuid.New()
-		providerID := uuid.New()
-		db := dbmock.NewMockStore(gomock.NewController(t))
-		db.EXPECT().GetChatAdvisorConfig(gomock.Any()).
-			Return(marshalConfig(t, codersdk.AdvisorConfig{ModelConfigID: modelID}), nil)
-		db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), modelID).Return(database.ChatModelConfig{
-			Model:        "gpt-6-preview",
-			AIProviderID: uuid.NullUUID{UUID: providerID, Valid: true},
-		}, nil)
-		db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).
-			Return(database.AIProvider{}, sql.ErrConnDone)
-
-		payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
-		// The provider is unknown, but the already-resolved model still ships.
-		require.Equal(t, telemetry.AgentsExperimentUnknown, payload.Provider)
-		require.Equal(t, "gpt-6-preview", payload.Model)
+		require.Empty(t, payload.Overrides)
 	})
 }