mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: configure multiple AI Bridge providers of the same type (#23948)
_Disclaimer: produced mostly by Claude Opus 4.6 following detailed planning._ ## Summary - Support multiple instances of the same AI Bridge provider type via indexed env vars (`CODER_AIBRIDGE_PROVIDER_<N>_<KEY>`), following the `CODER_EXTERNAL_AUTH_<N>_<KEY>` pattern - Existing single-provider env vars (`CODER_AIBRIDGE_OPENAI_KEY`, etc.) continue to work unchanged - Setting both a legacy env var and an indexed provider with the same name errors at startup to prevent silent misconfiguration - Mark legacy provider fields (`OpenAI`, `Anthropic`, `Bedrock`) as deprecated in `AIBridgeConfig` in favor of `Providers` ## Example ```sh CODER_AIBRIDGE_PROVIDER_0_TYPE=anthropic CODER_AIBRIDGE_PROVIDER_0_NAME=anthropic-corp CODER_AIBRIDGE_PROVIDER_0_KEY=sk-ant-corp-xxx CODER_AIBRIDGE_PROVIDER_0_BASE_URL=https://llm-proxy.internal.example.com/anthropic CODER_AIBRIDGE_PROVIDER_1_TYPE=anthropic CODER_AIBRIDGE_PROVIDER_1_NAME=anthropic-direct CODER_AIBRIDGE_PROVIDER_1_KEY=sk-ant-direct-yyy ``` Each instance is routed by name: - /api/v2/aibridge/**anthropic-corp**/v1/messages - /api/v2/aibridge/**anthropic-direct**/v1/messages Closes [AIGOV-157](https://linear.app/codercom/issue/AIGOV-157/spike-to-understand-if-there-is-a-simple-way-to-handle-multi-api-key) --------- Signed-off-by: Danny Kopping <danny@coder.com>
This commit is contained in:
+117
-52
@@ -10,68 +10,17 @@ import (
|
||||
|
||||
"github.com/coder/aibridge"
|
||||
"github.com/coder/aibridge/config"
|
||||
agplaibridge "github.com/coder/coder/v2/coderd/aibridge"
|
||||
"github.com/coder/coder/v2/coderd/tracing"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/enterprise/aibridged"
|
||||
"github.com/coder/coder/v2/enterprise/coderd"
|
||||
)
|
||||
|
||||
func newAIBridgeDaemon(coderAPI *coderd.API) (*aibridged.Server, error) {
|
||||
func newAIBridgeDaemon(coderAPI *coderd.API, providers []aibridge.Provider) (*aibridged.Server, error) {
|
||||
ctx := context.Background()
|
||||
coderAPI.Logger.Debug(ctx, "starting in-memory aibridge daemon")
|
||||
|
||||
logger := coderAPI.Logger.Named("aibridged")
|
||||
cfg := coderAPI.DeploymentValues.AI.BridgeConfig
|
||||
|
||||
// Build circuit breaker config if enabled.
|
||||
var cbConfig *config.CircuitBreaker
|
||||
if cfg.CircuitBreakerEnabled.Value() {
|
||||
cbConfig = &config.CircuitBreaker{
|
||||
FailureThreshold: uint32(cfg.CircuitBreakerFailureThreshold.Value()), //nolint:gosec // Validated by serpent.Validate in deployment options.
|
||||
Interval: cfg.CircuitBreakerInterval.Value(),
|
||||
Timeout: cfg.CircuitBreakerTimeout.Value(),
|
||||
MaxRequests: uint32(cfg.CircuitBreakerMaxRequests.Value()), //nolint:gosec // Validated by serpent.Validate in deployment options.
|
||||
}
|
||||
}
|
||||
|
||||
// Setup supported providers with circuit breaker config.
|
||||
providers := []aibridge.Provider{
|
||||
aibridge.NewOpenAIProvider(aibridge.OpenAIConfig{
|
||||
Name: aibridge.ProviderOpenAI,
|
||||
BaseURL: cfg.OpenAI.BaseURL.String(),
|
||||
Key: cfg.OpenAI.Key.String(),
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}),
|
||||
aibridge.NewAnthropicProvider(aibridge.AnthropicConfig{
|
||||
Name: aibridge.ProviderAnthropic,
|
||||
BaseURL: cfg.Anthropic.BaseURL.String(),
|
||||
Key: cfg.Anthropic.Key.String(),
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}, getBedrockConfig(cfg.Bedrock)),
|
||||
aibridge.NewCopilotProvider(aibridge.CopilotConfig{
|
||||
Name: aibridge.ProviderCopilot,
|
||||
CircuitBreaker: cbConfig,
|
||||
}),
|
||||
aibridge.NewCopilotProvider(aibridge.CopilotConfig{
|
||||
Name: agplaibridge.ProviderCopilotBusiness,
|
||||
BaseURL: "https://" + agplaibridge.HostCopilotBusiness,
|
||||
CircuitBreaker: cbConfig,
|
||||
}),
|
||||
aibridge.NewCopilotProvider(aibridge.CopilotConfig{
|
||||
Name: agplaibridge.ProviderCopilotEnterprise,
|
||||
BaseURL: "https://" + agplaibridge.HostCopilotEnterprise,
|
||||
CircuitBreaker: cbConfig,
|
||||
}),
|
||||
aibridge.NewOpenAIProvider(aibridge.OpenAIConfig{
|
||||
Name: agplaibridge.ProviderChatGPT,
|
||||
BaseURL: agplaibridge.BaseURLChatGPT,
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}),
|
||||
}
|
||||
|
||||
reg := prometheus.WrapRegistererWithPrefix("coder_aibridged_", coderAPI.PrometheusRegistry)
|
||||
metrics := aibridge.NewMetrics(reg)
|
||||
@@ -93,6 +42,122 @@ func newAIBridgeDaemon(coderAPI *coderd.API) (*aibridged.Server, error) {
|
||||
return srv, nil
|
||||
}
|
||||
|
||||
// buildProviders constructs the list of aibridge providers from config.
|
||||
// It merges legacy single-provider env vars and indexed provider configs:
|
||||
// 1. Legacy providers (from CODER_AIBRIDGE_OPENAI_KEY, etc.) are added first.
|
||||
// If a legacy name conflicts with an indexed provider, startup fails with
|
||||
// a clear error asking the admin to remove one or the other.
|
||||
// 2. Indexed providers (from CODER_AIBRIDGE_PROVIDER_<N>_*) are added next.
|
||||
func buildProviders(cfg codersdk.AIBridgeConfig) ([]aibridge.Provider, error) {
|
||||
var cbConfig *config.CircuitBreaker
|
||||
if cfg.CircuitBreakerEnabled.Value() {
|
||||
cbConfig = &config.CircuitBreaker{
|
||||
FailureThreshold: uint32(cfg.CircuitBreakerFailureThreshold.Value()), //nolint:gosec // Validated by serpent.Validate in deployment options.
|
||||
Interval: cfg.CircuitBreakerInterval.Value(),
|
||||
Timeout: cfg.CircuitBreakerTimeout.Value(),
|
||||
MaxRequests: uint32(cfg.CircuitBreakerMaxRequests.Value()), //nolint:gosec // Validated by serpent.Validate in deployment options.
|
||||
}
|
||||
}
|
||||
|
||||
var providers []aibridge.Provider
|
||||
usedNames := make(map[string]struct{})
|
||||
|
||||
// Collect names from indexed providers so we can detect conflicts
|
||||
// with legacy providers.
|
||||
for _, p := range cfg.Providers {
|
||||
name := p.Name
|
||||
if name == "" {
|
||||
name = p.Type
|
||||
}
|
||||
usedNames[name] = struct{}{}
|
||||
}
|
||||
|
||||
// Add legacy OpenAI provider if configured.
|
||||
if cfg.LegacyOpenAI.Key.String() != "" {
|
||||
if _, conflict := usedNames[aibridge.ProviderOpenAI]; conflict {
|
||||
return nil, xerrors.Errorf("legacy CODER_AIBRIDGE_OPENAI_KEY conflicts with indexed provider named %q; remove one or the other", aibridge.ProviderOpenAI)
|
||||
}
|
||||
providers = append(providers, aibridge.NewOpenAIProvider(aibridge.OpenAIConfig{
|
||||
Name: aibridge.ProviderOpenAI,
|
||||
BaseURL: cfg.LegacyOpenAI.BaseURL.String(),
|
||||
Key: cfg.LegacyOpenAI.Key.String(),
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}))
|
||||
usedNames[aibridge.ProviderOpenAI] = struct{}{}
|
||||
}
|
||||
|
||||
// Add legacy Anthropic provider if configured. Bedrock credentials
|
||||
// alone are sufficient — an Anthropic API key is not required when
|
||||
// using AWS Bedrock.
|
||||
if cfg.LegacyAnthropic.Key.String() != "" || getBedrockConfig(cfg.LegacyBedrock) != nil {
|
||||
if _, conflict := usedNames[aibridge.ProviderAnthropic]; conflict {
|
||||
return nil, xerrors.Errorf("legacy CODER_AIBRIDGE_ANTHROPIC_KEY conflicts with indexed provider named %q; remove one or the other", aibridge.ProviderAnthropic)
|
||||
}
|
||||
providers = append(providers, aibridge.NewAnthropicProvider(aibridge.AnthropicConfig{
|
||||
Name: aibridge.ProviderAnthropic,
|
||||
BaseURL: cfg.LegacyAnthropic.BaseURL.String(),
|
||||
Key: cfg.LegacyAnthropic.Key.String(),
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}, getBedrockConfig(cfg.LegacyBedrock)))
|
||||
usedNames[aibridge.ProviderAnthropic] = struct{}{}
|
||||
}
|
||||
|
||||
// Add indexed providers.
|
||||
for _, p := range cfg.Providers {
|
||||
name := p.Name
|
||||
if name == "" {
|
||||
name = p.Type
|
||||
}
|
||||
switch p.Type {
|
||||
case aibridge.ProviderOpenAI:
|
||||
providers = append(providers, aibridge.NewOpenAIProvider(aibridge.OpenAIConfig{
|
||||
Name: name,
|
||||
BaseURL: p.BaseURL,
|
||||
Key: p.Key,
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}))
|
||||
case aibridge.ProviderAnthropic:
|
||||
providers = append(providers, aibridge.NewAnthropicProvider(aibridge.AnthropicConfig{
|
||||
Name: name,
|
||||
BaseURL: p.BaseURL,
|
||||
Key: p.Key,
|
||||
CircuitBreaker: cbConfig,
|
||||
SendActorHeaders: cfg.SendActorHeaders.Value(),
|
||||
}, bedrockConfigFromProvider(p)))
|
||||
case aibridge.ProviderCopilot:
|
||||
providers = append(providers, aibridge.NewCopilotProvider(aibridge.CopilotConfig{
|
||||
Name: name,
|
||||
BaseURL: p.BaseURL,
|
||||
CircuitBreaker: cbConfig,
|
||||
}))
|
||||
default:
|
||||
return nil, xerrors.Errorf("unknown provider type %q for provider %q", p.Type, name)
|
||||
}
|
||||
}
|
||||
|
||||
return providers, nil
|
||||
}
|
||||
|
||||
// bedrockConfigFromProvider converts Bedrock fields from an indexed
|
||||
// AIBridgeProviderConfig into an aibridge AWSBedrockConfig.
|
||||
// Returns nil if no Bedrock fields are set.
|
||||
func bedrockConfigFromProvider(p codersdk.AIBridgeProviderConfig) *aibridge.AWSBedrockConfig {
|
||||
if p.BedrockRegion == "" && p.BedrockBaseURL == "" && p.BedrockAccessKey == "" && p.BedrockAccessKeySecret == "" {
|
||||
return nil
|
||||
}
|
||||
return &aibridge.AWSBedrockConfig{
|
||||
BaseURL: p.BedrockBaseURL,
|
||||
Region: p.BedrockRegion,
|
||||
AccessKey: p.BedrockAccessKey,
|
||||
AccessKeySecret: p.BedrockAccessKeySecret,
|
||||
Model: p.BedrockModel,
|
||||
SmallFastModel: p.BedrockSmallFastModel,
|
||||
}
|
||||
}
|
||||
|
||||
func getBedrockConfig(cfg codersdk.AIBridgeBedrockConfig) *aibridge.AWSBedrockConfig {
|
||||
if cfg.Region.String() == "" && cfg.BaseURL.String() == "" && cfg.AccessKey.String() == "" && cfg.AccessKeySecret.String() == "" {
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
//go:build !slim
|
||||
|
||||
package cli
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/aibridge"
|
||||
agplaibridge "github.com/coder/coder/v2/coderd/aibridge"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/serpent"
|
||||
)
|
||||
|
||||
func TestBuildProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("EmptyConfig", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
providers, err := buildProviders(codersdk.AIBridgeConfig{})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, providers)
|
||||
})
|
||||
|
||||
t.Run("LegacyOnly", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{}
|
||||
cfg.LegacyOpenAI.Key = serpent.String("sk-openai")
|
||||
cfg.LegacyAnthropic.Key = serpent.String("sk-anthropic")
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
names := providerNames(providers)
|
||||
assert.Contains(t, names, aibridge.ProviderOpenAI)
|
||||
assert.Contains(t, names, aibridge.ProviderAnthropic)
|
||||
assert.Len(t, names, 2)
|
||||
})
|
||||
|
||||
t.Run("IndexedOnly", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderAnthropic, Name: "anthropic-zdr", Key: "sk-zdr"},
|
||||
{Type: aibridge.ProviderOpenAI, Name: "openai-azure", Key: "sk-azure", BaseURL: "https://azure.openai.com"},
|
||||
},
|
||||
}
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
names := providerNames(providers)
|
||||
assert.Equal(t, []string{"anthropic-zdr", "openai-azure"}, names)
|
||||
})
|
||||
|
||||
t.Run("LegacyOpenAIConflictsWithIndexed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderOpenAI, Name: aibridge.ProviderOpenAI, Key: "sk-indexed"},
|
||||
},
|
||||
}
|
||||
cfg.LegacyOpenAI.Key = serpent.String("sk-legacy")
|
||||
|
||||
_, err := buildProviders(cfg)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "conflicts with indexed provider")
|
||||
})
|
||||
|
||||
t.Run("LegacyAnthropicConflictsWithIndexed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderAnthropic, Name: aibridge.ProviderAnthropic, Key: "sk-indexed"},
|
||||
},
|
||||
}
|
||||
cfg.LegacyAnthropic.Key = serpent.String("sk-legacy")
|
||||
|
||||
_, err := buildProviders(cfg)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "conflicts with indexed provider")
|
||||
})
|
||||
|
||||
t.Run("MixedLegacyAndIndexed", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderAnthropic, Name: "anthropic-zdr", Key: "sk-zdr"},
|
||||
},
|
||||
}
|
||||
cfg.LegacyOpenAI.Key = serpent.String("sk-openai")
|
||||
cfg.LegacyAnthropic.Key = serpent.String("sk-anthropic")
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
names := providerNames(providers)
|
||||
assert.Contains(t, names, aibridge.ProviderOpenAI)
|
||||
assert.Contains(t, names, aibridge.ProviderAnthropic)
|
||||
assert.Contains(t, names, "anthropic-zdr")
|
||||
})
|
||||
|
||||
t.Run("LegacyAnthropicWithBedrock", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{}
|
||||
cfg.LegacyAnthropic.Key = serpent.String("sk-anthropic")
|
||||
cfg.LegacyBedrock.Region = serpent.String("us-west-2")
|
||||
cfg.LegacyBedrock.AccessKey = serpent.String("AKID")
|
||||
cfg.LegacyBedrock.AccessKeySecret = serpent.String("secret")
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
names := providerNames(providers)
|
||||
assert.Equal(t, []string{aibridge.ProviderAnthropic}, names)
|
||||
})
|
||||
|
||||
t.Run("LegacyBedrockWithoutAnthropicKey", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Bedrock credentials alone should be enough to create an
|
||||
// Anthropic provider — no CODER_AIBRIDGE_ANTHROPIC_KEY needed.
|
||||
cfg := codersdk.AIBridgeConfig{}
|
||||
cfg.LegacyBedrock.Region = serpent.String("us-west-2")
|
||||
cfg.LegacyBedrock.AccessKey = serpent.String("AKID")
|
||||
cfg.LegacyBedrock.AccessKeySecret = serpent.String("secret")
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, providers, 1)
|
||||
|
||||
p := providers[0]
|
||||
assert.Equal(t, aibridge.ProviderAnthropic, p.Type())
|
||||
assert.Equal(t, aibridge.ProviderAnthropic, p.Name())
|
||||
})
|
||||
|
||||
t.Run("UnknownType", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: "gemini", Name: "gemini-pro"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := buildProviders(cfg)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unknown provider type")
|
||||
})
|
||||
|
||||
t.Run("CopilotVariants", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Copilot providers can target any of the three GitHub
|
||||
// Copilot API hosts via an explicit BASE_URL.
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderCopilot, Name: aibridge.ProviderCopilot},
|
||||
{Type: aibridge.ProviderCopilot, Name: agplaibridge.ProviderCopilotBusiness, BaseURL: "https://" + agplaibridge.HostCopilotBusiness},
|
||||
{Type: aibridge.ProviderCopilot, Name: agplaibridge.ProviderCopilotEnterprise, BaseURL: "https://" + agplaibridge.HostCopilotEnterprise},
|
||||
},
|
||||
}
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, providers, 3)
|
||||
|
||||
assert.Equal(t, aibridge.ProviderCopilot, providers[0].Name())
|
||||
assert.Equal(t, agplaibridge.ProviderCopilotBusiness, providers[1].Name())
|
||||
assert.Equal(t, "https://"+agplaibridge.HostCopilotBusiness, providers[1].BaseURL())
|
||||
assert.Equal(t, agplaibridge.ProviderCopilotEnterprise, providers[2].Name())
|
||||
assert.Equal(t, "https://"+agplaibridge.HostCopilotEnterprise, providers[2].BaseURL())
|
||||
})
|
||||
|
||||
t.Run("ChatGPTProvider", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
// ChatGPT is an OpenAI-compatible provider with a custom
|
||||
// base URL. Admins configure it as an indexed openai provider.
|
||||
cfg := codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderOpenAI, Name: agplaibridge.ProviderChatGPT, BaseURL: agplaibridge.BaseURLChatGPT},
|
||||
},
|
||||
}
|
||||
|
||||
providers, err := buildProviders(cfg)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, providers, 1)
|
||||
|
||||
assert.Equal(t, agplaibridge.ProviderChatGPT, providers[0].Name())
|
||||
assert.Equal(t, agplaibridge.BaseURLChatGPT, providers[0].BaseURL())
|
||||
})
|
||||
}
|
||||
|
||||
func providerNames(providers []aibridge.Provider) []string {
|
||||
names := make([]string, len(providers))
|
||||
for i, p := range providers {
|
||||
names[i] = p.Name()
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func TestDomainsFromProviders(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("ExtractsHostnames", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
providers, err := buildProviders(codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderOpenAI, Name: "openai", Key: "k"},
|
||||
{Type: aibridge.ProviderAnthropic, Name: "anthropic", Key: "k"},
|
||||
{Type: aibridge.ProviderOpenAI, Name: "custom", Key: "k", BaseURL: "https://custom-llm.example.com:8443/api"},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
domains, mapping := domainsFromProviders(providers)
|
||||
|
||||
assert.Contains(t, domains, "api.openai.com")
|
||||
assert.Contains(t, domains, "api.anthropic.com")
|
||||
assert.Contains(t, domains, "custom-llm.example.com")
|
||||
|
||||
assert.Equal(t, "openai", mapping("api.openai.com"))
|
||||
assert.Equal(t, "anthropic", mapping("api.anthropic.com"))
|
||||
assert.Equal(t, "custom", mapping("custom-llm.example.com"))
|
||||
assert.Empty(t, mapping("unknown.com"))
|
||||
})
|
||||
|
||||
t.Run("DeduplicatesSameHost", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
providers, err := buildProviders(codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderOpenAI, Name: "first", Key: "k", BaseURL: "https://api.example.com/v1"},
|
||||
{Type: aibridge.ProviderOpenAI, Name: "second", Key: "k", BaseURL: "https://api.example.com/v2"},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
domains, mapping := domainsFromProviders(providers)
|
||||
|
||||
// Count occurrences of api.example.com.
|
||||
count := 0
|
||||
for _, d := range domains {
|
||||
if d == "api.example.com" {
|
||||
count++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 1, count)
|
||||
// First provider wins.
|
||||
assert.Equal(t, "first", mapping("api.example.com"))
|
||||
})
|
||||
|
||||
t.Run("CaseInsensitive", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
providers, err := buildProviders(codersdk.AIBridgeConfig{
|
||||
Providers: []codersdk.AIBridgeProviderConfig{
|
||||
{Type: aibridge.ProviderOpenAI, Name: "provider", Key: "k", BaseURL: "https://API.Example.COM/v1"},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
domains, mapping := domainsFromProviders(providers)
|
||||
|
||||
assert.Contains(t, domains, "api.example.com")
|
||||
assert.Equal(t, "provider", mapping("API.Example.COM"))
|
||||
assert.Equal(t, "provider", mapping("api.example.com"))
|
||||
})
|
||||
}
|
||||
@@ -4,35 +4,41 @@ package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/coder/aibridge"
|
||||
"github.com/coder/coder/v2/enterprise/aibridgeproxyd"
|
||||
"github.com/coder/coder/v2/enterprise/coderd"
|
||||
)
|
||||
|
||||
func newAIBridgeProxyDaemon(coderAPI *coderd.API) (*aibridgeproxyd.Server, error) {
|
||||
func newAIBridgeProxyDaemon(coderAPI *coderd.API, providers []aibridge.Provider) (*aibridgeproxyd.Server, error) {
|
||||
ctx := context.Background()
|
||||
coderAPI.Logger.Debug(ctx, "starting in-memory aibridgeproxy daemon")
|
||||
|
||||
logger := coderAPI.Logger.Named("aibridgeproxyd")
|
||||
|
||||
domains, providerFromHost := domainsFromProviders(providers)
|
||||
|
||||
reg := prometheus.WrapRegistererWithPrefix("coder_aibridgeproxyd_", coderAPI.PrometheusRegistry)
|
||||
metrics := aibridgeproxyd.NewMetrics(reg)
|
||||
|
||||
srv, err := aibridgeproxyd.New(ctx, logger, aibridgeproxyd.Options{
|
||||
ListenAddr: coderAPI.DeploymentValues.AI.BridgeProxyConfig.ListenAddr.String(),
|
||||
TLSCertFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.TLSCertFile.String(),
|
||||
TLSKeyFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.TLSKeyFile.String(),
|
||||
CoderAccessURL: coderAPI.AccessURL.String(),
|
||||
MITMCertFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.MITMCertFile.String(),
|
||||
MITMKeyFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.MITMKeyFile.String(),
|
||||
DomainAllowlist: coderAPI.DeploymentValues.AI.BridgeProxyConfig.DomainAllowlist.Value(),
|
||||
UpstreamProxy: coderAPI.DeploymentValues.AI.BridgeProxyConfig.UpstreamProxy.String(),
|
||||
UpstreamProxyCA: coderAPI.DeploymentValues.AI.BridgeProxyConfig.UpstreamProxyCA.String(),
|
||||
AllowedPrivateCIDRs: coderAPI.DeploymentValues.AI.BridgeProxyConfig.AllowedPrivateCIDRs.Value(),
|
||||
Metrics: metrics,
|
||||
ListenAddr: coderAPI.DeploymentValues.AI.BridgeProxyConfig.ListenAddr.String(),
|
||||
TLSCertFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.TLSCertFile.String(),
|
||||
TLSKeyFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.TLSKeyFile.String(),
|
||||
CoderAccessURL: coderAPI.AccessURL.String(),
|
||||
MITMCertFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.MITMCertFile.String(),
|
||||
MITMKeyFile: coderAPI.DeploymentValues.AI.BridgeProxyConfig.MITMKeyFile.String(),
|
||||
DomainAllowlist: domains,
|
||||
AIBridgeProviderFromHost: providerFromHost,
|
||||
UpstreamProxy: coderAPI.DeploymentValues.AI.BridgeProxyConfig.UpstreamProxy.String(),
|
||||
UpstreamProxyCA: coderAPI.DeploymentValues.AI.BridgeProxyConfig.UpstreamProxyCA.String(),
|
||||
AllowedPrivateCIDRs: coderAPI.DeploymentValues.AI.BridgeProxyConfig.AllowedPrivateCIDRs.Value(),
|
||||
Metrics: metrics,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("failed to start in-memory aibridgeproxy daemon: %w", err)
|
||||
@@ -40,3 +46,35 @@ func newAIBridgeProxyDaemon(coderAPI *coderd.API) (*aibridgeproxyd.Server, error
|
||||
|
||||
return srv, nil
|
||||
}
|
||||
|
||||
// domainsFromProviders extracts distinct hostnames from providers' base
|
||||
// URLs and builds a host-to-provider-name mapping function. The returned
|
||||
// domain list is suitable for use as DomainAllowlist and the mapping
|
||||
// function is suitable for use as AIBridgeProviderFromHost.
|
||||
func domainsFromProviders(providers []aibridge.Provider) ([]string, func(string) string) {
|
||||
hostToProvider := make(map[string]string, len(providers))
|
||||
var domains []string
|
||||
for _, p := range providers {
|
||||
raw := p.BaseURL()
|
||||
if raw == "" {
|
||||
continue
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
continue
|
||||
}
|
||||
host := strings.ToLower(u.Hostname())
|
||||
if _, exists := hostToProvider[host]; exists {
|
||||
// First provider wins; duplicates are expected when
|
||||
// multiple providers share a base URL host (e.g. two
|
||||
// OpenAI providers using the same proxy).
|
||||
continue
|
||||
}
|
||||
hostToProvider[host] = p.Name()
|
||||
domains = append(domains, host)
|
||||
}
|
||||
|
||||
return domains, func(host string) string {
|
||||
return hostToProvider[strings.ToLower(host)]
|
||||
}
|
||||
}
|
||||
|
||||
+36
-26
@@ -18,7 +18,6 @@ import (
|
||||
agplcoderd "github.com/coder/coder/v2/coderd"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/cryptorand"
|
||||
"github.com/coder/coder/v2/enterprise/aibridged"
|
||||
"github.com/coder/coder/v2/enterprise/audit"
|
||||
"github.com/coder/coder/v2/enterprise/audit/backends"
|
||||
"github.com/coder/coder/v2/enterprise/coderd"
|
||||
@@ -162,38 +161,49 @@ func (r *RootCmd) Server(_ func()) *serpent.Command {
|
||||
usageCron.Start(ctx)
|
||||
closers.Add(usageCron)
|
||||
|
||||
// In-memory aibridge daemon.
|
||||
// TODO(@deansheather): the lifecycle of the aibridged server is
|
||||
// probably better managed by the enterprise API type itself. Managing
|
||||
// it in the API type means we can avoid starting it up when the license
|
||||
// is not entitled to the feature.
|
||||
var aibridgeDaemon *aibridged.Server
|
||||
if options.DeploymentValues.AI.BridgeConfig.Enabled {
|
||||
aibridgeDaemon, err = newAIBridgeDaemon(api)
|
||||
// Build the provider list and start AI Bridge daemons only when
|
||||
// at least one of the bridge or proxy features is enabled.
|
||||
bridgeEnabled := options.DeploymentValues.AI.BridgeConfig.Enabled.Value()
|
||||
proxyEnabled := options.DeploymentValues.AI.BridgeProxyConfig.Enabled.Value()
|
||||
if bridgeEnabled || proxyEnabled {
|
||||
providers, err := buildProviders(options.DeploymentValues.AI.BridgeConfig)
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("create aibridged: %w", err)
|
||||
return nil, nil, xerrors.Errorf("build aibridge providers: %w", err)
|
||||
}
|
||||
|
||||
api.RegisterInMemoryAIBridgedHTTPHandler(aibridgeDaemon)
|
||||
// In-memory aibridge daemon.
|
||||
// TODO(@deansheather): the lifecycle of the aibridged server is
|
||||
// probably better managed by the enterprise API type itself. Managing
|
||||
// it in the API type means we can avoid starting it up when the license
|
||||
// is not entitled to the feature.
|
||||
if bridgeEnabled {
|
||||
aibridgeDaemon, err := newAIBridgeDaemon(api, providers)
|
||||
if err != nil {
|
||||
return nil, nil, xerrors.Errorf("create aibridged: %w", err)
|
||||
}
|
||||
|
||||
// When running as an in-memory daemon, the HTTP handler is wired into the
|
||||
// coderd API and therefore is subject to its context. Calling Close() on
|
||||
// aibridged will NOT affect in-flight requests but those will be closed once
|
||||
// the API server is itself shutdown.
|
||||
closers.Add(aibridgeDaemon)
|
||||
}
|
||||
api.RegisterInMemoryAIBridgedHTTPHandler(aibridgeDaemon)
|
||||
|
||||
// In-memory AI Bridge Proxy daemon
|
||||
if options.DeploymentValues.AI.BridgeProxyConfig.Enabled.Value() {
|
||||
aiBridgeProxyServer, err := newAIBridgeProxyDaemon(api)
|
||||
if err != nil {
|
||||
_ = closers.Close()
|
||||
return nil, nil, xerrors.Errorf("create aibridgeproxyd: %w", err)
|
||||
// When running as an in-memory daemon, the HTTP handler is
|
||||
// wired into the coderd API and therefore is subject to its
|
||||
// context. Calling Close() on aibridged will NOT affect
|
||||
// in-flight requests but those will be closed once the API
|
||||
// server is itself shutdown.
|
||||
closers.Add(aibridgeDaemon)
|
||||
}
|
||||
closers.Add(aiBridgeProxyServer)
|
||||
|
||||
// Register the handler so coderd can serve the proxy endpoints.
|
||||
api.RegisterInMemoryAIBridgeProxydHTTPHandler(aiBridgeProxyServer.Handler())
|
||||
// In-memory AI Bridge Proxy daemon.
|
||||
if proxyEnabled {
|
||||
aiBridgeProxyServer, err := newAIBridgeProxyDaemon(api, providers)
|
||||
if err != nil {
|
||||
_ = closers.Close()
|
||||
return nil, nil, xerrors.Errorf("create aibridgeproxyd: %w", err)
|
||||
}
|
||||
closers.Add(aiBridgeProxyServer)
|
||||
|
||||
// Register the handler so coderd can serve the proxy endpoints.
|
||||
api.RegisterInMemoryAIBridgeProxydHTTPHandler(aiBridgeProxyServer.Handler())
|
||||
}
|
||||
}
|
||||
|
||||
return api.AGPL, closers, nil
|
||||
|
||||
Reference in New Issue
Block a user