mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
chore: bump fantasy to sync from upstream (#26440)
Closes CODAGT-572 ## Overview Bumps `charm.land/fantasy` to the head of `coder_2_33` (`v0.0.0-20260617050554-2e3ddbca75dd`) and adapts `chatd` to it. The fantasy bump: - Syncs upstream `charmbracelet/fantasy` main (v0.31.0) into `coder_2_33` (coder/fantasy#42). - Mirrors the request region when prefixing cross-region inference profiles, so a legacy (un-qualified) Bedrock model ID is prefixed for the same region the request is actually signed for. Pulling in the new fantasy version propagates its required transitive dependency upgrades (aws-sdk-go-v2, OpenTelemetry, google genai, `golang.org/x/*`, etc.) through MVS, which accounts for the bulk of the `go.mod`/`go.sum` churn. ## chatd changes - Thread a per-provider `Region` through `ConfiguredProvider` and `ProviderAPIKeys` (`RegionByProvider`), and merge/prune/resolve it alongside API keys and base URLs. - Source the Bedrock region from AI provider settings in `chatd` and pass `fantasybedrock.WithRegion` when a region is configured. - Migrate the runtime Bedrock title-generation model ID to a fully-qualified `global.anthropic.*` ID. - Emit a `finish_reason` in the test OpenAI streaming server so streams close on a terminal event, matching fantasy's fail-closed stream handling. ## Heads-up: most of this is short-lived Almost all of the `chatd` code in this PR only executes on the **direct (non-gateway) routing path** — the branch taken when `AIGatewayRoutingEnabled` is `false`. That flag was a transition crutch for AI Gateway routing, and it (plus the entire direct path / `x/chatd/chatprovider` package that backs it) is slated for removal in CODAGT-598. Under AI Gateway routing — which is the path every deployment is expected to run — the Bedrock region is resolved by aibridge directly from provider settings (`cli/aibridged.go` builds `aibridge.AWSBedrockConfig{Region: settings.Bedrock.Region}`), so none of the region plumbing added here is reached. Concretely, expect the following to be deleted alongside the direct path: - The `RegionByProvider` map, the `Region()` accessor, and the region preservation in merge/resolve plus the region pruning in `PruneDisabledProviderKeys` (`chatprovider.go`). - The `fantasybedrock.WithRegion(region)` branch in `ModelFromConfig` — only reachable on the direct path; the gateway path builds a `fantasyanthropic` client with no region key. - Reading `settings.Bedrock.Region` in `aiProviderConfigFromKeys` (`chatd.go`). - The region-specific tests in `chatprovider_test.go`, and the `chattest`/`model_coverage` adjustments that support direct-path testing. What survives the cleanup (independent of routing): - The `charm.land/fantasy` bump and its `go.mod`/`go.sum` transitive churn. - The fully-qualified `global.anthropic.*` Bedrock title-generation model ID in `quickgen.go` (a runtime-valid model identifier, not direct-path-specific). We're landing the full change anyway so the direct path stays correct for the remaining transition window; just don't be surprised when CODAGT-598 reclaims most of it. ## Notes Depends on coder/fantasy `coder_2_33` already containing the upstream sync and Bedrock region fix (merged via coder/fantasy#42 and coder/fantasy#43).
This commit is contained in:
@@ -4406,6 +4406,11 @@ func (p *Server) aiProviderConfigFromKeys(provider database.AIProvider, keys []d
|
||||
if !provider.Enabled {
|
||||
return chatprovider.ConfiguredProvider{}, xerrors.Errorf("AI provider %s is disabled", provider.ID)
|
||||
}
|
||||
settings, err := db2sdk.AIProviderSettings(provider.Settings)
|
||||
if err != nil {
|
||||
return chatprovider.ConfiguredProvider{}, xerrors.Errorf("decode AI provider settings: %w", err)
|
||||
}
|
||||
|
||||
apiKey := ""
|
||||
// GetAIProviderKeysByProviderID orders keys oldest first. chatd consumes
|
||||
// one provider-scoped key because runtime provider config has one API key slot.
|
||||
@@ -4415,11 +4420,16 @@ func (p *Server) aiProviderConfigFromKeys(provider database.AIProvider, keys []d
|
||||
break
|
||||
}
|
||||
}
|
||||
region := ""
|
||||
if settings.Bedrock != nil {
|
||||
region = strings.TrimSpace(settings.Bedrock.Region)
|
||||
}
|
||||
return chatprovider.ConfiguredProvider{
|
||||
ProviderID: provider.ID,
|
||||
Provider: string(provider.Type),
|
||||
APIKey: apiKey,
|
||||
BaseURL: provider.BaseUrl,
|
||||
Region: region,
|
||||
CentralAPIKeyEnabled: true,
|
||||
AllowUserAPIKey: p.allowBYOK,
|
||||
AllowCentralAPIKeyFallback: true,
|
||||
|
||||
@@ -262,6 +262,7 @@ func TestNormalizationFieldCoverage(t *testing.T) {
|
||||
"ClientMetadata": "skipped: client execution metadata not needed for debug panel",
|
||||
"ProviderExecuted": "skipped: provider vs client distinction not needed for debug panel",
|
||||
"ProviderMetadata": "skipped: opaque provider-specific metadata",
|
||||
"StopTurn": "skipped: control-flow flag is not rendered in the debug panel",
|
||||
},
|
||||
},
|
||||
|
||||
|
||||
@@ -82,6 +82,7 @@ type ProviderAPIKeys struct {
|
||||
Anthropic string
|
||||
ByProvider map[string]string
|
||||
BaseURLByProvider map[string]string
|
||||
RegionByProvider map[string]string
|
||||
}
|
||||
|
||||
// Empty reports whether no provider keys or base URL overrides are set.
|
||||
@@ -89,7 +90,8 @@ func (k ProviderAPIKeys) Empty() bool {
|
||||
return k.OpenAI == "" &&
|
||||
k.Anthropic == "" &&
|
||||
len(k.ByProvider) == 0 &&
|
||||
len(k.BaseURLByProvider) == 0
|
||||
len(k.BaseURLByProvider) == 0 &&
|
||||
len(k.RegionByProvider) == 0
|
||||
}
|
||||
|
||||
// UserProviderKey is a user-supplied API key for a specific provider.
|
||||
@@ -111,6 +113,7 @@ type ConfiguredProvider struct {
|
||||
Provider string
|
||||
APIKey string
|
||||
BaseURL string
|
||||
Region string
|
||||
CentralAPIKeyEnabled bool
|
||||
AllowUserAPIKey bool
|
||||
AllowCentralAPIKeyFallback bool
|
||||
@@ -166,6 +169,15 @@ func (k ProviderAPIKeys) BaseURL(provider string) string {
|
||||
return strings.TrimSpace(k.BaseURLByProvider[normalized])
|
||||
}
|
||||
|
||||
// Region returns the configured region for a provider.
|
||||
func (k ProviderAPIKeys) Region(provider string) string {
|
||||
normalized := NormalizeProvider(provider)
|
||||
if normalized == "" || k.RegionByProvider == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(k.RegionByProvider[normalized])
|
||||
}
|
||||
|
||||
// ProviderBaseURLHostname returns the normalized hostname from a provider base URL.
|
||||
func ProviderBaseURLHostname(baseURL string) string {
|
||||
parsed, ok := parseProviderBaseURL(baseURL)
|
||||
@@ -190,8 +202,28 @@ func parseProviderBaseURL(baseURL string) (*neturl.URL, bool) {
|
||||
return parsed, true
|
||||
}
|
||||
|
||||
// MergeProviderAPIKeys overlays configured provider keys over fallback keys.
|
||||
func MergeProviderAPIKeys(fallback ProviderAPIKeys, providers []ConfiguredProvider) ProviderAPIKeys {
|
||||
// setRegion records a normalized, non-empty region for a provider. The
|
||||
// RegionByProvider map is allocated lazily so an unused set stays nil, which
|
||||
// keeps Empty() and value comparisons stable.
|
||||
func (k *ProviderAPIKeys) setRegion(provider, region string) {
|
||||
if provider == "" {
|
||||
return
|
||||
}
|
||||
if region = strings.TrimSpace(region); region == "" {
|
||||
return
|
||||
}
|
||||
if k.RegionByProvider == nil {
|
||||
k.RegionByProvider = map[string]string{}
|
||||
}
|
||||
k.RegionByProvider[provider] = region
|
||||
}
|
||||
|
||||
// mergedFromFallback seeds a ProviderAPIKeys from a fallback set, normalizing
|
||||
// provider names and dropping blank values. ByProvider and BaseURLByProvider
|
||||
// are always non-nil; RegionByProvider stays nil until a region is set. The
|
||||
// legacy OpenAI/Anthropic keys are mirrored into ByProvider so callers can
|
||||
// look them up by provider name.
|
||||
func mergedFromFallback(fallback ProviderAPIKeys) ProviderAPIKeys {
|
||||
merged := ProviderAPIKeys{
|
||||
OpenAI: strings.TrimSpace(fallback.OpenAI),
|
||||
Anthropic: strings.TrimSpace(fallback.Anthropic),
|
||||
@@ -199,30 +231,34 @@ func MergeProviderAPIKeys(fallback ProviderAPIKeys, providers []ConfiguredProvid
|
||||
BaseURLByProvider: map[string]string{},
|
||||
}
|
||||
for provider, apiKey := range fallback.ByProvider {
|
||||
normalizedProvider := NormalizeProvider(provider)
|
||||
if normalizedProvider == "" {
|
||||
continue
|
||||
}
|
||||
if key := strings.TrimSpace(apiKey); key != "" {
|
||||
merged.ByProvider[normalizedProvider] = key
|
||||
if normalized := NormalizeProvider(provider); normalized != "" {
|
||||
if key := strings.TrimSpace(apiKey); key != "" {
|
||||
merged.ByProvider[normalized] = key
|
||||
}
|
||||
}
|
||||
}
|
||||
for provider, baseURL := range fallback.BaseURLByProvider {
|
||||
normalizedProvider := NormalizeProvider(provider)
|
||||
if normalizedProvider == "" {
|
||||
continue
|
||||
}
|
||||
if url := strings.TrimSpace(baseURL); url != "" {
|
||||
merged.BaseURLByProvider[normalizedProvider] = url
|
||||
if normalized := NormalizeProvider(provider); normalized != "" {
|
||||
if url := strings.TrimSpace(baseURL); url != "" {
|
||||
merged.BaseURLByProvider[normalized] = url
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for provider, region := range fallback.RegionByProvider {
|
||||
merged.setRegion(NormalizeProvider(provider), region)
|
||||
}
|
||||
if merged.OpenAI != "" {
|
||||
merged.ByProvider[fantasyopenai.Name] = merged.OpenAI
|
||||
}
|
||||
if merged.Anthropic != "" {
|
||||
merged.ByProvider[fantasyanthropic.Name] = merged.Anthropic
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// MergeProviderAPIKeys overlays configured provider keys over fallback keys.
|
||||
func MergeProviderAPIKeys(fallback ProviderAPIKeys, providers []ConfiguredProvider) ProviderAPIKeys {
|
||||
merged := mergedFromFallback(fallback)
|
||||
|
||||
for _, provider := range providers {
|
||||
normalizedProvider := NormalizeProvider(provider.Provider)
|
||||
@@ -236,6 +272,7 @@ func MergeProviderAPIKeys(fallback ProviderAPIKeys, providers []ConfiguredProvid
|
||||
if url := strings.TrimSpace(provider.BaseURL); url != "" {
|
||||
merged.BaseURLByProvider[normalizedProvider] = url
|
||||
}
|
||||
merged.setRegion(normalizedProvider, provider.Region)
|
||||
|
||||
switch normalizedProvider {
|
||||
case fantasyopenai.Name:
|
||||
@@ -261,36 +298,7 @@ func ResolveUserProviderKeys(
|
||||
providers []ConfiguredProvider,
|
||||
userKeys []UserProviderKey,
|
||||
) (ProviderAPIKeys, map[string]ProviderAvailability) {
|
||||
merged := ProviderAPIKeys{
|
||||
OpenAI: strings.TrimSpace(fallback.OpenAI),
|
||||
Anthropic: strings.TrimSpace(fallback.Anthropic),
|
||||
ByProvider: map[string]string{},
|
||||
BaseURLByProvider: map[string]string{},
|
||||
}
|
||||
for provider, apiKey := range fallback.ByProvider {
|
||||
normalizedProvider := NormalizeProvider(provider)
|
||||
if normalizedProvider == "" {
|
||||
continue
|
||||
}
|
||||
if key := strings.TrimSpace(apiKey); key != "" {
|
||||
merged.ByProvider[normalizedProvider] = key
|
||||
}
|
||||
}
|
||||
for provider, baseURL := range fallback.BaseURLByProvider {
|
||||
normalizedProvider := NormalizeProvider(provider)
|
||||
if normalizedProvider == "" {
|
||||
continue
|
||||
}
|
||||
if url := strings.TrimSpace(baseURL); url != "" {
|
||||
merged.BaseURLByProvider[normalizedProvider] = url
|
||||
}
|
||||
}
|
||||
if merged.OpenAI != "" {
|
||||
merged.ByProvider[fantasyopenai.Name] = merged.OpenAI
|
||||
}
|
||||
if merged.Anthropic != "" {
|
||||
merged.ByProvider[fantasyanthropic.Name] = merged.Anthropic
|
||||
}
|
||||
merged := mergedFromFallback(fallback)
|
||||
|
||||
userKeyByProviderID := make(map[uuid.UUID]string, len(userKeys))
|
||||
for _, userKey := range userKeys {
|
||||
@@ -312,6 +320,7 @@ func ResolveUserProviderKeys(
|
||||
if url := strings.TrimSpace(provider.BaseURL); url != "" {
|
||||
merged.BaseURLByProvider[normalizedProvider] = url
|
||||
}
|
||||
merged.setRegion(normalizedProvider, provider.Region)
|
||||
|
||||
var userKey string
|
||||
if provider.ProviderID != uuid.Nil {
|
||||
@@ -515,18 +524,29 @@ func (*ModelCatalog) ListConfiguredProviderAvailability(
|
||||
}
|
||||
|
||||
// PruneDisabledProviderKeys removes entries from keys that do not
|
||||
// belong to an enabled provider. It clears ByProvider and
|
||||
// BaseURLByProvider entries for disabled providers and zeroes the
|
||||
// legacy OpenAI and Anthropic fields when those providers are not
|
||||
// enabled.
|
||||
// belong to an enabled provider. It clears ByProvider,
|
||||
// BaseURLByProvider, and RegionByProvider entries for disabled
|
||||
// providers and zeroes the legacy OpenAI and Anthropic fields when
|
||||
// those providers are not enabled.
|
||||
func PruneDisabledProviderKeys(keys *ProviderAPIKeys, enabledProviders map[string]struct{}) {
|
||||
for provider := range keys.ByProvider {
|
||||
if _, ok := enabledProviders[provider]; ok {
|
||||
continue
|
||||
}
|
||||
delete(keys.ByProvider, provider)
|
||||
}
|
||||
for provider := range keys.BaseURLByProvider {
|
||||
if _, ok := enabledProviders[provider]; ok {
|
||||
continue
|
||||
}
|
||||
delete(keys.BaseURLByProvider, provider)
|
||||
}
|
||||
for provider := range keys.RegionByProvider {
|
||||
if _, ok := enabledProviders[provider]; ok {
|
||||
continue
|
||||
}
|
||||
delete(keys.RegionByProvider, provider)
|
||||
}
|
||||
if _, ok := enabledProviders[NormalizeProvider("openai")]; !ok {
|
||||
keys.OpenAI = ""
|
||||
}
|
||||
@@ -879,6 +899,9 @@ func ModelFromConfig(
|
||||
bedrockOpts := []fantasybedrock.Option{
|
||||
fantasybedrock.WithUserAgent(userAgent),
|
||||
}
|
||||
if region := providerKeys.Region(provider); region != "" {
|
||||
bedrockOpts = append(bedrockOpts, fantasybedrock.WithRegion(region))
|
||||
}
|
||||
if apiKey != "" {
|
||||
bedrockOpts = append(bedrockOpts, fantasybedrock.WithAPIKey(apiKey))
|
||||
}
|
||||
|
||||
@@ -290,6 +290,60 @@ func TestResolveUserProviderKeys(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderAPIKeysEmpty_RegionCountsAsConfigured(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
require.True(t, (chatprovider.ProviderAPIKeys{}).Empty())
|
||||
require.False(t, (chatprovider.ProviderAPIKeys{
|
||||
RegionByProvider: map[string]string{
|
||||
fantasybedrock.Name: "us-east-1",
|
||||
},
|
||||
}).Empty())
|
||||
}
|
||||
|
||||
func TestMergeProviderAPIKeys_PreservesProviderRegions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
merged := chatprovider.MergeProviderAPIKeys(
|
||||
chatprovider.ProviderAPIKeys{
|
||||
RegionByProvider: map[string]string{
|
||||
"BEDROCK": "us-east-1",
|
||||
},
|
||||
},
|
||||
[]chatprovider.ConfiguredProvider{{
|
||||
ProviderID: uuid.New(),
|
||||
Provider: fantasybedrock.Name,
|
||||
Region: "eu-central-1",
|
||||
}},
|
||||
)
|
||||
|
||||
require.Equal(t, "eu-central-1", merged.Region(fantasybedrock.Name))
|
||||
require.False(t, merged.Empty())
|
||||
}
|
||||
|
||||
func TestResolveUserProviderKeys_PreservesProviderRegions(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
keys, availability := chatprovider.ResolveUserProviderKeys(
|
||||
chatprovider.ProviderAPIKeys{
|
||||
RegionByProvider: map[string]string{
|
||||
"BEDROCK": "us-east-1",
|
||||
},
|
||||
},
|
||||
[]chatprovider.ConfiguredProvider{{
|
||||
ProviderID: uuid.New(),
|
||||
Provider: fantasybedrock.Name,
|
||||
Region: "eu-central-1",
|
||||
CentralAPIKeyEnabled: true,
|
||||
}},
|
||||
nil,
|
||||
)
|
||||
|
||||
require.Equal(t, "eu-central-1", keys.Region(fantasybedrock.Name))
|
||||
require.True(t, keys.HasProvider(fantasybedrock.Name))
|
||||
require.Equal(t, chatprovider.ProviderAvailability{Available: true}, availability[fantasybedrock.Name])
|
||||
}
|
||||
|
||||
type roundTripperFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (fn roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
@@ -740,6 +794,21 @@ func TestPruneDisabledProviderKeys(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "DisabledProviderRegionsRemoved",
|
||||
keys: chatprovider.ProviderAPIKeys{
|
||||
RegionByProvider: map[string]string{
|
||||
fantasybedrock.Name: "us-east-2",
|
||||
fantasyopenai.Name: "us-east-1",
|
||||
},
|
||||
},
|
||||
enabledProviders: enabledProviders(fantasybedrock.Name),
|
||||
want: chatprovider.ProviderAPIKeys{
|
||||
RegionByProvider: map[string]string{
|
||||
fantasybedrock.Name: "us-east-2",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "OpenAIDisabledClearsLegacyField",
|
||||
keys: chatprovider.ProviderAPIKeys{
|
||||
@@ -1059,12 +1128,12 @@ func TestModelFromConfig_BedrockStripsAnthropicHeaders(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
|
||||
t.Setenv("ANTHROPIC_API_KEY", "anthropic-env-key")
|
||||
t.Setenv("AWS_REGION", "us-east-2")
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "test-access-key")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
|
||||
t.Setenv("AWS_SESSION_TOKEN", "test-session-token")
|
||||
|
||||
type requestCapture struct {
|
||||
Path string
|
||||
Authorization string
|
||||
AnthropicVersion string
|
||||
XAPIKey string
|
||||
@@ -1077,6 +1146,7 @@ func TestModelFromConfig_BedrockStripsAnthropicHeaders(t *testing.T) {
|
||||
body, err := io.ReadAll(r.Body)
|
||||
|
||||
requests <- requestCapture{
|
||||
Path: r.URL.Path,
|
||||
Authorization: r.Header.Get("Authorization"),
|
||||
AnthropicVersion: r.Header.Get("Anthropic-Version"),
|
||||
XAPIKey: r.Header.Get("X-Api-Key"),
|
||||
@@ -1099,6 +1169,9 @@ func TestModelFromConfig_BedrockStripsAnthropicHeaders(t *testing.T) {
|
||||
BaseURLByProvider: map[string]string{
|
||||
fantasybedrock.Name: server.URL,
|
||||
},
|
||||
RegionByProvider: map[string]string{
|
||||
fantasybedrock.Name: "us-east-2",
|
||||
},
|
||||
},
|
||||
chatprovider.UserAgent(),
|
||||
nil,
|
||||
@@ -1121,6 +1194,7 @@ func TestModelFromConfig_BedrockStripsAnthropicHeaders(t *testing.T) {
|
||||
|
||||
got := testutil.TryReceive(ctx, t, requests)
|
||||
require.NoError(t, got.ReadError)
|
||||
require.Equal(t, "/model/us.anthropic.claude-opus-4-6-v1/invoke", got.Path)
|
||||
require.Empty(t, got.AnthropicVersion)
|
||||
require.Empty(t, got.XAPIKey)
|
||||
require.Contains(t, got.Authorization, "AWS4-HMAC-SHA256")
|
||||
@@ -1133,7 +1207,6 @@ func TestModelFromConfig_BedrockStreamingHeaders(t *testing.T) {
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
|
||||
t.Setenv("ANTHROPIC_API_KEY", "anthropic-env-key")
|
||||
t.Setenv("AWS_REGION", "us-east-2")
|
||||
t.Setenv("AWS_ACCESS_KEY_ID", "test-access-key")
|
||||
t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret-key")
|
||||
t.Setenv("AWS_SESSION_TOKEN", "test-session-token")
|
||||
@@ -1162,6 +1235,7 @@ func TestModelFromConfig_BedrockStreamingHeaders(t *testing.T) {
|
||||
|
||||
if err := writeBedrockAnthropicStream(w,
|
||||
`{"type":"message_start","message":{}}`,
|
||||
`{"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":1}}`,
|
||||
`{"type":"message_stop"}`,
|
||||
); err != nil {
|
||||
t.Errorf("write bedrock stream: %v", err)
|
||||
@@ -1171,7 +1245,7 @@ func TestModelFromConfig_BedrockStreamingHeaders(t *testing.T) {
|
||||
|
||||
model, err := chatprovider.ModelFromConfig(
|
||||
fantasybedrock.Name,
|
||||
"anthropic.claude-opus-4-6-v1",
|
||||
"global.anthropic.claude-opus-4-6-v1",
|
||||
chatprovider.ProviderAPIKeys{
|
||||
ByProvider: map[string]string{
|
||||
fantasybedrock.Name: "",
|
||||
@@ -1179,6 +1253,9 @@ func TestModelFromConfig_BedrockStreamingHeaders(t *testing.T) {
|
||||
BaseURLByProvider: map[string]string{
|
||||
fantasybedrock.Name: server.URL,
|
||||
},
|
||||
RegionByProvider: map[string]string{
|
||||
fantasybedrock.Name: "us-east-2",
|
||||
},
|
||||
},
|
||||
chatprovider.UserAgent(),
|
||||
nil,
|
||||
@@ -1201,12 +1278,11 @@ func TestModelFromConfig_BedrockStreamingHeaders(t *testing.T) {
|
||||
|
||||
for part := range stream {
|
||||
require.NotEqual(t, fantasy.StreamPartTypeError, part.Type)
|
||||
break
|
||||
}
|
||||
|
||||
got := testutil.TryReceive(ctx, t, requests)
|
||||
require.NoError(t, got.ReadError)
|
||||
require.Equal(t, "/model/us.anthropic.claude-opus-4-6-v1/invoke-with-response-stream", got.Path)
|
||||
require.Equal(t, "/model/global.anthropic.claude-opus-4-6-v1/invoke-with-response-stream", got.Path)
|
||||
require.Empty(t, got.Accept)
|
||||
require.Equal(t, "application/json", got.BedrockAccept)
|
||||
require.Contains(t, got.Authorization, "AWS4-HMAC-SHA256")
|
||||
|
||||
@@ -331,21 +331,7 @@ func writeChatCompletionsStreaming(w http.ResponseWriter, r *http.Request, chunk
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
var chunk OpenAIChunk
|
||||
var ok bool
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
log.Printf("writeChatCompletionsStreaming: request context canceled, stopping stream")
|
||||
return
|
||||
case chunk, ok = <-chunks:
|
||||
if !ok {
|
||||
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
writeChunk := func(chunk OpenAIChunk) bool {
|
||||
choicesData := make([]map[string]interface{}, len(chunk.Choices))
|
||||
for i, choice := range chunk.Choices {
|
||||
choiceData := map[string]interface{}{
|
||||
@@ -369,6 +355,9 @@ func writeChatCompletionsStreaming(w http.ResponseWriter, r *http.Request, chunk
|
||||
delta["tool_calls"] = choice.ToolCalls
|
||||
}
|
||||
if choice.FinishReason != "" {
|
||||
if choiceData["delta"] == nil {
|
||||
choiceData["delta"] = map[string]interface{}{}
|
||||
}
|
||||
choiceData["finish_reason"] = choice.FinishReason
|
||||
}
|
||||
choicesData[i] = choiceData
|
||||
@@ -384,13 +373,69 @@ func writeChatCompletionsStreaming(w http.ResponseWriter, r *http.Request, chunk
|
||||
|
||||
chunkBytes, err := json.Marshal(chunkData)
|
||||
if err != nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
if _, err := fmt.Fprintf(w, "data: %s\n\n", chunkBytes); err != nil {
|
||||
return
|
||||
return false
|
||||
}
|
||||
flusher.Flush()
|
||||
return true
|
||||
}
|
||||
|
||||
seenChoiceIndexes := make(map[int]struct{})
|
||||
var templateChunk OpenAIChunk
|
||||
sawFinishReason := false
|
||||
sawToolCalls := false
|
||||
|
||||
for {
|
||||
var chunk OpenAIChunk
|
||||
var ok bool
|
||||
select {
|
||||
case <-r.Context().Done():
|
||||
log.Printf("writeChatCompletionsStreaming: request context canceled, stopping stream")
|
||||
return
|
||||
case chunk, ok = <-chunks:
|
||||
if !ok {
|
||||
if !sawFinishReason && len(seenChoiceIndexes) > 0 {
|
||||
finishReason := "stop"
|
||||
if sawToolCalls {
|
||||
finishReason = "tool_calls"
|
||||
}
|
||||
choiceIndexes := make([]int, 0, len(seenChoiceIndexes))
|
||||
for index := range seenChoiceIndexes {
|
||||
choiceIndexes = append(choiceIndexes, index)
|
||||
}
|
||||
sort.Ints(choiceIndexes)
|
||||
finishChoices := make([]OpenAIChunkChoice, 0, len(choiceIndexes))
|
||||
for _, index := range choiceIndexes {
|
||||
finishChoices = append(finishChoices, OpenAIChunkChoice{Index: index, FinishReason: finishReason})
|
||||
}
|
||||
if !writeChunk(OpenAIChunk{
|
||||
ID: templateChunk.ID,
|
||||
Object: templateChunk.Object,
|
||||
Created: templateChunk.Created,
|
||||
Model: templateChunk.Model,
|
||||
Choices: finishChoices,
|
||||
}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
_, _ = fmt.Fprintf(w, "data: [DONE]\n\n")
|
||||
flusher.Flush()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
templateChunk = chunk
|
||||
for _, choice := range chunk.Choices {
|
||||
seenChoiceIndexes[choice.Index] = struct{}{}
|
||||
sawToolCalls = sawToolCalls || len(choice.ToolCalls) > 0
|
||||
sawFinishReason = sawFinishReason || choice.FinishReason != ""
|
||||
}
|
||||
if !writeChunk(chunk) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -60,7 +60,7 @@ var preferredTitleModels = []struct {
|
||||
{fantasyopenai.Name, "gpt-4o-mini"},
|
||||
{fantasygoogle.Name, "gemini-2.5-flash"},
|
||||
{fantasyazure.Name, "gpt-4o-mini"},
|
||||
{fantasybedrock.Name, "anthropic.claude-haiku-4-5-20251001-v1:0"},
|
||||
{fantasybedrock.Name, "global.anthropic.claude-haiku-4-5-20251001-v1:0"},
|
||||
{fantasyopenrouter.Name, "anthropic/claude-3.5-haiku"},
|
||||
{fantasyvercel.Name, "anthropic/claude-haiku-4.5"},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user