From 98027cdded50997c416ce7aa389e35993c410968 Mon Sep 17 00:00:00 2001 From: hongheshan-svg <237549909+hongheshan-svg@users.noreply.github.com> Date: Mon, 13 Jul 2026 23:22:05 +0800 Subject: [PATCH] fix(openai): enable HTTP/2 keep-alive PING to evict dead Codex connections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex/OpenAI upstream switched from a WebSocket pool to HTTP/2. The outbound H2 transport set neither ReadIdleTimeout nor ResponseHeaderTimeout (the OpenAI profile forces ResponseHeaderTimeout=0), so a pooled H2 connection silently killed by a proxy/NAT becomes a "dead connection": both ends believe it is alive and a request assigned to it hangs until the OS TCP retransmit timeout (minutes) before the first byte — observed as an 8m37s TTFT with an eventual 200. Occasional (only when a request lands on a dead pooled conn) and across all groups (shared OpenAI transport); worse on larger idle pools. Explicitly configure http2 on the openai_h2 transport and enable active PING health checks (ReadIdleTimeout=15s, PingTimeout=15s) so dead connections are detected and evicted at the source, instead of relying on ResponseHeaderTimeout as an after-the-fact backstop. Scoped to the openai_h2 path only; Claude/Gemini (default) and h1 modes are untouched. Co-Authored-By: Claude Opus 4.8 --- backend/internal/repository/http_upstream.go | 29 ++++++++ .../http_upstream_http2_keepalive_test.go | 67 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 backend/internal/repository/http_upstream_http2_keepalive_test.go diff --git a/backend/internal/repository/http_upstream.go b/backend/internal/repository/http_upstream.go index bb079b0789..3dfedf6493 100644 --- a/backend/internal/repository/http_upstream.go +++ b/backend/internal/repository/http_upstream.go @@ -21,6 +21,7 @@ import ( "github.com/andybalholm/brotli" "github.com/klauspost/compress/zstd" + "golang.org/x/net/http2" "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/pkg/proxyurl" @@ -59,6 +60,13 @@ const ( defaultOpenAIHTTP2FallbackErrorThreshold = 2 defaultOpenAIHTTP2FallbackWindow = 60 * time.Second defaultOpenAIHTTP2FallbackTTL = 10 * time.Minute + // OpenAI HTTP/2 连接健康探测：Codex 上游改走 HTTP/2 后，池化连接被代理/NAT + // 静默掐断会成为“死连接”（两端都以为存活），请求落上去会挂到 TCP 重传超时 + // （分钟级）。Go 的 http2.Transport 默认 ReadIdleTimeout=0（不发健康 PING）， + // 无法检测。启用主动 PING 探测：连接空闲 ReadIdleTimeout 后发 PING，PingTimeout + // 内无响应即判定死连接并关闭，从源头避免请求挂在死连接上。 + openAIHTTP2ReadIdleTimeout = 15 * time.Second + openAIHTTP2PingTimeout = 15 * time.Second // The Grok CLI proxy rejects requests that do not identify a supported // client version. Keep a known-good stable version in the binary while @@ -1101,6 +1109,11 @@ func buildUpstreamTransport(settings poolSettings, proxyURL *url.URL, protocolMo switch protocolMode { case upstreamProtocolModeOpenAIH2: transport.ForceAttemptHTTP2 = true + // 显式配置 http2 并启用 PING 健康探测，剔除代理/NAT 静默掐断的死连接， + // 避免请求挂在死连接上直到 TCP 重传超时（分钟级）。 + if _, err := enableOpenAIHTTP2KeepAlive(transport); err != nil { + return nil, err + } case upstreamProtocolModeOpenAIH1: transport.ForceAttemptHTTP2 = false transport.TLSNextProto = make(map[string]func(string, *tls.Conn) http.RoundTripper) @@ -1115,6 +1128,22 @@ func buildUpstreamTransport(settings poolSettings, proxyURL *url.URL, protocolMo return transport, nil } +// enableOpenAIHTTP2KeepAlive 在 http.Transport 上显式配置 HTTP/2 并启用连接健康探测。 +// Go 默认惰性配置 http2 且 ReadIdleTimeout=0（不发健康 PING），无法检测被代理/NAT +// 静默掐断的死连接。此处主动设置 ReadIdleTimeout/PingTimeout，让死连接被提前 PING +// 出并关闭，请求得以重建连接而非挂到 TCP 重传超时。返回底层 *http2.Transport 便于测试。 +func enableOpenAIHTTP2KeepAlive(transport *http.Transport) (*http2.Transport, error) { + h2, err := http2.ConfigureTransports(transport) + if err != nil { + return nil, err + } + if h2 != nil { + h2.ReadIdleTimeout = openAIHTTP2ReadIdleTimeout + h2.PingTimeout = openAIHTTP2PingTimeout + } + return h2, nil +} + // buildUpstreamTransportWithTLSFingerprint 构建带 TLS 指纹伪装的 Transport // 使用 utls 库模拟 Claude CLI 的 TLS 指纹 // diff --git a/backend/internal/repository/http_upstream_http2_keepalive_test.go b/backend/internal/repository/http_upstream_http2_keepalive_test.go new file mode 100644 index 0000000000..ead61da3b2 --- /dev/null +++ b/backend/internal/repository/http_upstream_http2_keepalive_test.go @@ -0,0 +1,67 @@ +package repository + +import ( + "net/http" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func http2KeepAliveTestPoolSettings() poolSettings { + return poolSettings{ + maxIdleConns: 10, + maxIdleConnsPerHost: 5, + maxConnsPerHost: 10, + idleConnTimeout: 90 * time.Second, + responseHeaderTimeout: time.Minute, + } +} + +// Codex/OpenAI 上游改走 HTTP/2 后，池化连接被代理/NAT 静默掐断会成为“死连接”： +// 两端都以为连接存活，请求落上去会挂到 TCP 重传超时（分钟级）才失败。Go 的 +// http2.Transport 默认 ReadIdleTimeout=0（不发健康 PING），无法检测这种死连接。 +// 必须显式启用主动 PING 探测，让死连接被提前剔除，而不是只靠 ResponseHeaderTimeout +// 事后兜底。 +func TestEnableOpenAIHTTP2KeepAlive_EnablesPingHealthCheck(t *testing.T) { + tr := &http.Transport{} + + h2, err := enableOpenAIHTTP2KeepAlive(tr) + require.NoError(t, err) + require.NotNil(t, h2, "必须返回已配置的 *http2.Transport") + + require.Positive(t, h2.ReadIdleTimeout, "必须启用空闲 PING 探测以剔除死连接") + require.Equal(t, openAIHTTP2ReadIdleTimeout, h2.ReadIdleTimeout) + require.Equal(t, openAIHTTP2PingTimeout, h2.PingTimeout, "PING 无响应必须有超时判定") + require.NotNil(t, tr.TLSNextProto["h2"], "http2 必须已挂到底层 http.Transport 上") +} + +// openai_h2 模式构建的 Transport 必须带上 H2 PING 健康探测，从源头剔除死连接。 +func TestBuildUpstreamTransport_OpenAIH2_EnablesPingHealthCheck(t *testing.T) { + tr, err := buildUpstreamTransport(http2KeepAliveTestPoolSettings(), nil, upstreamProtocolModeOpenAIH2) + require.NoError(t, err) + require.True(t, tr.ForceAttemptHTTP2, "openai_h2 必须启用 HTTP/2") + require.NotNil(t, tr.TLSNextProto["h2"], "openai_h2 必须显式配置 http2 以启用 ReadIdleTimeout") +} + +// 非 H2 模式（default/h1）不应因本次改动被误配置：default 走 Go 自动 H2（惰性配置， +// 构建时 TLSNextProto 仍为空），h1 模式显式禁用 H2。避免波及 Claude/Gemini 热路径。 +func TestBuildUpstreamTransport_NonOpenAIH2_NotEagerlyConfigured(t *testing.T) { + tr, err := buildUpstreamTransport(http2KeepAliveTestPoolSettings(), nil, upstreamProtocolModeDefault) + require.NoError(t, err) + require.Nil(t, tr.TLSNextProto["h2"], "default 模式不应在构建期主动配置 http2 keepalive") +} + +// 死连接在经 HTTP 代理（CONNECT 隧道）时最高发，这是带 proxy 账号的真实生产路径： +// 显式 http2 配置须与 Transport.Proxy 同时正确生效，不能相互干扰。 +func TestBuildUpstreamTransport_OpenAIH2_WithHTTPProxy_EnablesKeepAlive(t *testing.T) { + proxyURL, err := url.Parse("http://127.0.0.1:8080") + require.NoError(t, err) + + tr, err := buildUpstreamTransport(http2KeepAliveTestPoolSettings(), proxyURL, upstreamProtocolModeOpenAIH2) + require.NoError(t, err) + require.True(t, tr.ForceAttemptHTTP2) + require.NotNil(t, tr.TLSNextProto["h2"], "经代理的 openai_h2 也必须启用 http2 keepalive") + require.NotNil(t, tr.Proxy, "HTTP 代理仍须通过 Transport.Proxy 生效") +}