fix(images): 识别 response.incomplete + 记录软失败上游响应

修复 gpt-image-2 大图/编辑请求 502 报错（社区 issue #2232/#3135/#2516，现象：upstream did not return image output）。根因是上游生成超时/截断时返回 response.incomplete，旧逻辑只认 error/response.failed，导致： 1) 软失败报成模糊 502 且不触发 failover 换账号重试 2) 上游真实响应未记录，ops_error_logs 里上游信息全空，无法排查 - openAIImagesUpstreamErrorFromSSEPayload 增加 response.incomplete 识别：生成超时/截断(max_output_tokens 等) → 可重试 502 触发 failover； content_filter/moderation → 400 不重试 - 软失败兜底(无图无标准错误)记录上游诊断摘要到 ops（last_event/status/ incomplete_reason/body 片段），非流式+流式两条路径一致 - summarizeOpenAIImagesNoOutputBody 提取诊断信息，body 截断上限 1KB 基于官方 v0.1.137 净分支。测试: 5 个新单测 + 图片回归通过 (-tags unit)
2026-09-24 16:05:44 +08:00 · 2026-06-21 08:34:12 +08:00
parent eba9bea959
commit b0d5592ae2
2 changed files with 218 additions and 1 deletions
@@ -0,0 +1,123 @@
+package service
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+)
+
+// response.incomplete（生成超时/截断）应被识别为可重试的 502 上游错误，触发 failover。
+func TestExtractImagesUpstreamError_IncompleteIsRetryable(t *testing.T) {
+	body := "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_1\"}}\n\n" +
+		"data: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"resp_1\",\"status\":\"incomplete\",\"incomplete_details\":{\"reason\":\"max_output_tokens\"}}}\n\n"
+	got := extractOpenAIImagesUpstreamError([]byte(body))
+	if got == nil {
+		t.Fatal("incomplete event should produce an upstream error, got nil")
+	}
+	if got.StatusCode != http.StatusBadGateway {
+		t.Fatalf("incomplete(max_output_tokens) should be 502 retryable, got %d", got.StatusCode)
+	}
+	if !IsOpenAIImagesRetryableUpstreamError(got) {
+		t.Fatal("incomplete(max_output_tokens) should be retryable for failover")
+	}
+	if got.Code != "response_incomplete" {
+		t.Fatalf("unexpected code %q", got.Code)
+	}
+	if !strings.Contains(got.Message, "max_output_tokens") {
+		t.Fatalf("message should carry reason, got %q", got.Message)
+	}
+}
+
+// incomplete 因 content_filter → 400，重试无意义，不应触发 failover。
+func TestExtractImagesUpstreamError_IncompleteContentFilterNotRetryable(t *testing.T) {
+	body := "data: {\"type\":\"response.incomplete\",\"response\":{\"id\":\"r\",\"status\":\"incomplete\",\"incomplete_details\":{\"reason\":\"content_filter\"}}}\n\n"
+	got := extractOpenAIImagesUpstreamError([]byte(body))
+	if got == nil {
+		t.Fatal("content_filter incomplete should produce error")
+	}
+	if got.StatusCode != http.StatusBadRequest {
+		t.Fatalf("content_filter should be 400 (non-retryable), got %d", got.StatusCode)
+	}
+	if IsOpenAIImagesRetryableUpstreamError(got) {
+		t.Fatal("content_filter must NOT be retryable")
+	}
+}
+
+// 旧行为不变：error / response.failed 仍按原逻辑识别。
+func TestExtractImagesUpstreamError_ErrorAndFailedUnchanged(t *testing.T) {
+	errBody := "data: {\"type\":\"error\",\"error\":{\"type\":\"image_generation_user_error\",\"code\":\"moderation_blocked\",\"message\":\"rejected\"}}\n\n"
+	if got := extractOpenAIImagesUpstreamError([]byte(errBody)); got == nil || got.StatusCode != http.StatusBadRequest {
+		t.Fatalf("moderation_blocked should still be 400, got %+v", got)
+	}
+}
+
+// 上游既无图、又无任何可识别事件时，摘要函数应提取诊断信息。
+func TestSummarizeNoOutputBody_ExtractsDiagnostics(t *testing.T) {
+	body := "data: {\"type\":\"response.created\",\"response\":{\"id\":\"r\"}}\n\n" +
+		"data: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"r\",\"status\":\"in_progress\"}}\n\n"
+	summary := summarizeOpenAIImagesNoOutputBody([]byte(body))
+	if !strings.HasPrefix(summary, "no_image_output") {
+		t.Fatalf("summary should start with marker, got %q", summary)
+	}
+	if !strings.Contains(summary, "last_event=response.in_progress") {
+		t.Fatalf("summary should capture last event type, got %q", summary)
+	}
+	if !strings.Contains(summary, "status=in_progress") {
+		t.Fatalf("summary should capture response status, got %q", summary)
+	}
+}
+
+// 摘要应能抓到 incomplete_reason 并对超长 body 截断。
+func TestSummarizeNoOutputBody_IncompleteReasonAndTruncation(t *testing.T) {
+	long := strings.Repeat("x", 2000)
+	body := "data: {\"type\":\"response.incomplete\",\"response\":{\"status\":\"incomplete\",\"incomplete_details\":{\"reason\":\"max_output_tokens\"},\"junk\":\"" + long + "\"}}\n\n"
+	summary := summarizeOpenAIImagesNoOutputBody([]byte(body))
+	if !strings.Contains(summary, "incomplete_reason=max_output_tokens") {
+		t.Fatalf("should capture incomplete reason, got %q", summary[:120])
+	}
+	if !strings.Contains(summary, "truncated") {
+		t.Fatalf("oversized body should be truncated, len=%d", len(summary))
+	}
+}
+
+// 软失败（上游 completed 但无图，如偶发路由到 mini 模型）应返回可重试的
+// UpstreamFailoverError 且优先同账号重试，而非一次性失败。
+func TestImagesOAuthNonStreaming_CompletedNoImageTriggersSameAccountRetry(t *testing.T) {
+	// 上游 SSE：response.completed 但 output 为空（实测的真实失败形态）。
+	upstreamSSE := "event: response.created\n" +
+		"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_x\",\"status\":\"in_progress\",\"model\":\"gpt-5.4-mini-2026-03-17\",\"output\":[]}}\n\n" +
+		"event: response.completed\n" +
+		"data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_x\",\"status\":\"completed\",\"model\":\"gpt-5.4-mini-2026-03-17\",\"output\":[],\"tool_usage\":{\"image_gen\":{\"output_tokens\":0}}}}\n\n"
+
+	rec := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(rec)
+	c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/generations", nil)
+
+	resp := &http.Response{
+		StatusCode: http.StatusOK,
+		Header:     http.Header{},
+		Body:       io.NopCloser(strings.NewReader(upstreamSSE)),
+	}
+
+	svc := &OpenAIGatewayService{}
+	_, _, _, err := svc.handleOpenAIImagesOAuthNonStreamingResponse(resp, c, "b64_json", "gpt-image-2")
+
+	if err == nil {
+		t.Fatal("completed-but-no-image should return an error")
+	}
+	var failoverErr *UpstreamFailoverError
+	if !errors.As(err, &failoverErr) {
+		t.Fatalf("expected *UpstreamFailoverError to trigger retry, got %T: %v", err, err)
+	}
+	if failoverErr.StatusCode != http.StatusBadGateway {
+		t.Fatalf("expected 502, got %d", failoverErr.StatusCode)
+	}
+	if !failoverErr.RetryableOnSameAccount {
+		t.Fatal("soft-failure should prefer same-account retry (probabilistic upstream failure)")
+	}
+}
@@ -597,11 +597,89 @@ func openAIImagesUpstreamErrorFromSSEPayload(payload []byte) *OpenAIImagesUpstre
 	case "response.failed":
 		response := gjson.GetBytes(payload, "response")
 		return openAIImagesUpstreamErrorFromGJSON(response.Get("error"), response.Get("id").String())
+	case "response.incomplete":
+		// 上游在生成预算内未产出图片（超时/被截断），返回 response.incomplete 而非 error。
+		// 旧逻辑识别不到，统一报成模糊的 "upstream did not return image output" + 502，
+		// 且不触发 failover。这里把它显式建模为可重试的上游错误，使其能换账号重试。
+		return openAIImagesIncompleteUpstreamError(gjson.GetBytes(payload, "response"))
 	default:
 		return nil
 	}
 }

+// summarizeOpenAIImagesNoOutputBody 从上游 SSE 响应体提取诊断摘要，用于软失败时
+// 记录到 ops 日志（上游无图、无标准错误的场景）。提取最终事件类型、response.status、
+// incomplete_details.reason，并附 body 截断片段，便于事后定位上游到底返回了什么。
+func summarizeOpenAIImagesNoOutputBody(body []byte) string {
+	var lastType, status, incompleteReason string
+	forEachOpenAISSEDataPayload(string(body), func(payload []byte) {
+		if !gjson.ValidBytes(payload) {
+			return
+		}
+		if t := strings.TrimSpace(gjson.GetBytes(payload, "type").String()); t != "" {
+			lastType = t
+		}
+		if resp := gjson.GetBytes(payload, "response"); resp.Exists() {
+			if s := strings.TrimSpace(resp.Get("status").String()); s != "" {
+				status = s
+			}
+			if r := strings.TrimSpace(resp.Get("incomplete_details.reason").String()); r != "" {
+				incompleteReason = r
+			}
+		}
+	})
+	var b strings.Builder
+	b.WriteString("no_image_output")
+	if lastType != "" {
+		fmt.Fprintf(&b, " last_event=%s", lastType)
+	}
+	if status != "" {
+		fmt.Fprintf(&b, " status=%s", status)
+	}
+	if incompleteReason != "" {
+		fmt.Fprintf(&b, " incomplete_reason=%s", incompleteReason)
+	}
+	// 附 body 截断片段（脱敏后），上限 1KB，避免日志膨胀。
+	snippet := strings.TrimSpace(string(body))
+	const maxSnippet = 1024
+	if len(snippet) > maxSnippet {
+		snippet = snippet[:maxSnippet] + "...(truncated)"
+	}
+	if snippet != "" {
+		fmt.Fprintf(&b, " body=%s", snippet)
+	}
+	return b.String()
+}
+
+// openAIImagesIncompleteUpstreamError 从 response.incomplete 事件构建可重试的上游错误。
+// incomplete_details.reason 常见取值：max_output_tokens / content_filter 等。
+// content_filter 视为客户端错误（400，重试无意义）；其余（生成超时/截断）视为
+// 可重试的 502，触发 failover 换账号重试。
+func openAIImagesIncompleteUpstreamError(response gjson.Result) *OpenAIImagesUpstreamError {
+	if !response.Exists() {
+		return nil
+	}
+	reason := strings.TrimSpace(response.Get("incomplete_details.reason").String())
+	statusCode := http.StatusBadGateway // 默认可重试（生成未完成）
+	errType := "incomplete_error"
+	if strings.Contains(strings.ToLower(reason), "content_filter") ||
+		strings.Contains(strings.ToLower(reason), "moderation") {
+		statusCode = http.StatusBadRequest // 内容过滤，重试无意义
+		errType = "image_generation_user_error"
+	}
+	message := "Upstream did not complete image generation"
+	if reason != "" {
+		message = fmt.Sprintf("Upstream image generation incomplete: %s", reason)
+	}
+	return &OpenAIImagesUpstreamError{
+		StatusCode:        statusCode,
+		ErrorType:         errType,
+		Code:              "response_incomplete",
+		Message:           sanitizeUpstreamErrorMessage(message),
+		UpstreamRequestID: strings.TrimSpace(response.Get("id").String()),
+	}
+}
+
 func openAIImagesUpstreamErrorFromGJSON(errorObj gjson.Result, upstreamRequestID string) *OpenAIImagesUpstreamError {
 	if !errorObj.Exists() {
 		return nil
@@ -959,7 +1037,20 @@ func (s *OpenAIGatewayService) handleOpenAIImagesOAuthNonStreamingResponse(
 			}
 			return OpenAIUsage{}, 0, nil, upstreamErr
 		}
-		return OpenAIUsage{}, 0, nil, fmt.Errorf("upstream did not return image output")
+		// 软失败兜底：上游既无图、又无任何可识别的 error/failed/incomplete 事件
+		// （实测：上游偶发把请求路由到 gpt-5.x-mini，返回 response.completed 但 output 为空、
+		// image_gen 工具未执行）。这是上游的概率性失败——同账号有时成功有时失败。
+		// 处理：① 记录上游诊断摘要到 ops（last_event/status/model/body 片段）便于排查；
+		// ② 返回 UpstreamFailoverError 触发重试。因实测为「同账号概率性失败」，优先
+		//    RetryableOnSameAccount 同账号快速重试（默认 3 次，大概率某次正常出图），
+		//    用尽后由 handler 自然换账号 failover（switchCount 上限保护），既提高成功率
+		//    又不无谓消耗其他账号配额。
+		setOpsUpstreamError(c, http.StatusBadGateway, "upstream did not return image output", summarizeOpenAIImagesNoOutputBody(body))
+		return OpenAIUsage{}, 0, nil, &UpstreamFailoverError{
+			StatusCode:             http.StatusBadGateway,
+			ResponseBody:           body,
+			RetryableOnSameAccount: true,
+		}
 	}
 	if strings.TrimSpace(firstMeta.Model) == "" {
 		firstMeta.Model = strings.TrimSpace(fallbackModel)
@@ -1096,6 +1187,9 @@ func (s *OpenAIGatewayService) handleOpenAIImagesOAuthStreamingResponse(
 			}
 			if len(finalResults) == 0 {
 				outputErr := fmt.Errorf("upstream did not return image output")
+				// 软失败：response.completed 事件里没有图片。记录上游诊断摘要到 ops，
+				// 与非流式路径保持一致，避免上游响应信息丢失。
+				setOpsUpstreamError(c, http.StatusBadGateway, "upstream did not return image output", summarizeOpenAIImagesNoOutputBody(dataBytes))
 				s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(outputErr.Error()))
 				processDataErr = outputErr
 				processDataDone = true