diff --git a/backend/internal/handler/openai_gateway_credential_failover_test.go b/backend/internal/handler/openai_gateway_credential_failover_test.go index d94eae7fe9..4fd5df01d0 100644 --- a/backend/internal/handler/openai_gateway_credential_failover_test.go +++ b/backend/internal/handler/openai_gateway_credential_failover_test.go @@ -128,6 +128,44 @@ func TestOpenAICapacityFailoverExhaustionPreservesMessageAsServerError(t *testin }) } +func TestOpenAIManagedSingleAccountModelNotFoundExhaustionPreservesStructured400(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + body := []byte(`{"error":{"type":"invalid_request_error","code":"model_not_found","param":"model","message":"The requested model is unavailable on this account"}}`) + + (&OpenAIGatewayHandler{}).handleFailoverExhausted(c, &service.UpstreamFailoverError{ + StatusCode: http.StatusBadRequest, + ResponseBody: body, + }, false) + + require.Equal(t, http.StatusBadRequest, recorder.Code) + require.Equal(t, "invalid_request_error", gjson.Get(recorder.Body.String(), "error.type").String()) + require.Equal(t, "model_not_found", gjson.Get(recorder.Body.String(), "error.code").String()) + require.Equal(t, "model", gjson.Get(recorder.Body.String(), "error.param").String()) + require.Equal(t, "The requested model is unavailable on this account", gjson.Get(recorder.Body.String(), "error.message").String()) +} + +func TestOpenAIManagedModelNotFoundExhaustionSanitizesMessage(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + body := []byte(`{"error":{"code":"model_not_found","message":"Model not found; inspect https://upstream.example/debug?access_token=super-secret-value&model=x"}}`) + + (&OpenAIGatewayHandler{}).handleFailoverExhausted(c, &service.UpstreamFailoverError{ + StatusCode: http.StatusBadRequest, + ResponseBody: body, + }, false) + + require.Equal(t, http.StatusBadRequest, recorder.Code) + require.NotContains(t, recorder.Body.String(), "super-secret-value") + require.Contains(t, gjson.Get(recorder.Body.String(), "error.message").String(), "access_token=***") + recorded, ok := c.Get(service.OpsUpstreamErrorMessageKey) + require.True(t, ok) + require.NotContains(t, recorded, "super-secret-value") + require.Contains(t, recorded, "access_token=***") +} + func TestResponsesFailoverExhaustedAfterForwardedTerminalMarksOpsWithoutDuplicateFrame(t *testing.T) { gin.SetMode(gin.TestMode) recorder := httptest.NewRecorder() diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 9f03129edc..c7884c9dd2 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -3310,6 +3310,12 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE } statusCode := failoverErr.StatusCode responseBody := failoverErr.ResponseBody + if statusCode == http.StatusBadRequest && service.IsOpenAICompatibleModelNotFound400(responseBody) && !streamStarted { + upstreamMsg := service.SanitizeUpstreamErrorMessage(service.ExtractUpstreamErrorMessage(responseBody)) + service.SetOpsUpstreamError(c, statusCode, upstreamMsg, "") + service.WriteOpenAIUpstreamClientError(c, statusCode, responseBody, upstreamMsg) + return + } if service.IsOpenAISilentRefusalErrorBody(responseBody) { service.SetOpsUpstreamError(c, statusCode, service.OpenAISilentRefusalClientMessage(), "") h.handleStreamingAwareError(c, http.StatusBadGateway, "upstream_error", service.OpenAISilentRefusalClientMessage(), streamStarted) diff --git a/backend/internal/service/gateway_upstream_response.go b/backend/internal/service/gateway_upstream_response.go index e99c545694..5b1307b40e 100644 --- a/backend/internal/service/gateway_upstream_response.go +++ b/backend/internal/service/gateway_upstream_response.go @@ -285,6 +285,12 @@ func ExtractUpstreamErrorMessage(body []byte) string { return extractUpstreamErrorMessage(body) } +// SanitizeUpstreamErrorMessage redacts sensitive query parameter values before +// an upstream error message is recorded or returned to a client. +func SanitizeUpstreamErrorMessage(message string) string { + return sanitizeUpstreamErrorMessage(message) +} + func extractUpstreamErrorMessage(body []byte) string { // Claude 风格：{"type":"error","error":{"type":"...","message":"..."}} if m := gjson.GetBytes(body, "error.message").String(); strings.TrimSpace(m) != "" { diff --git a/backend/internal/service/model_not_found_error_test.go b/backend/internal/service/model_not_found_error_test.go index 2f8c83466f..d76d71d0f1 100644 --- a/backend/internal/service/model_not_found_error_test.go +++ b/backend/internal/service/model_not_found_error_test.go @@ -112,3 +112,29 @@ func TestIsOpenAICodexPlanGatedModelError(t *testing.T) { }) } } + +func TestIsOpenAICompatibleModelNotFound400(t *testing.T) { + tests := []struct { + name string + body string + want bool + }{ + {name: "structured code", body: `{"error":{"code":"model_not_found","message":"No such deployment"}}`, want: true}, + {name: "unknown provider", body: `{"error":{"message":"Unknown provider for model claude-x"}}`, want: true}, + {name: "model not found", body: `{"error":{"message":"Model not found: claude-x"}}`, want: true}, + {name: "model unsupported", body: `{"error":{"message":"The requested model is not supported"}}`, want: true}, + {name: "plain text compatible gateway", body: `unknown provider for model claude-x`, want: true}, + {name: "invalid parameter remains terminal", body: `{"error":{"code":"invalid_request_error","message":"Invalid value for temperature"}}`, want: false}, + {name: "structured non-matching code overrides model not found message", body: `{"error":{"code":"invalid_request_error","message":"Model not found: claude-x"}}`, want: false}, + {name: "structured non-matching error code overrides unsupported message", body: `{"error":{"code":"upstream_error","message":"The requested model is not supported"}}`, want: false}, + {name: "echoed phrase is ignored", body: `{"error":{"message":"Invalid request"},"echo":"model not found"}`, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isOpenAICompatibleModelNotFound400([]byte(tt.body)); got != tt.want { + t.Fatalf("isOpenAICompatibleModelNotFound400() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/internal/service/openai_access_state_failover_test.go b/backend/internal/service/openai_access_state_failover_test.go index a46f9ca571..48c8fb617e 100644 --- a/backend/internal/service/openai_access_state_failover_test.go +++ b/backend/internal/service/openai_access_state_failover_test.go @@ -77,7 +77,7 @@ func TestOpenAIUpstreamAccessStateClassification(t *testing.T) { if !tt.want { return } - require.True(t, (&OpenAIGatewayService{}).shouldFailoverOpenAIUpstreamResponse(http.StatusForbidden, "", body)) + require.True(t, (&OpenAIGatewayService{}).shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusForbidden, "", body)) require.True(t, shouldFailoverOpenAIPassthroughResponse(&Account{Type: AccountTypeOAuth}, http.StatusForbidden, body)) err := newOpenAIUpstreamFailoverError(http.StatusForbidden, nil, body, "", true) @@ -105,7 +105,7 @@ func TestOpenAIHTTPAccessStateDoesNotTrustBadRequestMessage(t *testing.T) { require.False(t, isOpenAIUpstreamAccessStateError("", body), "free-form stream messages are not durable account evidence") require.False(t, isOpenAIHTTPUpstreamAccessStateError(http.StatusBadRequest, "", body)) - require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadRequest, "", body)) + require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadRequest, "", body)) require.False(t, shouldFailoverOpenAIPassthroughResponse(&Account{Type: AccountTypeOAuth}, http.StatusBadRequest, body)) err := newOpenAIUpstreamFailoverError(http.StatusBadRequest, nil, body, "", false) @@ -148,7 +148,7 @@ func TestOpenAIHTTPAccessStateTrustsStructuredCode(t *testing.T) { body := []byte(`{"error":{"code":"organization_deactivated","message":"request rejected"}}`) require.True(t, isOpenAIHTTPUpstreamAccessStateError(http.StatusBadRequest, "", body)) - require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadRequest, "", body)) + require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadRequest, "", body)) require.True(t, svc.handleOpenAIAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body)) require.Equal(t, 1, repo.setErrorCalls) require.True(t, svc.isOpenAIAccountRuntimeBlocked(account)) @@ -189,7 +189,7 @@ func TestOpenAIHTTPAuthMessagesUseExistingStatusPolicies(t *testing.T) { func TestOpenAICyberPolicyWrapped5xxNeverFailsOver(t *testing.T) { body := []byte(`{"error":{"code":"cyber_policy","message":"blocked"}}`) svc := &OpenAIGatewayService{} - require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadGateway, "wrapped upstream failure", body)) + require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "wrapped upstream failure", body)) require.False(t, shouldFailoverOpenAIPassthroughResponse(&Account{Type: AccountTypeOAuth}, http.StatusBadGateway, body)) } diff --git a/backend/internal/service/openai_alpha_search.go b/backend/internal/service/openai_alpha_search.go index caff28208b..c80673405f 100644 --- a/backend/internal/service/openai_alpha_search.go +++ b/backend/internal/service/openai_alpha_search.go @@ -91,7 +91,7 @@ func (s *OpenAIGatewayService) ForwardAlphaSearch(ctx context.Context, c *gin.Co if resp.StatusCode >= http.StatusBadRequest { upstreamMessage := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody))) - if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMessage, respBody) || + if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMessage, respBody) || isOpenAIAlphaSearchEndpointUnsupported(account, resp.StatusCode) { resp.Body = io.NopCloser(bytes.NewReader(respBody)) // alpha/search 是独立的工具端点，单次 401 不能证明账号的模型调用 @@ -175,7 +175,7 @@ func (s *OpenAIGatewayService) forwardAlphaSearchViaResponsesWebSearch( if resp.StatusCode >= http.StatusBadRequest { upstreamMessage := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody))) - if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMessage, respBody) { + if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMessage, respBody) { resp.Body = io.NopCloser(bytes.NewReader(respBody)) // 仍按 alpha/search 工具请求处理：PAT 的工具链路失败不能直接永久置错。 shouldDisable := false diff --git a/backend/internal/service/openai_embeddings.go b/backend/internal/service/openai_embeddings.go index 0d1fcbd1e1..c4b8f79bda 100644 --- a/backend/internal/service/openai_embeddings.go +++ b/backend/internal/service/openai_embeddings.go @@ -118,7 +118,7 @@ func (s *OpenAIGatewayService) ForwardEmbeddings( upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody)) upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg) - if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) { + if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) { upstreamDetail := "" if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes diff --git a/backend/internal/service/openai_gateway_cc_pipeline.go b/backend/internal/service/openai_gateway_cc_pipeline.go index 6458509389..68640507df 100644 --- a/backend/internal/service/openai_gateway_cc_pipeline.go +++ b/backend/internal/service/openai_gateway_cc_pipeline.go @@ -88,7 +88,7 @@ func (s *OpenAIGatewayService) failoverOpenAIUpstreamHTTPError( upstreamMsg string, upstreamModel string, ) *UpstreamFailoverError { - shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) + shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) tempUnscheduled := false if c != nil && account != nil && account.Platform != PlatformGrok && !shouldFailover && !IsResponseCommitted(c) && s.rateLimitService != nil { tempUnscheduled = s.rateLimitService.CheckErrorPolicy(ctx, account, resp.StatusCode, respBody, upstreamModel) == ErrorPolicyTempUnscheduled diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index e92f8b680e..d389c6c7f8 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -1136,7 +1136,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco ) continue } - if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) { + if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) { upstreamDetail := "" if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody { maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes @@ -1211,7 +1211,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco _ = resp.Body.Close() } compactResp, compactBody := openAICompactFallbackErrorResponse(resp, signal) - if s.shouldFailoverOpenAIUpstreamResponse(compactResp.StatusCode, signal.message, compactBody) { + if s.shouldFailoverOpenAIUpstreamResponse(account, compactResp.StatusCode, signal.message, compactBody) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ ProxyID: opsUpstreamProxyID(account), ProxyName: opsUpstreamProxyName(account), diff --git a/backend/internal/service/openai_gateway_service_codex_cli_only_test.go b/backend/internal/service/openai_gateway_service_codex_cli_only_test.go index 850c5d0ce8..079153b2a6 100644 --- a/backend/internal/service/openai_gateway_service_codex_cli_only_test.go +++ b/backend/internal/service/openai_gateway_service_codex_cli_only_test.go @@ -356,9 +356,9 @@ func TestShouldFailoverOpenAIUpstreamResponseContextWindow502(t *testing.T) { svc := &OpenAIGatewayService{} body := []byte(`{"error":{"message":"Your input exceeds the context window of this model. Please adjust your input and try again.","type":"upstream_error","code":null}}`) - require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadGateway, "", body)) - require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadGateway, "temporary upstream outage", []byte(`{"error":{"message":"temporary upstream outage"}}`))) - require.True(t, svc.shouldFailoverOpenAIUpstreamResponse( + require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "", body)) + require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "temporary upstream outage", []byte(`{"error":{"message":"temporary upstream outage"}}`))) + require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "temporary upstream outage", []byte(`{"error":{"message":"temporary upstream outage"},"echo":"context_length_exceeded"}`), diff --git a/backend/internal/service/openai_gateway_upstream_errors.go b/backend/internal/service/openai_gateway_upstream_errors.go index db2cd5d9c0..b5985a226e 100644 --- a/backend/internal/service/openai_gateway_upstream_errors.go +++ b/backend/internal/service/openai_gateway_upstream_errors.go @@ -250,7 +250,7 @@ func (s *OpenAIGatewayService) shouldFailoverUpstreamError(statusCode int) bool } } -func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(statusCode int, upstreamMsg string, upstreamBody []byte) bool { +func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(account *Account, statusCode int, upstreamMsg string, upstreamBody []byte) bool { // cyber_policy is request-scoped even when an intermediary wraps the // provider response in a retryable 5xx status. Never punish or rotate the // selected credential for it. @@ -266,12 +266,46 @@ func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(statusCode i if isOpenAIRequestBodyTooLargeError(statusCode, upstreamMsg, upstreamBody) { return true } + // A missing model is account/provider availability, not a malformed client + // request. Keep this unconditional exception inside the OpenAI-compatible + // gateway and require an eligible account so Anthropic/Gemini paths retain + // their existing opt-in 400 behavior. + // A bare forwarding service has no account-selection owner to consume a + // failover sentinel. In that mode (used by direct/single-account callers), + // preserve the deterministic upstream 400 instead of returning an unwritten + // retry signal. Managed gateway instances always have an account repository; + // their handler can exclude this account and actually select another one. + if s != nil && s.accountRepo != nil && account != nil && account.IsOpenAICompatible() && statusCode == http.StatusBadRequest && + isOpenAICompatibleModelNotFound400(upstreamBody) { + return true + } if s.shouldFailoverUpstreamError(statusCode) { return true } return isOpenAITransientProcessingError(statusCode, upstreamMsg, upstreamBody) } +func isOpenAICompatibleModelNotFound400(respBody []byte) bool { + code := strings.TrimSpace(extractUpstreamErrorCode(respBody)) + if code != "" { + return strings.EqualFold(code, "model_not_found") + } + + msg := strings.ToLower(strings.TrimSpace(extractUpstreamErrorMessage(respBody))) + if msg == "" && !gjson.ValidBytes(respBody) { + msg = strings.ToLower(strings.TrimSpace(string(respBody))) + } + return strings.Contains(msg, "unknown provider for model") || + strings.Contains(msg, "model not found") || + strings.Contains(msg, "model is not supported") +} + +// IsOpenAICompatibleModelNotFound400 reports whether an OpenAI-compatible 400 +// is an account-specific missing-model response eligible for failover. +func IsOpenAICompatibleModelNotFound400(respBody []byte) bool { + return isOpenAICompatibleModelNotFound400(respBody) +} + // OpenAIRequestBodyTooLargeClientMessage is the fixed downstream message used // after all account-specific request body limit failovers are exhausted. const OpenAIRequestBodyTooLargeClientMessage = "Request payload is too large" diff --git a/backend/internal/service/openai_images.go b/backend/internal/service/openai_images.go index 371b2986c2..cf39931a91 100644 --- a/backend/internal/service/openai_images.go +++ b/backend/internal/service/openai_images.go @@ -650,7 +650,7 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesAPIKey( resp.Body = io.NopCloser(bytes.NewReader(respBody)) upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody)) upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg) - if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) { + if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ ProxyID: opsUpstreamProxyID(account), ProxyName: opsUpstreamProxyName(account), diff --git a/backend/internal/service/openai_images_responses.go b/backend/internal/service/openai_images_responses.go index 1ad5b5aaee..a0729dc97c 100644 --- a/backend/internal/service/openai_images_responses.go +++ b/backend/internal/service/openai_images_responses.go @@ -1856,7 +1856,7 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth( resp.Body = io.NopCloser(bytes.NewReader(respBody)) upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody)) upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg) - if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) { + if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ ProxyID: opsUpstreamProxyID(account), ProxyName: opsUpstreamProxyName(account), diff --git a/backend/internal/service/openai_live.go b/backend/internal/service/openai_live.go index 68d3bb725e..29922910bb 100644 --- a/backend/internal/service/openai_live.go +++ b/backend/internal/service/openai_live.go @@ -199,7 +199,7 @@ func (s *OpenAIGatewayService) CreateLiveCall( selection.ReleaseFunc() if createErr != nil { s.releaseLiveLease(account.ID, identity.UserID, identity.APIKeyID, leaseID) - if !s.shouldFailoverLiveCreateError(createErr) { + if !s.shouldFailoverLiveCreateError(account, createErr) { return nil, createErr } excluded[account.ID] = struct{}{} @@ -245,13 +245,13 @@ func (s *OpenAIGatewayService) CreateLiveCall( return nil, ErrLiveUnavailable } -func (s *OpenAIGatewayService) shouldFailoverLiveCreateError(err error) bool { +func (s *OpenAIGatewayService) shouldFailoverLiveCreateError(account *Account, err error) bool { var upstreamErr *UpstreamFailoverError if !errors.As(err, &upstreamErr) { // 凭证读取和网络传输错误都可能只影响当前账号或代理。 return true } - return s.shouldFailoverOpenAIUpstreamResponse( + return s.shouldFailoverOpenAIUpstreamResponse(account, upstreamErr.StatusCode, "", upstreamErr.ResponseBody, diff --git a/backend/internal/service/openai_live_test.go b/backend/internal/service/openai_live_test.go index 81cff13b8c..259a5c5ccf 100644 --- a/backend/internal/service/openai_live_test.go +++ b/backend/internal/service/openai_live_test.go @@ -210,17 +210,18 @@ func TestLiveSidebandNormalCloseEndsCall(t *testing.T) { func TestLiveCreateFailoverUsesExistingOpenAIPolicy(t *testing.T) { service := &OpenAIGatewayService{} - require.False(t, service.shouldFailoverLiveCreateError(&UpstreamFailoverError{ + account := newOpenAIUpstreamErrorTestAccount() + require.False(t, service.shouldFailoverLiveCreateError(account, &UpstreamFailoverError{ StatusCode: http.StatusBadRequest, ResponseBody: []byte(`{"error":{"message":"invalid session"}}`), })) - require.True(t, service.shouldFailoverLiveCreateError(&UpstreamFailoverError{ + require.True(t, service.shouldFailoverLiveCreateError(account, &UpstreamFailoverError{ StatusCode: http.StatusForbidden, })) - require.True(t, service.shouldFailoverLiveCreateError(&UpstreamFailoverError{ + require.True(t, service.shouldFailoverLiveCreateError(account, &UpstreamFailoverError{ StatusCode: http.StatusBadGateway, })) - require.True(t, service.shouldFailoverLiveCreateError(errors.New("transport failed"))) + require.True(t, service.shouldFailoverLiveCreateError(account, errors.New("transport failed"))) } func TestLiveCallIDFromLocation(t *testing.T) { diff --git a/backend/internal/service/openai_upstream_client_error.go b/backend/internal/service/openai_upstream_client_error.go index 97690e0ab7..431f5a7703 100644 --- a/backend/internal/service/openai_upstream_client_error.go +++ b/backend/internal/service/openai_upstream_client_error.go @@ -55,3 +55,9 @@ func writeOpenAIUpstreamClientError(c *gin.Context, statusCode int, body []byte, c.JSON(statusCode, gin.H{"error": errorPayload}) } + +// WriteOpenAIUpstreamClientError preserves a structured deterministic upstream +// client error when the handler has exhausted all eligible accounts. +func WriteOpenAIUpstreamClientError(c *gin.Context, statusCode int, body []byte, upstreamMsg string) { + writeOpenAIUpstreamClientError(c, statusCode, body, upstreamMsg) +} diff --git a/backend/internal/service/openai_upstream_client_error_test.go b/backend/internal/service/openai_upstream_client_error_test.go index 291c1c491e..cd50e700a6 100644 --- a/backend/internal/service/openai_upstream_client_error_test.go +++ b/backend/internal/service/openai_upstream_client_error_test.go @@ -45,6 +45,67 @@ func newOpenAIUpstreamErrorTestAccount() *Account { return &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Name: "acct"} } +type modelNotFoundManagedAccountRepo struct{ AccountRepository } + +func TestOpenAICompatibleModelNotFound400FailoverScope(t *testing.T) { + svc := &OpenAIGatewayService{accountRepo: &modelNotFoundManagedAccountRepo{}} + body := []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`) + + for _, tc := range []struct { + name string + account *Account + want bool + }{ + {name: "openai api key", account: &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey}, want: true}, + {name: "compatible provider", account: &Account{Platform: PlatformDeepseek, Type: AccountTypeAPIKey}, want: true}, + {name: "anthropic account", account: &Account{Platform: PlatformAnthropic, Type: AccountTypeAPIKey}, want: false}, + {name: "missing account", account: nil, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, svc.shouldFailoverOpenAIUpstreamResponse( + tc.account, http.StatusBadRequest, "model not found", body, + )) + }) + } + + require.False(t, svc.shouldFailoverOpenAIUpstreamResponse( + &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey}, + http.StatusBadRequest, + "Invalid value for temperature", + []byte(`{"error":{"code":"invalid_request_error","message":"Invalid value for temperature"}}`), + )) +} + +func TestOpenAICompatibleModelNotFound400WithoutManagedCandidatesRemainsTerminal(t *testing.T) { + svc := &OpenAIGatewayService{} + body := []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`) + + require.False(t, svc.shouldFailoverOpenAIUpstreamResponse( + &Account{Platform: PlatformOpenAI, Type: AccountTypeOAuth}, + http.StatusBadRequest, + "model not found", + body, + )) +} + +func TestFailoverOpenAIUpstreamHTTPError_ModelNotFoundIsNextAccountEligible(t *testing.T) { + c, _ := newOpenAIUpstreamErrorTestContext(t) + svc := &OpenAIGatewayService{cfg: &config.Config{}, accountRepo: &modelNotFoundManagedAccountRepo{}} + account := &Account{ID: 42, Name: "compatible", Platform: PlatformOpenAI, Type: AccountTypeAPIKey} + body := []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`) + resp := newOpenAIUpstreamErrorResponse(http.StatusBadRequest, string(body)) + + failoverErr := svc.failoverOpenAIUpstreamHTTPError( + context.Background(), c, account, resp, body, "model not found", "missing-model", + ) + + require.NotNil(t, failoverErr) + require.Equal(t, http.StatusBadRequest, failoverErr.StatusCode) + require.Equal(t, body, failoverErr.ResponseBody) + require.True(t, failoverErr.ShouldRetryNextAccount()) + require.False(t, IsResponseCommitted(c), "failover must remain eligible before writing downstream") +} + // 主复现：原生 Responses 路径必须回真实的 400 与上游诊断信息，而不是可重试的 502。 // // 归一成 502 时下游网关（CCH 等）会把确定性的 Schema 错误当成临时上游故障重试， diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index 03339deffc..52420c584a 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -590,7 +590,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( if upstreamMsg == "" { upstreamMsg = http.StatusText(resp.StatusCode) } - shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) + shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) if account.Platform == PlatformGrok { shouldFailover = s.shouldFailoverGrokUpstreamError(resp.StatusCode, respBody) s.handleGrokAccountUpstreamError(withGrokTeamRateLimitModel(ctx, resolveGrokWSUpstreamModel(account, body, originalModel)), account, resp.StatusCode, resp.Header, respBody)