mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 12:02:27 +08:00
Merge pull request #6711 from wucm667/fix/issue-6701-model-not-found-failover
Fix failover for model-not-found responses
This commit is contained in:
@@ -128,6 +128,44 @@ func TestOpenAICapacityFailoverExhaustionPreservesMessageAsServerError(t *testin
|
||||
})
|
||||
}
|
||||
|
||||
func TestOpenAIManagedSingleAccountModelNotFoundExhaustionPreservesStructured400(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
body := []byte(`{"error":{"type":"invalid_request_error","code":"model_not_found","param":"model","message":"The requested model is unavailable on this account"}}`)
|
||||
|
||||
(&OpenAIGatewayHandler{}).handleFailoverExhausted(c, &service.UpstreamFailoverError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
ResponseBody: body,
|
||||
}, false)
|
||||
|
||||
require.Equal(t, http.StatusBadRequest, recorder.Code)
|
||||
require.Equal(t, "invalid_request_error", gjson.Get(recorder.Body.String(), "error.type").String())
|
||||
require.Equal(t, "model_not_found", gjson.Get(recorder.Body.String(), "error.code").String())
|
||||
require.Equal(t, "model", gjson.Get(recorder.Body.String(), "error.param").String())
|
||||
require.Equal(t, "The requested model is unavailable on this account", gjson.Get(recorder.Body.String(), "error.message").String())
|
||||
}
|
||||
|
||||
func TestOpenAIManagedModelNotFoundExhaustionSanitizesMessage(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
body := []byte(`{"error":{"code":"model_not_found","message":"Model not found; inspect https://upstream.example/debug?access_token=super-secret-value&model=x"}}`)
|
||||
|
||||
(&OpenAIGatewayHandler{}).handleFailoverExhausted(c, &service.UpstreamFailoverError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
ResponseBody: body,
|
||||
}, false)
|
||||
|
||||
require.Equal(t, http.StatusBadRequest, recorder.Code)
|
||||
require.NotContains(t, recorder.Body.String(), "super-secret-value")
|
||||
require.Contains(t, gjson.Get(recorder.Body.String(), "error.message").String(), "access_token=***")
|
||||
recorded, ok := c.Get(service.OpsUpstreamErrorMessageKey)
|
||||
require.True(t, ok)
|
||||
require.NotContains(t, recorded, "super-secret-value")
|
||||
require.Contains(t, recorded, "access_token=***")
|
||||
}
|
||||
|
||||
func TestResponsesFailoverExhaustedAfterForwardedTerminalMarksOpsWithoutDuplicateFrame(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
recorder := httptest.NewRecorder()
|
||||
|
||||
@@ -3310,6 +3310,12 @@ func (h *OpenAIGatewayHandler) handleFailoverExhausted(c *gin.Context, failoverE
|
||||
}
|
||||
statusCode := failoverErr.StatusCode
|
||||
responseBody := failoverErr.ResponseBody
|
||||
if statusCode == http.StatusBadRequest && service.IsOpenAICompatibleModelNotFound400(responseBody) && !streamStarted {
|
||||
upstreamMsg := service.SanitizeUpstreamErrorMessage(service.ExtractUpstreamErrorMessage(responseBody))
|
||||
service.SetOpsUpstreamError(c, statusCode, upstreamMsg, "")
|
||||
service.WriteOpenAIUpstreamClientError(c, statusCode, responseBody, upstreamMsg)
|
||||
return
|
||||
}
|
||||
if service.IsOpenAISilentRefusalErrorBody(responseBody) {
|
||||
service.SetOpsUpstreamError(c, statusCode, service.OpenAISilentRefusalClientMessage(), "")
|
||||
h.handleStreamingAwareError(c, http.StatusBadGateway, "upstream_error", service.OpenAISilentRefusalClientMessage(), streamStarted)
|
||||
|
||||
@@ -285,6 +285,12 @@ func ExtractUpstreamErrorMessage(body []byte) string {
|
||||
return extractUpstreamErrorMessage(body)
|
||||
}
|
||||
|
||||
// SanitizeUpstreamErrorMessage redacts sensitive query parameter values before
|
||||
// an upstream error message is recorded or returned to a client.
|
||||
func SanitizeUpstreamErrorMessage(message string) string {
|
||||
return sanitizeUpstreamErrorMessage(message)
|
||||
}
|
||||
|
||||
func extractUpstreamErrorMessage(body []byte) string {
|
||||
// Claude 风格:{"type":"error","error":{"type":"...","message":"..."}}
|
||||
if m := gjson.GetBytes(body, "error.message").String(); strings.TrimSpace(m) != "" {
|
||||
|
||||
@@ -112,3 +112,29 @@ func TestIsOpenAICodexPlanGatedModelError(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsOpenAICompatibleModelNotFound400(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want bool
|
||||
}{
|
||||
{name: "structured code", body: `{"error":{"code":"model_not_found","message":"No such deployment"}}`, want: true},
|
||||
{name: "unknown provider", body: `{"error":{"message":"Unknown provider for model claude-x"}}`, want: true},
|
||||
{name: "model not found", body: `{"error":{"message":"Model not found: claude-x"}}`, want: true},
|
||||
{name: "model unsupported", body: `{"error":{"message":"The requested model is not supported"}}`, want: true},
|
||||
{name: "plain text compatible gateway", body: `unknown provider for model claude-x`, want: true},
|
||||
{name: "invalid parameter remains terminal", body: `{"error":{"code":"invalid_request_error","message":"Invalid value for temperature"}}`, want: false},
|
||||
{name: "structured non-matching code overrides model not found message", body: `{"error":{"code":"invalid_request_error","message":"Model not found: claude-x"}}`, want: false},
|
||||
{name: "structured non-matching error code overrides unsupported message", body: `{"error":{"code":"upstream_error","message":"The requested model is not supported"}}`, want: false},
|
||||
{name: "echoed phrase is ignored", body: `{"error":{"message":"Invalid request"},"echo":"model not found"}`, want: false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isOpenAICompatibleModelNotFound400([]byte(tt.body)); got != tt.want {
|
||||
t.Fatalf("isOpenAICompatibleModelNotFound400() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,7 +77,7 @@ func TestOpenAIUpstreamAccessStateClassification(t *testing.T) {
|
||||
if !tt.want {
|
||||
return
|
||||
}
|
||||
require.True(t, (&OpenAIGatewayService{}).shouldFailoverOpenAIUpstreamResponse(http.StatusForbidden, "", body))
|
||||
require.True(t, (&OpenAIGatewayService{}).shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusForbidden, "", body))
|
||||
require.True(t, shouldFailoverOpenAIPassthroughResponse(&Account{Type: AccountTypeOAuth}, http.StatusForbidden, body))
|
||||
|
||||
err := newOpenAIUpstreamFailoverError(http.StatusForbidden, nil, body, "", true)
|
||||
@@ -105,7 +105,7 @@ func TestOpenAIHTTPAccessStateDoesNotTrustBadRequestMessage(t *testing.T) {
|
||||
|
||||
require.False(t, isOpenAIUpstreamAccessStateError("", body), "free-form stream messages are not durable account evidence")
|
||||
require.False(t, isOpenAIHTTPUpstreamAccessStateError(http.StatusBadRequest, "", body))
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadRequest, "", body))
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadRequest, "", body))
|
||||
require.False(t, shouldFailoverOpenAIPassthroughResponse(&Account{Type: AccountTypeOAuth}, http.StatusBadRequest, body))
|
||||
|
||||
err := newOpenAIUpstreamFailoverError(http.StatusBadRequest, nil, body, "", false)
|
||||
@@ -148,7 +148,7 @@ func TestOpenAIHTTPAccessStateTrustsStructuredCode(t *testing.T) {
|
||||
body := []byte(`{"error":{"code":"organization_deactivated","message":"request rejected"}}`)
|
||||
|
||||
require.True(t, isOpenAIHTTPUpstreamAccessStateError(http.StatusBadRequest, "", body))
|
||||
require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadRequest, "", body))
|
||||
require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadRequest, "", body))
|
||||
require.True(t, svc.handleOpenAIAccountUpstreamError(context.Background(), account, http.StatusBadRequest, nil, body))
|
||||
require.Equal(t, 1, repo.setErrorCalls)
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
@@ -189,7 +189,7 @@ func TestOpenAIHTTPAuthMessagesUseExistingStatusPolicies(t *testing.T) {
|
||||
func TestOpenAICyberPolicyWrapped5xxNeverFailsOver(t *testing.T) {
|
||||
body := []byte(`{"error":{"code":"cyber_policy","message":"blocked"}}`)
|
||||
svc := &OpenAIGatewayService{}
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadGateway, "wrapped upstream failure", body))
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "wrapped upstream failure", body))
|
||||
require.False(t, shouldFailoverOpenAIPassthroughResponse(&Account{Type: AccountTypeOAuth}, http.StatusBadGateway, body))
|
||||
}
|
||||
|
||||
|
||||
@@ -91,7 +91,7 @@ func (s *OpenAIGatewayService) ForwardAlphaSearch(ctx context.Context, c *gin.Co
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
upstreamMessage := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody)))
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMessage, respBody) ||
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMessage, respBody) ||
|
||||
isOpenAIAlphaSearchEndpointUnsupported(account, resp.StatusCode) {
|
||||
resp.Body = io.NopCloser(bytes.NewReader(respBody))
|
||||
// alpha/search 是独立的工具端点,单次 401 不能证明账号的模型调用
|
||||
@@ -175,7 +175,7 @@ func (s *OpenAIGatewayService) forwardAlphaSearchViaResponsesWebSearch(
|
||||
|
||||
if resp.StatusCode >= http.StatusBadRequest {
|
||||
upstreamMessage := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody)))
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMessage, respBody) {
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMessage, respBody) {
|
||||
resp.Body = io.NopCloser(bytes.NewReader(respBody))
|
||||
// 仍按 alpha/search 工具请求处理:PAT 的工具链路失败不能直接永久置错。
|
||||
shouldDisable := false
|
||||
|
||||
@@ -118,7 +118,7 @@ func (s *OpenAIGatewayService) ForwardEmbeddings(
|
||||
|
||||
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
|
||||
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) {
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) {
|
||||
upstreamDetail := ""
|
||||
if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
|
||||
maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes
|
||||
|
||||
@@ -88,7 +88,7 @@ func (s *OpenAIGatewayService) failoverOpenAIUpstreamHTTPError(
|
||||
upstreamMsg string,
|
||||
upstreamModel string,
|
||||
) *UpstreamFailoverError {
|
||||
shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody)
|
||||
shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody)
|
||||
tempUnscheduled := false
|
||||
if c != nil && account != nil && account.Platform != PlatformGrok && !shouldFailover && !IsResponseCommitted(c) && s.rateLimitService != nil {
|
||||
tempUnscheduled = s.rateLimitService.CheckErrorPolicy(ctx, account, resp.StatusCode, respBody, upstreamModel) == ErrorPolicyTempUnscheduled
|
||||
|
||||
@@ -1136,7 +1136,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
)
|
||||
continue
|
||||
}
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) {
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) {
|
||||
upstreamDetail := ""
|
||||
if s.cfg != nil && s.cfg.Gateway.LogUpstreamErrorBody {
|
||||
maxBytes := s.cfg.Gateway.LogUpstreamErrorBodyMaxBytes
|
||||
@@ -1211,7 +1211,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
compactResp, compactBody := openAICompactFallbackErrorResponse(resp, signal)
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(compactResp.StatusCode, signal.message, compactBody) {
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, compactResp.StatusCode, signal.message, compactBody) {
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
ProxyID: opsUpstreamProxyID(account),
|
||||
ProxyName: opsUpstreamProxyName(account),
|
||||
|
||||
@@ -356,9 +356,9 @@ func TestShouldFailoverOpenAIUpstreamResponseContextWindow502(t *testing.T) {
|
||||
svc := &OpenAIGatewayService{}
|
||||
body := []byte(`{"error":{"message":"Your input exceeds the context window of this model. Please adjust your input and try again.","type":"upstream_error","code":null}}`)
|
||||
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadGateway, "", body))
|
||||
require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(http.StatusBadGateway, "temporary upstream outage", []byte(`{"error":{"message":"temporary upstream outage"}}`)))
|
||||
require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "", body))
|
||||
require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(), http.StatusBadGateway, "temporary upstream outage", []byte(`{"error":{"message":"temporary upstream outage"}}`)))
|
||||
require.True(t, svc.shouldFailoverOpenAIUpstreamResponse(newOpenAIUpstreamErrorTestAccount(),
|
||||
http.StatusBadGateway,
|
||||
"temporary upstream outage",
|
||||
[]byte(`{"error":{"message":"temporary upstream outage"},"echo":"context_length_exceeded"}`),
|
||||
|
||||
@@ -250,7 +250,7 @@ func (s *OpenAIGatewayService) shouldFailoverUpstreamError(statusCode int) bool
|
||||
}
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(statusCode int, upstreamMsg string, upstreamBody []byte) bool {
|
||||
func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(account *Account, statusCode int, upstreamMsg string, upstreamBody []byte) bool {
|
||||
// cyber_policy is request-scoped even when an intermediary wraps the
|
||||
// provider response in a retryable 5xx status. Never punish or rotate the
|
||||
// selected credential for it.
|
||||
@@ -266,12 +266,46 @@ func (s *OpenAIGatewayService) shouldFailoverOpenAIUpstreamResponse(statusCode i
|
||||
if isOpenAIRequestBodyTooLargeError(statusCode, upstreamMsg, upstreamBody) {
|
||||
return true
|
||||
}
|
||||
// A missing model is account/provider availability, not a malformed client
|
||||
// request. Keep this unconditional exception inside the OpenAI-compatible
|
||||
// gateway and require an eligible account so Anthropic/Gemini paths retain
|
||||
// their existing opt-in 400 behavior.
|
||||
// A bare forwarding service has no account-selection owner to consume a
|
||||
// failover sentinel. In that mode (used by direct/single-account callers),
|
||||
// preserve the deterministic upstream 400 instead of returning an unwritten
|
||||
// retry signal. Managed gateway instances always have an account repository;
|
||||
// their handler can exclude this account and actually select another one.
|
||||
if s != nil && s.accountRepo != nil && account != nil && account.IsOpenAICompatible() && statusCode == http.StatusBadRequest &&
|
||||
isOpenAICompatibleModelNotFound400(upstreamBody) {
|
||||
return true
|
||||
}
|
||||
if s.shouldFailoverUpstreamError(statusCode) {
|
||||
return true
|
||||
}
|
||||
return isOpenAITransientProcessingError(statusCode, upstreamMsg, upstreamBody)
|
||||
}
|
||||
|
||||
func isOpenAICompatibleModelNotFound400(respBody []byte) bool {
|
||||
code := strings.TrimSpace(extractUpstreamErrorCode(respBody))
|
||||
if code != "" {
|
||||
return strings.EqualFold(code, "model_not_found")
|
||||
}
|
||||
|
||||
msg := strings.ToLower(strings.TrimSpace(extractUpstreamErrorMessage(respBody)))
|
||||
if msg == "" && !gjson.ValidBytes(respBody) {
|
||||
msg = strings.ToLower(strings.TrimSpace(string(respBody)))
|
||||
}
|
||||
return strings.Contains(msg, "unknown provider for model") ||
|
||||
strings.Contains(msg, "model not found") ||
|
||||
strings.Contains(msg, "model is not supported")
|
||||
}
|
||||
|
||||
// IsOpenAICompatibleModelNotFound400 reports whether an OpenAI-compatible 400
|
||||
// is an account-specific missing-model response eligible for failover.
|
||||
func IsOpenAICompatibleModelNotFound400(respBody []byte) bool {
|
||||
return isOpenAICompatibleModelNotFound400(respBody)
|
||||
}
|
||||
|
||||
// OpenAIRequestBodyTooLargeClientMessage is the fixed downstream message used
|
||||
// after all account-specific request body limit failovers are exhausted.
|
||||
const OpenAIRequestBodyTooLargeClientMessage = "Request payload is too large"
|
||||
|
||||
@@ -650,7 +650,7 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesAPIKey(
|
||||
resp.Body = io.NopCloser(bytes.NewReader(respBody))
|
||||
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
|
||||
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) {
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) {
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
ProxyID: opsUpstreamProxyID(account),
|
||||
ProxyName: opsUpstreamProxyName(account),
|
||||
|
||||
@@ -1856,7 +1856,7 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth(
|
||||
resp.Body = io.NopCloser(bytes.NewReader(respBody))
|
||||
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
|
||||
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) {
|
||||
if s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody) {
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
ProxyID: opsUpstreamProxyID(account),
|
||||
ProxyName: opsUpstreamProxyName(account),
|
||||
|
||||
@@ -199,7 +199,7 @@ func (s *OpenAIGatewayService) CreateLiveCall(
|
||||
selection.ReleaseFunc()
|
||||
if createErr != nil {
|
||||
s.releaseLiveLease(account.ID, identity.UserID, identity.APIKeyID, leaseID)
|
||||
if !s.shouldFailoverLiveCreateError(createErr) {
|
||||
if !s.shouldFailoverLiveCreateError(account, createErr) {
|
||||
return nil, createErr
|
||||
}
|
||||
excluded[account.ID] = struct{}{}
|
||||
@@ -245,13 +245,13 @@ func (s *OpenAIGatewayService) CreateLiveCall(
|
||||
return nil, ErrLiveUnavailable
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) shouldFailoverLiveCreateError(err error) bool {
|
||||
func (s *OpenAIGatewayService) shouldFailoverLiveCreateError(account *Account, err error) bool {
|
||||
var upstreamErr *UpstreamFailoverError
|
||||
if !errors.As(err, &upstreamErr) {
|
||||
// 凭证读取和网络传输错误都可能只影响当前账号或代理。
|
||||
return true
|
||||
}
|
||||
return s.shouldFailoverOpenAIUpstreamResponse(
|
||||
return s.shouldFailoverOpenAIUpstreamResponse(account,
|
||||
upstreamErr.StatusCode,
|
||||
"",
|
||||
upstreamErr.ResponseBody,
|
||||
|
||||
@@ -210,17 +210,18 @@ func TestLiveSidebandNormalCloseEndsCall(t *testing.T) {
|
||||
|
||||
func TestLiveCreateFailoverUsesExistingOpenAIPolicy(t *testing.T) {
|
||||
service := &OpenAIGatewayService{}
|
||||
require.False(t, service.shouldFailoverLiveCreateError(&UpstreamFailoverError{
|
||||
account := newOpenAIUpstreamErrorTestAccount()
|
||||
require.False(t, service.shouldFailoverLiveCreateError(account, &UpstreamFailoverError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
ResponseBody: []byte(`{"error":{"message":"invalid session"}}`),
|
||||
}))
|
||||
require.True(t, service.shouldFailoverLiveCreateError(&UpstreamFailoverError{
|
||||
require.True(t, service.shouldFailoverLiveCreateError(account, &UpstreamFailoverError{
|
||||
StatusCode: http.StatusForbidden,
|
||||
}))
|
||||
require.True(t, service.shouldFailoverLiveCreateError(&UpstreamFailoverError{
|
||||
require.True(t, service.shouldFailoverLiveCreateError(account, &UpstreamFailoverError{
|
||||
StatusCode: http.StatusBadGateway,
|
||||
}))
|
||||
require.True(t, service.shouldFailoverLiveCreateError(errors.New("transport failed")))
|
||||
require.True(t, service.shouldFailoverLiveCreateError(account, errors.New("transport failed")))
|
||||
}
|
||||
|
||||
func TestLiveCallIDFromLocation(t *testing.T) {
|
||||
|
||||
@@ -55,3 +55,9 @@ func writeOpenAIUpstreamClientError(c *gin.Context, statusCode int, body []byte,
|
||||
|
||||
c.JSON(statusCode, gin.H{"error": errorPayload})
|
||||
}
|
||||
|
||||
// WriteOpenAIUpstreamClientError preserves a structured deterministic upstream
|
||||
// client error when the handler has exhausted all eligible accounts.
|
||||
func WriteOpenAIUpstreamClientError(c *gin.Context, statusCode int, body []byte, upstreamMsg string) {
|
||||
writeOpenAIUpstreamClientError(c, statusCode, body, upstreamMsg)
|
||||
}
|
||||
|
||||
@@ -45,6 +45,67 @@ func newOpenAIUpstreamErrorTestAccount() *Account {
|
||||
return &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Name: "acct"}
|
||||
}
|
||||
|
||||
type modelNotFoundManagedAccountRepo struct{ AccountRepository }
|
||||
|
||||
func TestOpenAICompatibleModelNotFound400FailoverScope(t *testing.T) {
|
||||
svc := &OpenAIGatewayService{accountRepo: &modelNotFoundManagedAccountRepo{}}
|
||||
body := []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`)
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
account *Account
|
||||
want bool
|
||||
}{
|
||||
{name: "openai api key", account: &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey}, want: true},
|
||||
{name: "compatible provider", account: &Account{Platform: PlatformDeepseek, Type: AccountTypeAPIKey}, want: true},
|
||||
{name: "anthropic account", account: &Account{Platform: PlatformAnthropic, Type: AccountTypeAPIKey}, want: false},
|
||||
{name: "missing account", account: nil, want: false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
require.Equal(t, tc.want, svc.shouldFailoverOpenAIUpstreamResponse(
|
||||
tc.account, http.StatusBadRequest, "model not found", body,
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(
|
||||
&Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey},
|
||||
http.StatusBadRequest,
|
||||
"Invalid value for temperature",
|
||||
[]byte(`{"error":{"code":"invalid_request_error","message":"Invalid value for temperature"}}`),
|
||||
))
|
||||
}
|
||||
|
||||
func TestOpenAICompatibleModelNotFound400WithoutManagedCandidatesRemainsTerminal(t *testing.T) {
|
||||
svc := &OpenAIGatewayService{}
|
||||
body := []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`)
|
||||
|
||||
require.False(t, svc.shouldFailoverOpenAIUpstreamResponse(
|
||||
&Account{Platform: PlatformOpenAI, Type: AccountTypeOAuth},
|
||||
http.StatusBadRequest,
|
||||
"model not found",
|
||||
body,
|
||||
))
|
||||
}
|
||||
|
||||
func TestFailoverOpenAIUpstreamHTTPError_ModelNotFoundIsNextAccountEligible(t *testing.T) {
|
||||
c, _ := newOpenAIUpstreamErrorTestContext(t)
|
||||
svc := &OpenAIGatewayService{cfg: &config.Config{}, accountRepo: &modelNotFoundManagedAccountRepo{}}
|
||||
account := &Account{ID: 42, Name: "compatible", Platform: PlatformOpenAI, Type: AccountTypeAPIKey}
|
||||
body := []byte(`{"error":{"code":"model_not_found","message":"model not found"}}`)
|
||||
resp := newOpenAIUpstreamErrorResponse(http.StatusBadRequest, string(body))
|
||||
|
||||
failoverErr := svc.failoverOpenAIUpstreamHTTPError(
|
||||
context.Background(), c, account, resp, body, "model not found", "missing-model",
|
||||
)
|
||||
|
||||
require.NotNil(t, failoverErr)
|
||||
require.Equal(t, http.StatusBadRequest, failoverErr.StatusCode)
|
||||
require.Equal(t, body, failoverErr.ResponseBody)
|
||||
require.True(t, failoverErr.ShouldRetryNextAccount())
|
||||
require.False(t, IsResponseCommitted(c), "failover must remain eligible before writing downstream")
|
||||
}
|
||||
|
||||
// 主复现:原生 Responses 路径必须回真实的 400 与上游诊断信息,而不是可重试的 502。
|
||||
//
|
||||
// 归一成 502 时下游网关(CCH 等)会把确定性的 Schema 错误当成临时上游故障重试,
|
||||
|
||||
@@ -590,7 +590,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn(
|
||||
if upstreamMsg == "" {
|
||||
upstreamMsg = http.StatusText(resp.StatusCode)
|
||||
}
|
||||
shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody)
|
||||
shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(account, resp.StatusCode, upstreamMsg, respBody)
|
||||
if account.Platform == PlatformGrok {
|
||||
shouldFailover = s.shouldFailoverGrokUpstreamError(resp.StatusCode, respBody)
|
||||
s.handleGrokAccountUpstreamError(withGrokTeamRateLimitModel(ctx, resolveGrokWSUpstreamModel(account, body, originalModel)), account, resp.StatusCode, resp.Header, respBody)
|
||||
|
||||
Reference in New Issue
Block a user