diff --git a/backend/internal/service/ratelimit_service.go b/backend/internal/service/ratelimit_service.go index 3bd467f464..47f234f85d 100644 --- a/backend/internal/service/ratelimit_service.go +++ b/backend/internal/service/ratelimit_service.go @@ -366,12 +366,16 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc // otherwise a broad "rate limit" keyword rule can shorten a multi-hour // cooldown to a local temporary pause. if statusCode == http.StatusTooManyRequests && account.Platform == PlatformAnthropic { + // Fable may be rejected because the organization has no usage credits for + // this model. Anthropic reports that as 429, but it is a model entitlement + // failure rather than a shared account window exhaustion. + fableCreditsRequired := s.persistAnthropicFableCreditsRequired(ctx, account, headers, responseBody, firstRequestedModel(requestedModel)) // 7d_oi 是 Fable 模型专属的 7d 窗口:只标记模型级限流,账号对其他模型仍可调度。 fableLimited := s.persistAnthropicFableWindowLimit(ctx, account, headers) if s.persistAnthropicExhaustedWindowLimit(ctx, account, headers) { return false } - if fableLimited { + if fableCreditsRequired || fableLimited { return false } } @@ -1486,7 +1490,59 @@ func (s *RateLimitService) persistAnthropicExhaustedWindowLimit(ctx context.Cont return true } -const anthropicFableWindowReason = "anthropic_7d_oi_window_exhausted" +const ( + anthropicFableWindowReason = "anthropic_7d_oi_window_exhausted" + anthropicFableCreditsRequiredReason = "anthropic_fable_credits_required" +) + +// persistAnthropicFableCreditsRequired handles Anthropic's credits_required +// response for Fable. Although the upstream status is 429, this response only +// says that the organization cannot use Fable; marking the whole account rate +// limited would unnecessarily stop Sonnet, Opus, and Haiku scheduling. +func (s *RateLimitService) persistAnthropicFableCreditsRequired(ctx context.Context, account *Account, headers http.Header, responseBody []byte, requestedModel string) bool { + if s == nil || s.accountRepo == nil || account == nil { + return false + } + if !strings.EqualFold(strings.TrimSpace(gjson.GetBytes(responseBody, "error.details.error_code").String()), "credits_required") { + return false + } + + model := strings.TrimSpace(gjson.GetBytes(responseBody, "error.details.model").String()) + if model == "" { + model = strings.TrimSpace(requestedModel) + } + if !isAnthropicFableModel(model) { + return false + } + + now := time.Now() + resetAt, ok := parseAnthropicResetTimestamp(headers.Get("anthropic-ratelimit-unified-reset"), now, 366*24*time.Hour) + if !ok { + cooldown, enabled := s.get429FallbackCooldown(ctx, account) + if !enabled { + slog.Info("anthropic_fable_credits_required_cooldown_ignored", "account_id", account.ID) + return true + } + resetAt = now.Add(cooldown) + } + + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, anthropicFableRateLimitKey, resetAt, anthropicFableCreditsRequiredReason); err != nil { + slog.Warn("anthropic_fable_credits_required_rate_limit_set_failed", + "account_id", account.ID, + "scope", anthropicFableRateLimitKey, + "reset_at", resetAt, + "error", err) + // The response is still known to be Fable-specific. Do not widen a + // persistence failure into an account-level rate limit. + return true + } + slog.Info("anthropic_fable_credits_required_model_rate_limited", + "account_id", account.ID, + "scope", anthropicFableRateLimitKey, + "reset_at", resetAt, + "reset_in", time.Until(resetAt).Truncate(time.Second)) + return true +} // selectAnthropicFableWindowLimit parses the Anthropic 7d_oi per-model window // headers (the Fable-only 7d window, e.g. anthropic-ratelimit-unified-7d_oi-*). diff --git a/backend/internal/service/ratelimit_service_anthropic_window_limit_test.go b/backend/internal/service/ratelimit_service_anthropic_window_limit_test.go index a13e8fe333..328bf07fe6 100644 --- a/backend/internal/service/ratelimit_service_anthropic_window_limit_test.go +++ b/backend/internal/service/ratelimit_service_anthropic_window_limit_test.go @@ -160,6 +160,82 @@ func TestHandleUpstreamError_Anthropic7dOiOnlyMarksModelRateLimit(t *testing.T) require.Equal(t, 0.41, repo.lastExtraUpdates["session_window_utilization"]) } +func TestHandleUpstreamError_AnthropicFableCreditsRequiredOnlyMarksModelRateLimit(t *testing.T) { + resetAt := time.Now().Add(29 * 24 * time.Hour).Truncate(time.Second) + headers := http.Header{} + headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(resetAt.Unix(), 10)) + + repo := &anthropicWindowLimitRepo{} + svc := NewRateLimitService(repo, nil, nil, nil, nil) + account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic} + body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required","model":"claude-fable-5","disabled_reason":"org_level_disabled"},"message":"Usage credits are required for this model."}}`) + + shouldDisable := svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, body, "claude-fable-5") + + require.False(t, shouldDisable) + require.Zero(t, repo.rateLimitCalls, "Fable entitlement failure must not rate limit the whole account") + require.Zero(t, repo.sessionWindowCalls, "Fable entitlement failure must not rewrite the shared session window") + require.Equal(t, 1, repo.modelRateLimitCalls) + require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope) + require.Equal(t, resetAt, repo.lastModelRateLimitReset) +} + +func TestHandleUpstreamError_AnthropicFableCreditsRequiredFallsBackToRequestedModel(t *testing.T) { + startedAt := time.Now() + repo := &anthropicWindowLimitRepo{} + svc := NewRateLimitService(repo, nil, nil, nil, nil) + account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic} + body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required"},"message":"Usage credits are required for this model."}}`) + + svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{}, body, "claude-fable-5[1m]") + + require.Zero(t, repo.rateLimitCalls) + require.Equal(t, 1, repo.modelRateLimitCalls) + require.Equal(t, anthropicFableRateLimitKey, repo.lastModelRateLimitScope) + require.True(t, repo.lastModelRateLimitReset.After(startedAt)) +} + +func TestHandleUpstreamError_AnthropicNonFableCreditsRequiredKeepsLegacyBehavior(t *testing.T) { + resetAt := time.Now().Add(2 * time.Hour).Truncate(time.Second) + headers := http.Header{} + headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(resetAt.Unix(), 10)) + + repo := &anthropicWindowLimitRepo{} + svc := NewRateLimitService(repo, nil, nil, nil, nil) + account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic} + body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required","model":"claude-opus-5"},"message":"Usage credits are required for this model."}}`) + + svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, body, "claude-opus-5") + + require.Zero(t, repo.modelRateLimitCalls) + require.Equal(t, 1, repo.rateLimitCalls) + require.Equal(t, resetAt, repo.lastRateLimitReset) +} + +func TestHandleUpstreamError_AnthropicSharedWindowStillWinsWithFableCreditsRequired(t *testing.T) { + now := time.Now() + reset5h := now.Add(2 * time.Hour).Truncate(time.Second) + reset7d := now.Add(80 * time.Hour).Truncate(time.Second) + headers := http.Header{} + headers.Set("anthropic-ratelimit-unified-5h-status", "rejected") + headers.Set("anthropic-ratelimit-unified-5h-utilization", "1.0") + headers.Set("anthropic-ratelimit-unified-5h-reset", strconv.FormatInt(reset5h.Unix(), 10)) + headers.Set("anthropic-ratelimit-unified-7d-status", "allowed") + headers.Set("anthropic-ratelimit-unified-7d-reset", strconv.FormatInt(reset7d.Unix(), 10)) + headers.Set("anthropic-ratelimit-unified-reset", strconv.FormatInt(reset5h.Unix(), 10)) + + repo := &anthropicWindowLimitRepo{} + svc := NewRateLimitService(repo, nil, nil, nil, nil) + account := &Account{ID: 42, Type: AccountTypeOAuth, Platform: PlatformAnthropic} + body := []byte(`{"type":"error","error":{"details":{"error_code":"credits_required","model":"claude-fable-5"}}}`) + + svc.HandleUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, body, "claude-fable-5") + + require.Equal(t, 1, repo.modelRateLimitCalls, "Fable-specific failure should still be recorded") + require.Equal(t, 1, repo.rateLimitCalls, "an explicitly rejected shared window must still rate limit the account") + require.Equal(t, reset5h, repo.lastRateLimitReset) +} + func TestHandleUpstreamError_Anthropic5hWindowStillWinsOver7dOi(t *testing.T) { // 5h 窗口 rejected 时必须仍按账号级限流处理(用 5h reset),同时记录 Fable 模型限流。 now := time.Now()