Files
sub2api/backend/internal/service/ollama_cloud_usage_reset_test.go
T
alfadb a1d5968b24 feat(ollama): async rate-limit reset for Ollama Cloud usage windows
Real Ollama Cloud 429s are driven by ollama.com usage windows, not by
response headers that OpenAI/Anthropic/CN rate-limit parsing understands.
Handle them separately: apply a never-shortening temporary cooldown first,
then schedule an async probe that learns the true usage-window reset point
with a controlled, coalesced per-api_key refresh.

- ratelimit_service: route real Ollama Cloud usage accounts to a dedicated
  429 path before the CN-provider branch.
- ollama_cloud_usage: add a background probe coordinator (started in Start,
  waited by Stop) that fans out model-429 probes to one controlled refresh
  per api_key group, reusing the recent group snapshot for coalesced members.
- repository/account_repo: add SetRateLimitedIfUnchanged, a write-back CAS
  counterpart to ClearRateLimitIfObserved, so an async reset never overwrites
  a newer 429, an admin clear, a re-armed generation, or a key/state change
  observed by another writer between read and write.
- wire: inject OllamaCloudUsageService into RateLimitService.

Tests: unit tests for service and repository pass; targeted race tests pass;
build and vet clean; git diff --check clean. The CAS integration test compiles
but is not run (no Docker available).
2026-09-13 10:41:47 +08:00

244 lines
6.7 KiB
Go

package service
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func mustT(t time.Time) *time.Time { return &t }
func TestOllamaCloudUsageExhaustionResetAt(t *testing.T) {
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
minFetchedAt := now.Add(-time.Hour)
at := func(h, m int) *time.Time {
return mustT(now.Add(time.Duration(h)*time.Hour + time.Duration(m)*time.Minute))
}
window := func(percent float64, reset *time.Time) *OllamaCloudUsageWindow {
return &OllamaCloudUsageWindow{UsedPercent: percent, ResetAt: reset}
}
okSnapshot := func(data *OllamaCloudUsageData) *OllamaCloudUsageSnapshot {
return &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK,
Data: data,
FetchedAt: mustT(now.Add(-time.Minute)),
}
}
tests := []struct {
name string
snapshot *OllamaCloudUsageSnapshot
now time.Time
minFresh time.Time
want time.Time
wantOK bool
}{
{
name: "nil snapshot",
wantOK: false,
},
{
name: "failed status unusable",
snapshot: &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusFailed,
Data: &OllamaCloudUsageData{FiveHour: window(100, at(2, 0))},
},
wantOK: false,
},
{
name: "nil data unusable",
snapshot: &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK,
FetchedAt: mustT(now.Add(-time.Minute)),
},
wantOK: false,
},
{
name: "stale snapshot rejected",
snapshot: func() *OllamaCloudUsageSnapshot {
s := okSnapshot(&OllamaCloudUsageData{FiveHour: window(100, at(2, 0))})
s.FetchedAt = mustT(now.Add(-3 * time.Hour))
return s
}(),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "no window exhausted yields no recovery",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(50, at(2, 0)),
SevenDay: window(30, at(48, 0)),
}),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "five hour exhausted below boundary not counted",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(99.999, at(2, 0)),
}),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "five hour exactly 100 future reset",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
}),
minFresh: minFetchedAt,
want: now.Add(2 * time.Hour),
wantOK: true,
},
{
name: "five hour exhausted past reset yields no recovery",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(-1, 0)),
}),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "five hour exhausted missing reset yields no recovery",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, nil),
}),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "seven day exhausted alone",
snapshot: okSnapshot(&OllamaCloudUsageData{
SevenDay: window(120, at(30, 0)),
}),
minFresh: minFetchedAt,
want: now.Add(30 * time.Hour),
wantOK: true,
},
{
name: "seven day exhausted missing reset blocks complete recovery",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
SevenDay: window(100, nil),
}),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "seven day exhausted past reset blocks complete recovery",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
SevenDay: window(100, at(-1, 0)),
}),
minFresh: minFetchedAt,
wantOK: false,
},
{
name: "both exhausted returns later reset",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
SevenDay: window(100, at(48, 0)),
}),
minFresh: minFetchedAt,
want: now.Add(48 * time.Hour),
wantOK: true,
},
{
name: "both exhausted returns later even when seven day is sooner",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(48, 0)),
SevenDay: window(100, at(2, 0)),
}),
minFresh: minFetchedAt,
want: now.Add(48 * time.Hour),
wantOK: true,
},
{
name: "non exhausted window reset must not affect",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
SevenDay: window(40, at(1, 0)), // past + not exhausted -> ignored
}),
minFresh: minFetchedAt,
want: now.Add(2 * time.Hour),
wantOK: true,
},
{
name: "non exhausted window with missing reset must not affect",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
SevenDay: window(40, nil),
}),
minFresh: minFetchedAt,
want: now.Add(2 * time.Hour),
wantOK: true,
},
{
name: "absent seven day does not block",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(2, 0)),
}),
minFresh: minFetchedAt,
want: now.Add(2 * time.Hour),
wantOK: true,
},
{
name: "future reset at now boundary treated as not future",
snapshot: okSnapshot(&OllamaCloudUsageData{
FiveHour: window(100, at(0, 0)),
}),
minFresh: minFetchedAt,
wantOK: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
testNow := tt.now
if testNow.IsZero() {
testNow = now
}
got, ok := ollamaCloudUsageExhaustionResetAt(tt.snapshot, testNow, tt.minFresh)
require.Equal(t, tt.wantOK, ok)
if tt.wantOK {
require.True(t, got.Equal(tt.want), "reset = %v, want %v", got, tt.want)
} else {
require.True(t, got.IsZero())
}
})
}
}
// TestOllamaCloudUsageExhaustionResetAtMissingFetchedAt checks the fetched-at
// gate separately because the okSnapshot helper always sets FetchedAt.
func TestOllamaCloudUsageExhaustionResetAtMissingFetchedAt(t *testing.T) {
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
snapshot := &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK,
Data: &OllamaCloudUsageData{
FiveHour: &OllamaCloudUsageWindow{UsedPercent: 100, ResetAt: mustT(now.Add(2 * time.Hour))},
},
}
got, ok := ollamaCloudUsageExhaustionResetAt(snapshot, now, now.Add(-time.Hour))
require.False(t, ok)
require.True(t, got.IsZero())
}
// TestOllamaCloudUsageExhaustionResetAtFetchedBoundary pins the inclusive
// minFetchedAt boundary: a snapshot fetched exactly at the bound is usable.
func TestOllamaCloudUsageExhaustionResetAtFetchedBoundary(t *testing.T) {
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
bound := now.Add(-time.Hour)
snapshot := &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK,
FetchedAt: &bound, // exactly at the validity bound -> accepted
Data: &OllamaCloudUsageData{
FiveHour: &OllamaCloudUsageWindow{UsedPercent: 100, ResetAt: mustT(now.Add(2 * time.Hour))},
},
}
got, ok := ollamaCloudUsageExhaustionResetAt(snapshot, now, bound)
require.True(t, ok)
require.True(t, got.Equal(now.Add(2*time.Hour)))
}