feat(ollama): 按模型请求刷新云端用量

将 Ollama Cloud settings HTML 自动抓取从固定间隔轮询改为请求驱动的
trailing debounce + max-wait:无新请求不再抓取,连续请求最晚在
max-wait 强制刷新;失败退避仍优先于活动 due。新增 debounce_minutes
(默认 1),interval_minutes 保留为最长等待兼容字段。
This commit is contained in:
alfadb
2026-07-25 15:37:05 +08:00
parent 43d4bae246
commit 0f72b7dca7
17 changed files with 504 additions and 122 deletions
@@ -273,4 +273,5 @@ func TestGetOllamaCloudUsageSettingsHandlerSuccess(t *testing.T) {
require.Equal(t, http.StatusOK, recorder.Code) require.Equal(t, http.StatusOK, recorder.Code)
require.Contains(t, recorder.Body.String(), `"enabled":false`) require.Contains(t, recorder.Body.String(), `"enabled":false`)
require.Contains(t, recorder.Body.String(), `"interval_minutes":60`) require.Contains(t, recorder.Body.String(), `"interval_minutes":60`)
require.Contains(t, recorder.Body.String(), `"debounce_minutes":1`)
} }
@@ -354,9 +354,31 @@ func canonicalJSON(raw string) string {
return string(encoded) return string(encoded)
} }
// ListDueOllamaCloudUsageAccounts returns at most one due representative per // ollamaCloudUsageParseRFC3339SQL reuses the verified RFC3339(/Nano) parse path
// exact API key before hydration, preventing one shared group from consuming a // for a snapshot timestamp expression. Invalid or missing values fail open to NULL.
// whole runner cycle. func ollamaCloudUsageParseRFC3339SQL(expression string) string {
return `CASE
WHEN ` + expression + ` IS NULL THEN NULL
WHEN ` + expression + ` ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$'
THEN jsonb_path_query_first_tz(
to_jsonb(regexp_replace(
` + expression + `,
'(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$',
'\1\2'
)),
'$.datetime()', '{}'::jsonb, true
) #>> '{}'
ELSE NULL
END`
}
// ListDueOllamaCloudUsageAccounts returns at most one activity-driven candidate
// per exact API key. It stamps Account.LastUsedAt with the group MAX(last_used_at)
// so the service due pure function can decide without hydrating the whole table.
//
// SQL only prefilters potential candidates (missing/invalid snapshot, or group
// activity after the last successful fetch / failed attempt). Final due timing
// (debounce, max-wait, failure backoff) is evaluated in service code.
func (r *accountRepository) ListDueOllamaCloudUsageAccounts(ctx context.Context, now time.Time, limit int) ([]service.Account, error) { func (r *accountRepository) ListDueOllamaCloudUsageAccounts(ctx context.Context, now time.Time, limit int) ([]service.Account, error) {
if limit <= 0 { if limit <= 0 {
return []service.Account{}, nil return []service.Account{}, nil
@@ -364,64 +386,90 @@ func (r *accountRepository) ListDueOllamaCloudUsageAccounts(ctx context.Context,
if r == nil || r.sql == nil { if r == nil || r.sql == nil {
return nil, errors.New("account repository SQL executor not configured") return nil, errors.New("account repository SQL executor not configured")
} }
// now is retained in the signature for callers; activity due is evaluated in service.
_ = now
rows, err := r.sql.QueryContext(ctx, ` rows, err := r.sql.QueryContext(ctx, `
WITH candidates AS ( WITH eligible AS (
SELECT id, credentials ->> 'api_key' AS api_key, SELECT id,
extra #>> '{ollama_cloud_usage_snapshot,next_refresh_at}' AS next_refresh_at credentials ->> 'api_key' AS api_key,
last_used_at,
extra -> 'ollama_cloud_usage_snapshot' AS snapshot
FROM accounts FROM accounts
WHERE deleted_at IS NULL WHERE deleted_at IS NULL
AND status = 'active' AND status = 'active'
AND `+ollamaCloudUsageEligibleSQL+` AND `+ollamaCloudUsageEligibleSQL+`
AND jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string' AND jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'
AND extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb AND extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb
), group_activity AS (
SELECT credentials ->> 'api_key' AS api_key,
MAX(last_used_at) AS group_last_used_at
FROM accounts
WHERE deleted_at IS NULL
AND `+ollamaCloudUsageEligibleSQL+`
AND jsonb_typeof(credentials -> 'api_key') = 'string'
GROUP BY credentials ->> 'api_key'
), joined AS (
SELECT e.id, e.api_key, e.snapshot, g.group_last_used_at,
e.snapshot #>> '{status}' AS status,
e.snapshot #>> '{fetched_at}' AS fetched_at,
e.snapshot #>> '{last_attempt_at}' AS last_attempt_at
FROM eligible e
JOIN group_activity g ON g.api_key = e.api_key
), parsed AS MATERIALIZED ( ), parsed AS MATERIALIZED (
SELECT id, api_key, next_refresh_at, SELECT id, api_key, snapshot, group_last_used_at, status,
next_refresh_at ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$' AS rfc3339_shape, `+ollamaCloudUsageParseRFC3339SQL("fetched_at")+` AS parsed_fetched_at,
jsonb_path_query_first_tz( `+ollamaCloudUsageParseRFC3339SQL("last_attempt_at")+` AS parsed_last_attempt_at
to_jsonb(regexp_replace( FROM joined
next_refresh_at, ), candidates AS (
'(\.[0-9]{6})[0-9]+(Z|[+-][0-9]{2}:[0-9]{2})$',
'\1\2'
)),
'$.datetime()', '{}'::jsonb, true
) #>> '{}' AS parsed_next_refresh_at
FROM candidates
), due AS (
SELECT *, SELECT *,
CASE WHEN next_refresh_at IS NULL OR NOT rfc3339_shape OR parsed_next_refresh_at IS NULL THEN 0 ELSE 1 END AS due_class CASE
WHEN snapshot IS NULL OR snapshot = 'null'::jsonb OR status IS NULL
OR status NOT IN ('ok', 'failed', 'unauthorized') THEN 0
WHEN status = 'ok' AND parsed_fetched_at IS NULL THEN 0
WHEN status IN ('failed', 'unauthorized') AND parsed_last_attempt_at IS NULL THEN 0
WHEN status = 'ok'
AND group_last_used_at IS NOT NULL
AND group_last_used_at > parsed_fetched_at::timestamptz THEN 1
WHEN status IN ('failed', 'unauthorized')
AND group_last_used_at IS NOT NULL
AND group_last_used_at > parsed_last_attempt_at::timestamptz THEN 1
ELSE NULL
END AS due_class
FROM parsed FROM parsed
WHERE next_refresh_at IS NULL
OR NOT rfc3339_shape
OR parsed_next_refresh_at IS NULL
OR parsed_next_refresh_at::timestamptz <= $1
), ranked AS ( ), ranked AS (
SELECT *, row_number() OVER ( SELECT id, api_key, group_last_used_at, due_class,
PARTITION BY api_key row_number() OVER (
ORDER BY due_class, PARTITION BY api_key
CASE WHEN rfc3339_shape AND parsed_next_refresh_at IS NOT NULL THEN parsed_next_refresh_at::timestamptz END NULLS FIRST, ORDER BY due_class,
id group_last_used_at NULLS FIRST,
) AS group_rank id
FROM due ) AS group_rank
FROM candidates
WHERE due_class IS NOT NULL
) )
SELECT id SELECT id, group_last_used_at
FROM ranked FROM ranked
WHERE group_rank = 1 WHERE group_rank = 1
ORDER BY due_class, ORDER BY due_class, group_last_used_at NULLS FIRST, id
CASE WHEN rfc3339_shape AND parsed_next_refresh_at IS NOT NULL THEN parsed_next_refresh_at::timestamptz END NULLS FIRST, LIMIT $1
id `, limit)
LIMIT $2
`, now.UTC(), limit)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer func() { _ = rows.Close() }() defer func() { _ = rows.Close() }()
type dueRow struct {
id int64
groupLastUsed *time.Time
}
rowsOut := make([]dueRow, 0, limit)
ids := make([]int64, 0, limit) ids := make([]int64, 0, limit)
for rows.Next() { for rows.Next() {
var id int64 var row dueRow
if err := rows.Scan(&id); err != nil { if err := rows.Scan(&row.id, &row.groupLastUsed); err != nil {
return nil, err return nil, err
} }
ids = append(ids, id) rowsOut = append(rowsOut, row)
ids = append(ids, row.id)
} }
if err := rows.Err(); err != nil { if err := rows.Err(); err != nil {
return nil, err return nil, err
@@ -430,11 +478,26 @@ func (r *accountRepository) ListDueOllamaCloudUsageAccounts(ctx context.Context,
if err != nil { if err != nil {
return nil, err return nil, err
} }
result := make([]service.Account, 0, len(accounts)) byID := make(map[int64]*service.Account, len(accounts))
for _, account := range accounts { for _, account := range accounts {
if account != nil { if account != nil {
result = append(result, *account) byID[account.ID] = account
} }
} }
result := make([]service.Account, 0, len(rowsOut))
for _, row := range rowsOut {
account := byID[row.id]
if account == nil {
continue
}
// Stamp group MAX(last_used_at) for service due evaluation.
if row.groupLastUsed != nil {
ts := row.groupLastUsed.UTC()
account.LastUsedAt = &ts
} else {
account.LastUsedAt = nil
}
result = append(result, *account)
}
return result, nil return result, nil
} }
@@ -23,31 +23,40 @@ func TestListDueOllamaCloudUsageAccountsOrderingLimitAndProxyHydration(t *testin
Username: "user", Password: "pass", Status: service.StatusActive, Username: "user", Password: "pass", Status: service.StatusActive,
}) })
createAccount := func(name, baseURL string, proxyID *int64, nextRefreshAt *time.Time) *service.Account { createAccount := func(name, baseURL string, proxyID *int64, snapshot map[string]any, lastUsed *time.Time) *service.Account {
t.Helper() t.Helper()
extra := map[string]any{ extra := map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture", service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true, service.OllamaCloudUsageAutoRefreshExtraKey: true,
} }
if nextRefreshAt != nil { if snapshot != nil {
extra[service.OllamaCloudUsageSnapshotExtraKey] = map[string]any{ extra[service.OllamaCloudUsageSnapshotExtraKey] = snapshot
"status": service.OllamaCloudUsageStatusOK, "next_refresh_at": nextRefreshAt.UTC().Format(time.RFC3339Nano),
}
} }
return mustCreateAccount(t, tx.Client(), &service.Account{ return mustCreateAccount(t, tx.Client(), &service.Account{
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey, Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": name, "base_url": baseURL}, Credentials: map[string]any{"api_key": name, "base_url": baseURL},
Extra: extra, ProxyID: proxyID, Extra: extra, ProxyID: proxyID, LastUsedAt: lastUsed,
}) })
} }
uppercasePath := createAccount("ollama-uppercase-path", "https://ollama.com/V1", nil, nil) uppercasePath := createAccount("ollama-uppercase-path", "https://ollama.com/V1", nil, nil, nil)
missingSnapshot := createAccount("ollama-due-missing", "HTTPS://WWW.OLLAMA.COM:443/v1", &proxy.ID, nil) missingSnapshot := createAccount("ollama-due-missing", "HTTPS://WWW.OLLAMA.COM:443/v1", &proxy.ID, nil, nil)
oldest := now.Add(-2 * time.Hour) fetched := now.Add(-2 * time.Hour)
due := createAccount("ollama-due-oldest", "https://ollama.com", nil, &oldest) activity := now.Add(-5 * time.Minute)
future := now.Add(time.Minute) due := createAccount("ollama-due-activity", "https://ollama.com", nil, map[string]any{
_ = createAccount("ollama-not-due", "https://ollama.com", nil, &future) "status": service.OllamaCloudUsageStatusOK,
_ = createAccount("ollama-ineligible", "https://ollama.com.evil.test", nil, nil) "fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
}, &activity)
// Success snapshot without newer activity must not be listed.
_ = createAccount("ollama-not-due-idle", "https://ollama.com", nil, map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": now.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
"last_attempt_at": now.Add(-time.Hour).UTC().Format(time.RFC3339Nano),
"next_refresh_at": now.Add(-time.Minute).UTC().Format(time.RFC3339Nano),
}, nil)
_ = createAccount("ollama-ineligible", "https://ollama.com.evil.test", nil, nil, nil)
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, 2) accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, 2)
@@ -55,49 +64,83 @@ func TestListDueOllamaCloudUsageAccountsOrderingLimitAndProxyHydration(t *testin
require.Len(t, accounts, 2) require.Len(t, accounts, 2)
require.Equal(t, missingSnapshot.ID, accounts[0].ID) require.Equal(t, missingSnapshot.ID, accounts[0].ID)
require.Equal(t, due.ID, accounts[1].ID) require.Equal(t, due.ID, accounts[1].ID)
require.NotNil(t, accounts[1].LastUsedAt)
require.WithinDuration(t, activity.UTC(), accounts[1].LastUsedAt.UTC(), time.Second)
require.NotContains(t, accountIDs(accounts), uppercasePath.ID) require.NotContains(t, accountIDs(accounts), uppercasePath.ID)
require.NotNil(t, accounts[0].Proxy) require.NotNil(t, accounts[0].Proxy)
require.Equal(t, proxy.ID, accounts[0].Proxy.ID) require.Equal(t, proxy.ID, accounts[0].Proxy.ID)
require.Equal(t, proxy.URL(), accounts[0].Proxy.URL()) require.Equal(t, proxy.URL(), accounts[0].Proxy.URL())
} }
func TestListDueOllamaCloudUsageAccountsParsesRFC3339NanoAndFailsOpen(t *testing.T) { func TestListDueOllamaCloudUsageAccountsUsesGroupMaxLastUsedAndFailsOpen(t *testing.T) {
ctx := context.Background() ctx := context.Background()
tx := testEntTx(t) tx := testEntTx(t)
repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil) repo := newAccountRepositoryWithSQL(tx.Client(), tx, nil)
now := time.Date(2026, time.July, 22, 14, 0, 0, 0, time.UTC) now := time.Date(2026, time.July, 22, 14, 0, 0, 0, time.UTC)
fetched := now.Add(-30 * time.Minute)
older := now.Add(-10 * time.Minute)
newer := now.Add(-2 * time.Minute)
create := func(name, nextRefreshAt string) *service.Account { leader := mustCreateAccount(t, tx.Client(), &service.Account{
t.Helper() Name: "ollama-group-leader", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
return mustCreateAccount(t, tx.Client(), &service.Account{ Credentials: map[string]any{"api_key": "shared-key", "base_url": "https://ollama.com"},
Name: name, Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey, Extra: map[string]any{
Credentials: map[string]any{"api_key": name, "base_url": "https://ollama.com"}, service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
Extra: map[string]any{ service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture", service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
service.OllamaCloudUsageAutoRefreshExtraKey: true, "status": service.OllamaCloudUsageStatusOK,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{ "fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"status": service.OllamaCloudUsageStatusOK, "next_refresh_at": nextRefreshAt, "last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
}, "next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
}, },
}) },
} LastUsedAt: &older,
})
sevenDigitsOffset := create("ollama-nano-seven", "2026-07-22T11:00:00.1234567-02:00") _ = mustCreateAccount(t, tx.Client(), &service.Account{
eightDigitsOffset := create("ollama-nano-eight", "2026-07-22T11:00:00.12345678+01:00") Name: "ollama-group-sibling", Platform: service.PlatformAnthropic, Type: service.AccountTypeAPIKey,
nineDigitsZ := create("ollama-nano-nine", "2026-07-22T09:00:00.123456789Z") Credentials: map[string]any{"api_key": "shared-key", "base_url": "https://www.ollama.com/v1"},
invalidCalendar := create("ollama-nano-invalid", "2026-02-30T09:00:00.123456789Z") LastUsedAt: &newer,
future := create("ollama-nano-future", "2026-07-22T15:00:00.123456789Z") })
invalid := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-invalid-snapshot", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "invalid-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK, "fetched_at": "2026-02-30T09:00:00.123456789Z",
},
},
})
idle := mustCreateAccount(t, tx.Client(), &service.Account{
Name: "ollama-idle-ok", Platform: service.PlatformOpenAI, Type: service.AccountTypeAPIKey,
Credentials: map[string]any{"api_key": "idle-key", "base_url": "https://ollama.com"},
Extra: map[string]any{
service.OllamaCloudUsageSessionExtraKey: "cipher:wos-session=fixture",
service.OllamaCloudUsageAutoRefreshExtraKey: true,
service.OllamaCloudUsageSnapshotExtraKey: map[string]any{
"status": service.OllamaCloudUsageStatusOK,
"fetched_at": fetched.UTC().Format(time.RFC3339Nano),
"last_attempt_at": fetched.UTC().Format(time.RFC3339Nano),
"next_refresh_at": fetched.Add(time.Hour).UTC().Format(time.RFC3339Nano),
},
},
})
accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, 10) accounts, err := repo.ListDueOllamaCloudUsageAccounts(ctx, now, 10)
require.NoError(t, err, "invalid stored values must not abort the query") require.NoError(t, err, "invalid stored values must not abort the query")
require.Equal(t, []int64{ ids := accountIDs(accounts)
invalidCalendar.ID, require.Contains(t, ids, invalid.ID)
nineDigitsZ.ID, require.Contains(t, ids, leader.ID)
eightDigitsOffset.ID, require.NotContains(t, ids, idle.ID)
sevenDigitsOffset.ID, for _, account := range accounts {
}, accountIDs(accounts)) if account.ID == leader.ID {
require.NotContains(t, accountIDs(accounts), future.ID) require.NotNil(t, account.LastUsedAt)
require.WithinDuration(t, newer.UTC(), account.LastUsedAt.UTC(), time.Second,
"group MAX(last_used_at) must come from the sibling")
}
}
} }
func TestLockAndMergeAccountProbeExtraCoalescesNullableOllamaGroupIdentity(t *testing.T) { func TestLockAndMergeAccountProbeExtraCoalescesNullableOllamaGroupIdentity(t *testing.T) {
@@ -191,9 +191,9 @@ func TestListDueOllamaCloudUsageAccountsFiltersOrdersAndLimits(t *testing.T) {
t.Cleanup(func() { _ = db.Close() }) t.Cleanup(func() { _ = db.Close() })
now := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC) now := time.Date(2026, time.July, 22, 12, 0, 0, 0, time.UTC)
var capturedSQL string var capturedSQL string
mock.ExpectQuery("WITH candidates AS"). mock.ExpectQuery("WITH eligible AS").
WithArgs(now, 20). WithArgs(20).
WillReturnRows(sqlmock.NewRows([]string{"id"})) WillReturnRows(sqlmock.NewRows([]string{"id", "group_last_used_at"}))
repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil) repo := newAccountRepositoryWithSQL(nil, captureQuerySQL{db: db, captured: &capturedSQL}, nil)
accounts, err := repo.ListDueOllamaCloudUsageAccounts(context.Background(), now, 20) accounts, err := repo.ListDueOllamaCloudUsageAccounts(context.Background(), now, 20)
@@ -209,10 +209,12 @@ func TestListDueOllamaCloudUsageAccountsFiltersOrdersAndLimits(t *testing.T) {
ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"), ollamaCloudBaseURLMatchesSQL("credentials ->> 'base_url'"),
"jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'", "jsonb_typeof(extra -> 'ollama_cloud_usage_session') = 'string'",
`extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb`, `extra @> '{"ollama_cloud_usage_auto_refresh": true}'::jsonb`,
"parsed_next_refresh_at::timestamptz <= $1", "MAX(last_used_at) AS group_last_used_at",
"PARTITION BY api_key", "PARTITION BY api_key",
"WHERE group_rank = 1", "WHERE group_rank = 1",
"LIMIT $2", "LIMIT $1",
"group_last_used_at > parsed_fetched_at::timestamptz",
"group_last_used_at > parsed_last_attempt_at::timestamptz",
} { } {
require.Contains(t, normalized, clause) require.Contains(t, normalized, clause)
} }
@@ -114,6 +114,9 @@ func (s *GatewayService) forwardAnthropicAPIKeyPassthroughWithInput(
if resp != nil && resp.Body != nil { if resp != nil && resp.Body != nil {
_ = resp.Body.Close() _ = resp.Body.Close()
} }
if !errors.Is(err, context.Canceled) {
scheduleOllamaCloudUsageActivity(s.deferredService, account)
}
safeErr := sanitizeUpstreamErrorMessage(err.Error()) safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "") setOpsUpstreamError(c, 0, safeErr, "")
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
@@ -369,6 +369,10 @@ func (s *GatewayService) Forward(ctx context.Context, c *gin.Context, account *A
if resp != nil && resp.Body != nil { if resp != nil && resp.Body != nil {
_ = resp.Body.Close() _ = resp.Body.Close()
} }
// Transport attempt left local validation; count Ollama Cloud activity.
if !errors.Is(err, context.Canceled) {
scheduleOllamaCloudUsageActivity(s.deferredService, account)
}
// Ensure the client receives an error response (handlers assume Forward writes on non-failover errors). // Ensure the client receives an error response (handlers assume Forward writes on non-failover errors).
safeErr := sanitizeUpstreamErrorMessage(err.Error()) safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "") setOpsUpstreamError(c, 0, safeErr, "")
@@ -356,6 +356,8 @@ func (s *GatewayService) readUpstreamErrorBody(resp *http.Response) ([]byte, err
} }
func (s *GatewayService) handleErrorResponse(ctx context.Context, resp *http.Response, c *gin.Context, account *Account, requestedModel ...string) (*ForwardResult, error) { func (s *GatewayService) handleErrorResponse(ctx context.Context, resp *http.Response, c *gin.Context, account *Account, requestedModel ...string) (*ForwardResult, error) {
// Upstream returned a non-success HTTP status; count Ollama Cloud activity.
scheduleOllamaCloudUsageActivity(s.deferredService, account)
body, readErr := s.readUpstreamErrorBody(resp) body, readErr := s.readUpstreamErrorBody(resp)
if readErr != nil { if readErr != nil {
// 读取失败时 body 可能被截断,错误分类会基于不完整数据;记录日志以便排查, // 读取失败时 body 可能被截断,错误分类会基于不完整数据;记录日志以便排查,
+159 -8
View File
@@ -36,6 +36,9 @@ const (
ollamaCloudUsageDefaultIntervalMinutes = 60 ollamaCloudUsageDefaultIntervalMinutes = 60
ollamaCloudUsageMinIntervalMinutes = 15 ollamaCloudUsageMinIntervalMinutes = 15
ollamaCloudUsageMaxIntervalMinutes = 24 * 60 ollamaCloudUsageMaxIntervalMinutes = 24 * 60
ollamaCloudUsageDefaultDebounceMinutes = 1
ollamaCloudUsageMinDebounceMinutes = 1
ollamaCloudUsageMaxDebounceMinutes = 60
ollamaCloudUsageCycleInterval = time.Minute ollamaCloudUsageCycleInterval = time.Minute
ollamaCloudUsageManualRefreshInterval = 30 * time.Second ollamaCloudUsageManualRefreshInterval = 30 * time.Second
ollamaCloudUsageRequestTimeout = 15 * time.Second ollamaCloudUsageRequestTimeout = 15 * time.Second
@@ -76,10 +79,15 @@ const (
OllamaCloudUsageStatusFailed = "failed" OllamaCloudUsageStatusFailed = "failed"
) )
// OllamaCloudUsageSettings controls the opt-in periodic refresh runner. // OllamaCloudUsageSettings controls the opt-in request-driven refresh runner.
//
// IntervalMinutes is the max-wait bound: when model requests keep arriving and
// the trailing debounce keeps sliding, a refresh is forced after this long.
// DebounceMinutes is the quiet period after the latest request in a group.
type OllamaCloudUsageSettings struct { type OllamaCloudUsageSettings struct {
Enabled bool `json:"enabled"` Enabled bool `json:"enabled"`
IntervalMinutes int `json:"interval_minutes"` IntervalMinutes int `json:"interval_minutes"` // max wait while requests continue
DebounceMinutes int `json:"debounce_minutes"` // trailing quiet period after last request
} }
// OllamaCloudUsageWindow is a narrow, sanitized view of one official usage window. // OllamaCloudUsageWindow is a narrow, sanitized view of one official usage window.
@@ -114,6 +122,12 @@ type OllamaCloudUsageData struct {
} }
// OllamaCloudUsageSnapshot is the only usage observation persisted in account extra. // OllamaCloudUsageSnapshot is the only usage observation persisted in account extra.
//
// NextRefreshAt remains a persisted compatibility field. For status=ok it is a
// max-wait horizon marker only; automatic success refreshes are driven by model
// request activity (group last_used_at + debounce/max-wait), not by this field
// alone. For failed/unauthorized snapshots it is the failure not-before time
// (Retry-After / exponential backoff) and is enforced as max(activityDue, NextRefreshAt).
type OllamaCloudUsageSnapshot struct { type OllamaCloudUsageSnapshot struct {
Status string `json:"status"` Status string `json:"status"`
Data *OllamaCloudUsageData `json:"data,omitempty"` Data *OllamaCloudUsageData `json:"data,omitempty"`
@@ -168,6 +182,9 @@ func (s *SettingService) GetOllamaCloudUsageSettings(ctx context.Context) (*Olla
if settings.IntervalMinutes == 0 { if settings.IntervalMinutes == 0 {
settings.IntervalMinutes = defaults.IntervalMinutes settings.IntervalMinutes = defaults.IntervalMinutes
} }
if settings.DebounceMinutes == 0 {
settings.DebounceMinutes = defaults.DebounceMinutes
}
normalizeOllamaCloudUsageSettings(&settings) normalizeOllamaCloudUsageSettings(&settings)
return &settings, nil return &settings, nil
} }
@@ -185,6 +202,12 @@ func (s *SettingService) SetOllamaCloudUsageSettings(ctx context.Context, settin
fmt.Sprintf("interval_minutes must be between %d and %d", ollamaCloudUsageMinIntervalMinutes, ollamaCloudUsageMaxIntervalMinutes), fmt.Sprintf("interval_minutes must be between %d and %d", ollamaCloudUsageMinIntervalMinutes, ollamaCloudUsageMaxIntervalMinutes),
) )
} }
if settings.DebounceMinutes < ollamaCloudUsageMinDebounceMinutes || settings.DebounceMinutes > ollamaCloudUsageMaxDebounceMinutes {
return infraerrors.BadRequest(
"INVALID_OLLAMA_CLOUD_USAGE_DEBOUNCE",
fmt.Sprintf("debounce_minutes must be between %d and %d", ollamaCloudUsageMinDebounceMinutes, ollamaCloudUsageMaxDebounceMinutes),
)
}
normalizeOllamaCloudUsageSettings(settings) normalizeOllamaCloudUsageSettings(settings)
data, err := json.Marshal(settings) data, err := json.Marshal(settings)
if err != nil { if err != nil {
@@ -194,7 +217,11 @@ func (s *SettingService) SetOllamaCloudUsageSettings(ctx context.Context, settin
} }
func defaultOllamaCloudUsageSettings() *OllamaCloudUsageSettings { func defaultOllamaCloudUsageSettings() *OllamaCloudUsageSettings {
return &OllamaCloudUsageSettings{Enabled: false, IntervalMinutes: ollamaCloudUsageDefaultIntervalMinutes} return &OllamaCloudUsageSettings{
Enabled: false,
IntervalMinutes: ollamaCloudUsageDefaultIntervalMinutes,
DebounceMinutes: ollamaCloudUsageDefaultDebounceMinutes,
}
} }
func normalizeOllamaCloudUsageSettings(settings *OllamaCloudUsageSettings) { func normalizeOllamaCloudUsageSettings(settings *OllamaCloudUsageSettings) {
@@ -204,6 +231,116 @@ func normalizeOllamaCloudUsageSettings(settings *OllamaCloudUsageSettings) {
if settings.IntervalMinutes > ollamaCloudUsageMaxIntervalMinutes { if settings.IntervalMinutes > ollamaCloudUsageMaxIntervalMinutes {
settings.IntervalMinutes = ollamaCloudUsageMaxIntervalMinutes settings.IntervalMinutes = ollamaCloudUsageMaxIntervalMinutes
} }
if settings.DebounceMinutes <= 0 {
settings.DebounceMinutes = ollamaCloudUsageDefaultDebounceMinutes
}
if settings.DebounceMinutes < ollamaCloudUsageMinDebounceMinutes {
settings.DebounceMinutes = ollamaCloudUsageMinDebounceMinutes
}
if settings.DebounceMinutes > ollamaCloudUsageMaxDebounceMinutes {
settings.DebounceMinutes = ollamaCloudUsageMaxDebounceMinutes
}
}
func ollamaCloudUsageDurations(settings *OllamaCloudUsageSettings) (debounce, maxWait time.Duration) {
normalized := defaultOllamaCloudUsageSettings()
if settings != nil {
*normalized = *settings
}
normalizeOllamaCloudUsageSettings(normalized)
return time.Duration(normalized.DebounceMinutes) * time.Minute,
time.Duration(normalized.IntervalMinutes) * time.Minute
}
// ollamaCloudUsageIsAutoRefreshDue decides whether a configured auto-refresh
// group should fetch now. groupLastUsedAt must be MAX(last_used_at) across the
// exact api_key group so shared multi-platform accounts do not miss activity.
//
// Success: a request must be newer than fetched_at; dueAt = min(lastUsed+debounce, fetchedAt+maxWait).
// Failure: a request must be newer than last_attempt_at; activity due uses the same min formula,
// then dueAt = max(activityDue, next_refresh_at) so Retry-After / exponential backoff win.
// Missing or invalid snapshots fail open to a first fetch.
func ollamaCloudUsageIsAutoRefreshDue(
snapshot *OllamaCloudUsageSnapshot,
groupLastUsedAt *time.Time,
now time.Time,
debounce, maxWait time.Duration,
) bool {
dueAt, ok := ollamaCloudUsageAutoRefreshDueAt(snapshot, groupLastUsedAt, debounce, maxWait)
if !ok {
return false
}
return !now.Before(dueAt)
}
func ollamaCloudUsageAutoRefreshDueAt(
snapshot *OllamaCloudUsageSnapshot,
groupLastUsedAt *time.Time,
debounce, maxWait time.Duration,
) (time.Time, bool) {
if debounce <= 0 {
debounce = time.Duration(ollamaCloudUsageDefaultDebounceMinutes) * time.Minute
}
if maxWait <= 0 {
maxWait = time.Duration(ollamaCloudUsageDefaultIntervalMinutes) * time.Minute
}
if snapshot == nil {
return time.Time{}, true
}
switch snapshot.Status {
case OllamaCloudUsageStatusOK:
if snapshot.FetchedAt == nil || snapshot.FetchedAt.IsZero() {
return time.Time{}, true
}
fetchedAt := snapshot.FetchedAt.UTC()
if groupLastUsedAt == nil || !groupLastUsedAt.After(fetchedAt) {
return time.Time{}, false
}
lastUsed := groupLastUsedAt.UTC()
return minTime(lastUsed.Add(debounce), fetchedAt.Add(maxWait)), true
case OllamaCloudUsageStatusFailed, OllamaCloudUsageStatusUnauthorized:
if snapshot.LastAttemptAt.IsZero() {
return time.Time{}, true
}
lastAttempt := snapshot.LastAttemptAt.UTC()
if groupLastUsedAt == nil || !groupLastUsedAt.After(lastAttempt) {
return time.Time{}, false
}
lastUsed := groupLastUsedAt.UTC()
activityDue := minTime(lastUsed.Add(debounce), lastAttempt.Add(maxWait))
if !snapshot.NextRefreshAt.IsZero() && snapshot.NextRefreshAt.UTC().After(activityDue) {
return snapshot.NextRefreshAt.UTC(), true
}
return activityDue, true
default:
return time.Time{}, true
}
}
// maxOllamaCloudUsageGroupLastUsed returns the newest last_used_at among group members.
func maxOllamaCloudUsageGroupLastUsed(accounts []Account) *time.Time {
var latest *time.Time
for i := range accounts {
candidate := accounts[i].LastUsedAt
if candidate == nil || candidate.IsZero() {
continue
}
if latest == nil || candidate.After(*latest) {
ts := candidate.UTC()
latest = &ts
}
}
return latest
}
// scheduleOllamaCloudUsageActivity records that an Ollama Cloud API-key account
// actually attempted an upstream model request (including 429/5xx/transport errors).
// Local auth/validation failures must not call this. DeferredService dedupes writes.
func scheduleOllamaCloudUsageActivity(deferred *DeferredService, account *Account) {
if deferred == nil || account == nil || !IsOllamaCloudUsageAccount(account) {
return
}
deferred.ScheduleLastUsedUpdate(account.ID)
} }
// OllamaCloudUsageService refreshes the official settings HTML without affecting routing state. // OllamaCloudUsageService refreshes the official settings HTML without affecting routing state.
@@ -531,7 +668,7 @@ func (s *OllamaCloudUsageService) Refresh(ctx context.Context, accountID int64)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if _, err := s.refreshAccount(ctx, accountID, settings.IntervalMinutes, false); err != nil { if _, err := s.refreshAccount(ctx, accountID, settings, false); err != nil {
return nil, err return nil, err
} }
return s.GetState(ctx, accountID) return s.GetState(ctx, accountID)
@@ -561,6 +698,7 @@ func (s *OllamaCloudUsageService) RunDue(ctx context.Context) error {
return ErrOllamaCloudUsageUnavailable return ErrOllamaCloudUsageUnavailable
} }
now := s.currentTime() now := s.currentTime()
debounce, maxWait := ollamaCloudUsageDurations(settings)
accounts, err := writer.ListDueOllamaCloudUsageAccounts(ctx, now, ollamaCloudUsageMaxPerCycle) accounts, err := writer.ListDueOllamaCloudUsageAccounts(ctx, now, ollamaCloudUsageMaxPerCycle)
if err != nil { if err != nil {
return fmt.Errorf("list due Ollama Cloud usage accounts: %w", err) return fmt.Errorf("list due Ollama Cloud usage accounts: %w", err)
@@ -577,13 +715,15 @@ func (s *OllamaCloudUsageService) RunDue(ctx context.Context) error {
continue continue
} }
seenGroups[fingerprint] = struct{}{} seenGroups[fingerprint] = struct{}{}
if snapshot := decodeOllamaCloudUsageSnapshot(account.Extra); snapshot != nil && now.Before(snapshot.NextRefreshAt) { snapshot := decodeOllamaCloudUsageSnapshot(account.Extra)
// ListDue stamps Account.LastUsedAt with the api_key group MAX(last_used_at).
if !ollamaCloudUsageIsAutoRefreshDue(snapshot, account.LastUsedAt, now, debounce, maxWait) {
continue continue
} }
accountID := account.ID accountID := account.ID
expected := account expected := account
group.Go(func() error { group.Go(func() error {
if _, refreshErr := s.refreshAccount(ctx, accountID, settings.IntervalMinutes, true); refreshErr != nil { if _, refreshErr := s.refreshAccount(ctx, accountID, settings, true); refreshErr != nil {
if errors.Is(refreshErr, ErrOllamaCloudUsageIdentityChanged) { if errors.Is(refreshErr, ErrOllamaCloudUsageIdentityChanged) {
if disableErr := writer.DisableOllamaCloudUsageAutoRefresh(ctx, &expected); disableErr != nil { if disableErr := writer.DisableOllamaCloudUsageAutoRefresh(ctx, &expected); disableErr != nil {
logger.LegacyPrintf("service.ollama_cloud_usage", "disable_auto_refresh_failed: account_id=%d err=%v", accountID, disableErr) logger.LegacyPrintf("service.ollama_cloud_usage", "disable_auto_refresh_failed: account_id=%d err=%v", accountID, disableErr)
@@ -598,10 +738,15 @@ func (s *OllamaCloudUsageService) RunDue(ctx context.Context) error {
return group.Wait() return group.Wait()
} }
func (s *OllamaCloudUsageService) refreshAccount(ctx context.Context, accountID int64, intervalMinutes int, requireEnabled bool) (*OllamaCloudUsageSnapshot, error) { func (s *OllamaCloudUsageService) refreshAccount(ctx context.Context, accountID int64, settings *OllamaCloudUsageSettings, requireEnabled bool) (*OllamaCloudUsageSnapshot, error) {
if s == nil || s.accountRepo == nil { if s == nil || s.accountRepo == nil {
return nil, ErrOllamaCloudUsageUnavailable return nil, ErrOllamaCloudUsageUnavailable
} }
if settings == nil {
settings = defaultOllamaCloudUsageSettings()
}
intervalMinutes := settings.IntervalMinutes
debounce, maxWait := ollamaCloudUsageDurations(settings)
anchor, err := s.accountRepo.GetByID(ctx, accountID) anchor, err := s.accountRepo.GetByID(ctx, accountID)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -650,7 +795,13 @@ func (s *OllamaCloudUsageService) refreshAccount(ctx context.Context, accountID
if !account.IsActive() || !ollamaCloudUsageAutoRefreshEnabled(account) { if !account.IsActive() || !ollamaCloudUsageAutoRefreshEnabled(account) {
return nil, nil return nil, nil
} }
if snapshot := decodeOllamaCloudUsageSnapshot(account.Extra); snapshot != nil && s.currentTime().Before(snapshot.NextRefreshAt) { groupLastUsed := account.LastUsedAt
if writer, ok := s.accountRepo.(ollamaCloudUsageRepository); ok {
if siblings, listErr := writer.ListOllamaCloudUsageGroupAccounts(ctx, []*Account{account}); listErr == nil {
groupLastUsed = maxOllamaCloudUsageGroupLastUsed(siblings)
}
}
if !ollamaCloudUsageIsAutoRefreshDue(decodeOllamaCloudUsageSnapshot(account.Extra), groupLastUsed, s.currentTime(), debounce, maxWait) {
return nil, nil return nil, nil
} }
} }
@@ -313,15 +313,87 @@ func TestOllamaCloudUsageSettingsDefaultOffAndValidation(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
require.False(t, settings.Enabled) require.False(t, settings.Enabled)
require.Equal(t, 60, settings.IntervalMinutes) require.Equal(t, 60, settings.IntervalMinutes)
require.Equal(t, 1, settings.DebounceMinutes)
err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 14}) err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 14, DebounceMinutes: 1})
require.Error(t, err) require.Error(t, err)
err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90}) err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90, DebounceMinutes: 0})
require.Error(t, err)
err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90, DebounceMinutes: 61})
require.Error(t, err)
err = settingsService.SetOllamaCloudUsageSettings(context.Background(), &OllamaCloudUsageSettings{Enabled: true, IntervalMinutes: 90, DebounceMinutes: 2})
require.NoError(t, err) require.NoError(t, err)
settings, err = settingsService.GetOllamaCloudUsageSettings(context.Background()) settings, err = settingsService.GetOllamaCloudUsageSettings(context.Background())
require.NoError(t, err) require.NoError(t, err)
require.True(t, settings.Enabled) require.True(t, settings.Enabled)
require.Equal(t, 90, settings.IntervalMinutes) require.Equal(t, 90, settings.IntervalMinutes)
require.Equal(t, 2, settings.DebounceMinutes)
// Legacy JSON without debounce_minutes defaults to 1.
repo.values[SettingKeyOllamaCloudUsageSettings] = `{"enabled":true,"interval_minutes":45}`
settings, err = settingsService.GetOllamaCloudUsageSettings(context.Background())
require.NoError(t, err)
require.Equal(t, 45, settings.IntervalMinutes)
require.Equal(t, 1, settings.DebounceMinutes)
}
func TestOllamaCloudUsageIsAutoRefreshDue(t *testing.T) {
debounce := time.Minute
maxWait := time.Hour
now := time.Date(2026, time.July, 25, 12, 0, 0, 0, time.UTC)
fetched := now.Add(-30 * time.Minute)
ptr := func(ts time.Time) *time.Time { return &ts }
require.True(t, ollamaCloudUsageIsAutoRefreshDue(nil, nil, now, debounce, maxWait), "missing snapshot first due")
require.True(t, ollamaCloudUsageIsAutoRefreshDue(&OllamaCloudUsageSnapshot{Status: "bogus"}, nil, now, debounce, maxWait), "invalid status first due")
okSnap := &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(fetched),
LastAttemptAt: fetched, NextRefreshAt: fetched.Add(maxWait),
}
require.False(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, nil, now, debounce, maxWait), "no request after success")
require.False(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, ptr(fetched), now, debounce, maxWait), "request not after fetched_at")
require.False(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, ptr(now.Add(-30*time.Second)), now, debounce, maxWait), "debounce not elapsed")
require.True(t, ollamaCloudUsageIsAutoRefreshDue(okSnap, ptr(now.Add(-time.Minute)), now, debounce, maxWait), "single request quiet for debounce")
// Continuous requests: last used is now, but max-wait from old fetch forces due.
oldFetched := now.Add(-2 * time.Hour)
oldSnap := &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK, FetchedAt: ptr(oldFetched),
LastAttemptAt: oldFetched, NextRefreshAt: oldFetched.Add(maxWait),
}
require.True(t, ollamaCloudUsageIsAutoRefreshDue(oldSnap, ptr(now), now, debounce, maxWait), "max-wait forces due while requests continue")
// First request after a very old snapshot is immediately due because fetched+maxWait is past.
require.True(t, ollamaCloudUsageIsAutoRefreshDue(oldSnap, ptr(now.Add(-time.Second)), now, debounce, maxWait), "stale snapshot first request immediate")
failSnap := &OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusFailed, FetchedAt: ptr(fetched),
LastAttemptAt: now.Add(-10 * time.Minute), NextRefreshAt: now.Add(20 * time.Minute),
}
require.False(t, ollamaCloudUsageIsAutoRefreshDue(failSnap, nil, now, debounce, maxWait), "failure without new request")
require.False(t, ollamaCloudUsageIsAutoRefreshDue(failSnap, ptr(now.Add(-time.Minute)), now, debounce, maxWait), "failure blocked by backoff")
failSnap.NextRefreshAt = now.Add(-time.Second)
require.True(t, ollamaCloudUsageIsAutoRefreshDue(failSnap, ptr(now.Add(-time.Minute)), now, debounce, maxWait), "failure after backoff with new request")
require.True(t, ollamaCloudUsageIsAutoRefreshDue(&OllamaCloudUsageSnapshot{
Status: OllamaCloudUsageStatusOK, LastAttemptAt: now,
}, nil, now, debounce, maxWait), "ok without fetched_at fails open")
}
func TestScheduleOllamaCloudUsageActivityOnlyForOllama(t *testing.T) {
deferred := NewDeferredService(nil, nil, time.Second)
ollama := ollamaUsageAccount(1)
other := ollamaUsageAccount(2)
other.Credentials["base_url"] = "https://api.openai.com"
scheduleOllamaCloudUsageActivity(deferred, ollama)
scheduleOllamaCloudUsageActivity(deferred, other)
scheduleOllamaCloudUsageActivity(nil, ollama)
_, ok := deferred.lastUsedUpdates.Load(int64(1))
require.True(t, ok)
_, ok = deferred.lastUsedUpdates.Load(int64(2))
require.False(t, ok)
} }
func TestIsOllamaCloudUsageAccountStrictOfficialHost(t *testing.T) { func TestIsOllamaCloudUsageAccountStrictOfficialHost(t *testing.T) {
@@ -51,6 +51,10 @@ func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Cont
if account != nil && account.Platform == PlatformGrok && isGrokContentPolicyRejection(statusCode, responseBody) { if account != nil && account.Platform == PlatformGrok && isGrokContentPolicyRejection(statusCode, responseBody) {
return false return false
} }
// Any non-2xx upstream HTTP response means the model request was actually sent.
if s != nil {
scheduleOllamaCloudUsageActivity(s.deferredService, account)
}
stateCtx, cancel := openAIAccountStateContext(ctx) stateCtx, cancel := openAIAccountStateContext(ctx)
defer cancel() defer cancel()
@@ -124,6 +124,11 @@ func (s *OpenAIGatewayService) handleOpenAIUpstreamTransportError(ctx context.Co
return err return err
} }
// Transport attempt reached the network path; count as Ollama Cloud activity.
if s != nil {
scheduleOllamaCloudUsageActivity(s.deferredService, account)
}
if classifyOpenAITransportError(err).Persistent { if classifyOpenAITransportError(err).Persistent {
s.tempUnscheduleOpenAITransportError(ctx, account, safeErr) s.tempUnscheduleOpenAITransportError(ctx, account, safeErr)
} }
@@ -38,7 +38,7 @@ describe('admin Ollama Cloud usage API', () => {
}) })
it('uses dedicated global settings endpoints', async () => { it('uses dedicated global settings endpoints', async () => {
const settings = { enabled: false, interval_minutes: 60 } const settings = { enabled: false, interval_minutes: 60, debounce_minutes: 1 }
get.mockResolvedValueOnce({ data: settings }) get.mockResolvedValueOnce({ data: settings })
put.mockResolvedValueOnce({ data: settings }) put.mockResolvedValueOnce({ data: settings })
@@ -335,11 +335,13 @@ export default {
}, },
ollamaCloudUsage: { ollamaCloudUsage: {
title: 'Ollama Cloud Usage Refresh', title: 'Ollama Cloud Usage Refresh',
description: 'Periodically refresh official Ollama settings-page usage for individually opted-in accounts. Disabled by default.', description: 'Refresh official Ollama settings-page usage driven by model requests for individually opted-in accounts. Disabled by default. Idle accounts are not polled.',
enabled: 'Enable global automatic refresh', enabled: 'Enable global automatic refresh',
enabledHint: 'Only accounts with a stored browser session and their own automatic refresh switch enabled are refreshed. Manual refresh remains available.', enabledHint: 'Only accounts with a stored browser session and their own automatic refresh switch enabled are refreshed, and only after subsequent model requests. Manual refresh remains available.',
intervalMinutes: 'Refresh interval (minutes)', intervalMinutes: 'Max wait while requests continue (minutes)',
intervalHint: 'Range: 15–1440 minutes. Failures use bounded exponential backoff.', intervalHint: 'Range: 15–1440 minutes. When continuous requests keep sliding the debounce, force a refresh after this wait.',
debounceMinutes: 'Quiet period after last request (minutes)',
debounceHint: 'Range: 1–60 minutes. Refresh after the latest model request has been quiet for this long.',
saved: 'Ollama Cloud usage refresh settings saved', saved: 'Ollama Cloud usage refresh settings saved',
saveFailed: 'Failed to save Ollama Cloud usage refresh settings' saveFailed: 'Failed to save Ollama Cloud usage refresh settings'
}, },
@@ -328,11 +328,13 @@ export default {
}, },
ollamaCloudUsage: { ollamaCloudUsage: {
title: 'Ollama Cloud 用量刷新', title: 'Ollama Cloud 用量刷新',
description: '定期刷新账号在 Ollama 官方设置页展示的用量;默认关闭。', description: '在模型请求驱动下刷新账号在 Ollama 官方设置页展示的用量;默认关闭。无新请求时不会自动抓取。',
enabled: '启用全局自动刷新', enabled: '启用全局自动刷新',
enabledHint: '仅刷新已保存浏览器会话且账号自身也开启自动刷新的账号;手动刷新不受影响。', enabledHint: '仅刷新已保存浏览器会话且账号自身也开启自动刷新的账号;需有后续模型请求才会触发。手动刷新不受影响。',
intervalMinutes: '刷新周期(分钟)', intervalMinutes: '持续请求最长等待(分钟)',
intervalHint: '范围 15–1440 分钟。失败后按有上限的指数退避重试。', intervalHint: '范围 15–1440 分钟。请求持续不断导致 debounce 一直后移时,最晚在此时间强制刷新。',
debounceMinutes: '请求安静等待(分钟)',
debounceHint: '范围 1–60 分钟。最后一次模型请求安静满此时长后再抓取用量。',
saved: 'Ollama Cloud 用量刷新设置已保存', saved: 'Ollama Cloud 用量刷新设置已保存',
saveFailed: '保存 Ollama Cloud 用量刷新设置失败' saveFailed: '保存 Ollama Cloud 用量刷新设置失败'
}, },
+3
View File
@@ -1023,7 +1023,10 @@ export interface OllamaCloudUsageState {
export interface OllamaCloudUsageSettings { export interface OllamaCloudUsageSettings {
enabled: boolean enabled: boolean
/** Max wait while model requests keep arriving (minutes). */
interval_minutes: number interval_minutes: number
/** Trailing quiet period after the latest model request (minutes). */
debounce_minutes: number
} }
export interface Account { export interface Account {
+38 -17
View File
@@ -4220,23 +4220,43 @@
data-testid="ollama-cloud-usage-global-enabled" data-testid="ollama-cloud-usage-global-enabled"
/> />
</div> </div>
<div v-if="ollamaCloudUsageForm.enabled" class="border-t border-gray-100 pt-4 dark:border-dark-700"> <div v-if="ollamaCloudUsageForm.enabled" class="space-y-4 border-t border-gray-100 pt-4 dark:border-dark-700">
<label class="mb-2 block text-sm font-medium text-gray-700 dark:text-gray-300" for="ollama-cloud-usage-interval"> <div>
{{ t("admin.settings.ollamaCloudUsage.intervalMinutes") }} <label class="mb-2 block text-sm font-medium text-gray-700 dark:text-gray-300" for="ollama-cloud-usage-debounce">
</label> {{ t("admin.settings.ollamaCloudUsage.debounceMinutes") }}
<input </label>
id="ollama-cloud-usage-interval" <input
v-model.number="ollamaCloudUsageForm.interval_minutes" id="ollama-cloud-usage-debounce"
type="number" v-model.number="ollamaCloudUsageForm.debounce_minutes"
min="15" type="number"
max="1440" min="1"
class="input w-32" max="60"
data-testid="ollama-cloud-usage-global-interval" class="input w-32"
@keydown.enter.prevent="saveOllamaCloudUsageSettings" data-testid="ollama-cloud-usage-global-debounce"
/> @keydown.enter.prevent="saveOllamaCloudUsageSettings"
<p class="mt-1.5 text-xs text-gray-500 dark:text-gray-400"> />
{{ t("admin.settings.ollamaCloudUsage.intervalHint") }} <p class="mt-1.5 text-xs text-gray-500 dark:text-gray-400">
</p> {{ t("admin.settings.ollamaCloudUsage.debounceHint") }}
</p>
</div>
<div>
<label class="mb-2 block text-sm font-medium text-gray-700 dark:text-gray-300" for="ollama-cloud-usage-interval">
{{ t("admin.settings.ollamaCloudUsage.intervalMinutes") }}
</label>
<input
id="ollama-cloud-usage-interval"
v-model.number="ollamaCloudUsageForm.interval_minutes"
type="number"
min="15"
max="1440"
class="input w-32"
data-testid="ollama-cloud-usage-global-interval"
@keydown.enter.prevent="saveOllamaCloudUsageSettings"
/>
<p class="mt-1.5 text-xs text-gray-500 dark:text-gray-400">
{{ t("admin.settings.ollamaCloudUsage.intervalHint") }}
</p>
</div>
</div> </div>
<div class="flex justify-end border-t border-gray-100 pt-4 dark:border-dark-700"> <div class="flex justify-end border-t border-gray-100 pt-4 dark:border-dark-700">
<button <button
@@ -7934,6 +7954,7 @@ const ollamaCloudUsageSaving = ref(false);
const ollamaCloudUsageForm = reactive({ const ollamaCloudUsageForm = reactive({
enabled: false, enabled: false,
interval_minutes: 60, interval_minutes: 60,
debounce_minutes: 1,
}); });
// Overload Cooldown (529) 状态 // Overload Cooldown (529) 状态
@@ -50,6 +50,7 @@ const {
getOllamaCloudUsageSettings: vi.fn().mockResolvedValue({ getOllamaCloudUsageSettings: vi.fn().mockResolvedValue({
enabled: false, enabled: false,
interval_minutes: 60, interval_minutes: 60,
debounce_minutes: 1,
}), }),
updateOllamaCloudUsageSettings: vi.fn().mockImplementation(async (payload) => payload), updateOllamaCloudUsageSettings: vi.fn().mockImplementation(async (payload) => payload),
getGroups: vi.fn(), getGroups: vi.fn(),
@@ -647,6 +648,7 @@ describe("admin SettingsView payment visible method controls", () => {
getOllamaCloudUsageSettings.mockResolvedValue({ getOllamaCloudUsageSettings.mockResolvedValue({
enabled: false, enabled: false,
interval_minutes: 60, interval_minutes: 60,
debounce_minutes: 1,
}); });
updateOllamaCloudUsageSettings.mockImplementation(async (payload) => payload); updateOllamaCloudUsageSettings.mockImplementation(async (payload) => payload);
getGroups.mockResolvedValue([]); getGroups.mockResolvedValue([]);
@@ -1007,6 +1009,7 @@ describe("admin SettingsView payment visible method controls", () => {
expect(card.find('[data-testid="ollama-cloud-usage-global-interval"]').exists()).toBe(false); expect(card.find('[data-testid="ollama-cloud-usage-global-interval"]').exists()).toBe(false);
await card.get('[data-testid="ollama-cloud-usage-global-enabled"]').setValue(true); await card.get('[data-testid="ollama-cloud-usage-global-enabled"]').setValue(true);
await card.get('[data-testid="ollama-cloud-usage-global-debounce"]').setValue(3);
await card.get('[data-testid="ollama-cloud-usage-global-interval"]').setValue(90); await card.get('[data-testid="ollama-cloud-usage-global-interval"]').setValue(90);
await card.get('[data-testid="ollama-cloud-usage-global-save"]').trigger("click"); await card.get('[data-testid="ollama-cloud-usage-global-save"]').trigger("click");
await flushPromises(); await flushPromises();
@@ -1014,6 +1017,7 @@ describe("admin SettingsView payment visible method controls", () => {
expect(updateOllamaCloudUsageSettings).toHaveBeenCalledWith({ expect(updateOllamaCloudUsageSettings).toHaveBeenCalledWith({
enabled: true, enabled: true,
interval_minutes: 90, interval_minutes: 90,
debounce_minutes: 3,
}); });
}); });