fix(gemini): Vertex SA 已不存在时账号置为 error 并换号

- 换取 SA access token 时 token 端点返回 400
  {"error":"invalid_grant","error_description":"Invalid grant: account not found"}
  表示服务账号已被删除,凭据再也换不到 token;此前三条 Gemini 转发路径把它当作
  本地构造请求失败直接回 502,不换号也不改账号状态,账号持续被调度
- 识别该错误后账号置为 error 并停止调度(Service account token rejected (400): …),
  记录凭据获取阶段(stage=account_auth)的 Ops 事件,请求交给 handler 换号
- 只匹配 400 + invalid_grant + account not found:invalid_grant 还涵盖时钟偏差、
  签名无效等情况,其他换 token 失败保持原行为
- token 端点地址改为包级变量,测试中替换为本地服务
This commit is contained in:
feeeei
2026-10-14 11:58:37 +08:00
parent 9c2cca9e62
commit 20c4bf5ae5
5 changed files with 313 additions and 2 deletions
@@ -118,6 +118,9 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions(
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil {
return nil, failoverErr
}
return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", err.Error())
}
requestIDHeader = idHeader
@@ -716,6 +716,9 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil {
return nil, failoverErr
}
// Local build error: don't retry.
if strings.Contains(err.Error(), "missing project_id") {
return nil, s.writeClaudeError(c, http.StatusBadRequest, "invalid_request_error", err.Error())
@@ -1245,6 +1248,9 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin.
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return nil, err
}
if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil {
return nil, failoverErr
}
// Local build error: don't retry.
if strings.Contains(err.Error(), "missing project_id") {
return nil, s.writeGoogleError(c, http.StatusBadRequest, err.Error())
@@ -0,0 +1,60 @@
package service
import (
"context"
"log/slog"
"net/http"
"github.com/gin-gonic/gin"
)
// vertexServiceAccountNotFoundReason 标识换取 SA access token 时 SA 已不存在。
const vertexServiceAccountNotFoundReason GatewayFailureReason = "vertex_service_account_not_found"
// geminiVertexCredentialFailoverBody 是 SA 凭据失效时挂在 failover 错误上的 Google 格式错误体,
// 供 handler 在换号耗尽后做透传规则匹配与错误信息提取。
var geminiVertexCredentialFailoverBody = []byte(`{"error":{"code":401,"message":"Upstream authentication failed","status":"UNAUTHENTICATED"}}`)
// handleVertexServiceAccountNotFound 处理 Gemini 三条转发路径(原生 / Claude 兼容 /
// Chat Completions 兼容)构造上游请求时换取 SA access token 失败、且 SA 已不存在的情况:
//
// - 账号置为 error 并停止调度,错误信息带上 token 端点的 error_description;
// - 记录凭据获取阶段(stage=account_auth)的 Ops 事件,token 端点的状态码只出现在 Message 中;
// - 返回 *UpstreamFailoverError(401),由 handler 的 failover 循环换号。
//
// 不是该类失败时返回 nil,由调用方继续处理。本函数不写响应。
func (s *GeminiMessagesCompatService) handleVertexServiceAccountNotFound(ctx context.Context, c *gin.Context, account *Account, err error) *UpstreamFailoverError {
tokenErr, ok := vertexServiceAccountNotFoundError(err)
if !ok {
return nil
}
msg := "Service account token rejected (400): " + tokenErr.Description
stateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout)
defer cancel()
if s.rateLimitService != nil {
s.rateLimitService.handleAuthError(stateCtx, account, msg)
} else if s.accountRepo != nil {
if setErr := s.accountRepo.SetError(stateCtx, account.ID, msg); setErr != nil {
slog.Warn("account_set_error_failed", "account_id", account.ID, "error", setErr)
}
}
event := OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
Kind: "failover",
Stage: string(GatewayFailureStageAccountAuth),
Scope: string(GatewayFailureScopeAccount),
Reason: string(vertexServiceAccountNotFoundReason),
Message: sanitizeUpstreamErrorMessage(err.Error()),
}
event.ProxyID, event.ProxyName = opsUpstreamProxyAttribution(account)
appendOpsUpstreamError(c, event)
return &UpstreamFailoverError{
StatusCode: http.StatusUnauthorized,
ResponseBody: geminiVertexCredentialFailoverBody,
}
}
@@ -0,0 +1,207 @@
//go:build unit
package service
import (
"bytes"
"context"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
// ---------------------------------------------------------------------------
// 换取 SA access token 时 SA 已不存在(token 端点 400 invalid_grant
// "Invalid grant: account not found")的契约,三条转发路径一致:
// - 账号置为 error 并停止调度,错误信息带 token 端点的 error_description;
// - 不发出上游请求,返回 *UpstreamFailoverError(401) 交给 handler 换号,service 不写响应;
// - Ops 事件记为凭据获取阶段(stage=account_auth);
// - 其他换 token 失败不改账号状态、不换号,直接回写 502。
// ---------------------------------------------------------------------------
const vertexAccountNotFoundTokenBody = `{"error":"invalid_grant","error_description":"Invalid grant: account not found"}`
func newVertexTokenTestAccount(t *testing.T) *Account {
t.Helper()
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})
saJSON, err := json.Marshal(map[string]string{
"type": "service_account",
"project_id": "proj-1",
"private_key_id": "kid-1",
"private_key": string(pemBytes),
"client_email": "svc@proj-1.iam.gserviceaccount.com",
})
require.NoError(t, err)
return &Account{
ID: 820,
Platform: PlatformGemini,
Type: AccountTypeServiceAccount,
Credentials: map[string]any{
"service_account_json": string(saJSON),
"project_id": "proj-1",
"location": "global",
},
}
}
func useVertexTokenEndpoint(t *testing.T, status int, body string) {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_, _ = w.Write([]byte(body))
}))
t.Cleanup(server.Close)
original := vertexServiceAccountTokenURL
vertexServiceAccountTokenURL = server.URL
t.Cleanup(func() { vertexServiceAccountTokenURL = original })
}
func newVertexTokenErrorService() (*GeminiMessagesCompatService, *geminiCompatHTTPUpstreamStub, *errorPolicyRepoStub) {
repo := &errorPolicyRepoStub{}
httpStub := &geminiCompatHTTPUpstreamStub{}
svc := &GeminiMessagesCompatService{
accountRepo: repo,
httpUpstream: httpStub,
tokenProvider: NewGeminiTokenProvider(repo, nil, nil),
rateLimitService: NewRateLimitService(repo, nil, &config.Config{}, nil, nil),
cfg: &config.Config{},
}
return svc, httpStub, repo
}
func requireVertexAccountNotFoundHandled(t *testing.T, c *gin.Context, err error, httpStub *geminiCompatHTTPUpstreamStub, repo *errorPolicyRepoStub) {
t.Helper()
var failoverErr *UpstreamFailoverError
require.True(t, errors.As(err, &failoverErr), "SA 不存在应换号,got %T: %v", err, err)
require.Equal(t, http.StatusUnauthorized, failoverErr.StatusCode)
require.Equal(t, string(geminiVertexCredentialFailoverBody), string(failoverErr.ResponseBody))
require.True(t, failoverErr.ShouldRetryNextAccount())
require.False(t, failoverErr.RetryableOnSameAccount)
require.Equal(t, 1, repo.setErrCalls, "SA 不存在应把账号置为 error")
require.Equal(t, "Service account token rejected (400): Invalid grant: account not found", repo.lastErrorMsg)
require.Zero(t, httpStub.calls, "换不到 token 时不应发出上游请求")
require.False(t, c.Writer.Written(), "换号场景不应写客户端响应")
raw, ok := c.Get(OpsUpstreamErrorsKey)
require.True(t, ok, "应记录 ops 上游错误事件")
events := raw.([]*OpsUpstreamErrorEvent)
require.Len(t, events, 1)
require.Equal(t, "failover", events[0].Kind)
require.Equal(t, int64(820), events[0].AccountID)
require.Equal(t, string(GatewayFailureStageAccountAuth), events[0].Stage)
require.Equal(t, string(GatewayFailureScopeAccount), events[0].Scope)
require.Equal(t, string(vertexServiceAccountNotFoundReason), events[0].Reason)
require.Zero(t, events[0].UpstreamStatusCode, "换 token 失败不占用上游状态码")
require.Contains(t, events[0].Message, "service account token request returned 400: Invalid grant: account not found")
}
func TestExchangeVertexServiceAccountToken_AccountNotFoundIsRecognized(t *testing.T) {
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
key, err := parseVertexServiceAccountKey(newVertexTokenTestAccount(t))
require.NoError(t, err)
_, _, err = exchangeVertexServiceAccountToken(contextWithTestTimeout(t), key, "")
require.EqualError(t, err, "service account token request returned 400: Invalid grant: account not found")
tokenErr, ok := vertexServiceAccountNotFoundError(fmt.Errorf("build request: %w", err))
require.True(t, ok)
require.Equal(t, "invalid_grant", tokenErr.Code)
require.Equal(t, "Invalid grant: account not found", tokenErr.Description)
}
func TestVertexServiceAccountNotFoundError_OnlyMatchesAccountNotFound(t *testing.T) {
tests := []struct {
name string
err error
}{
{"other_invalid_grant", &vertexTokenRequestError{StatusCode: 400, Code: "invalid_grant", Description: "Invalid JWT Signature."}},
{"not_400", &vertexTokenRequestError{StatusCode: 401, Code: "invalid_grant", Description: "Invalid grant: account not found"}},
{"other_code", &vertexTokenRequestError{StatusCode: 400, Code: "invalid_request", Description: "Invalid grant: account not found"}},
{"server_error", &vertexTokenRequestError{StatusCode: 500, Message: "backend error"}},
{"plain_error", errors.New("service account token request returned 400: Invalid grant: account not found")},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, ok := vertexServiceAccountNotFoundError(tt.err)
require.False(t, ok)
})
}
}
func TestGeminiForwardNative_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) {
gin.SetMode(gin.TestMode)
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
svc, httpStub, repo := newVertexTokenErrorService()
c, _ := newGeminiNativeTestContext(t)
result, err := svc.ForwardNative(context.Background(), c, newVertexTokenTestAccount(t),
"gemini-3.8-flash", "streamGenerateContent", true, []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`))
require.Nil(t, result)
requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo)
}
func TestGeminiForward_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) {
gin.SetMode(gin.TestMode)
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
svc, httpStub, repo := newVertexTokenErrorService()
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
body := []byte(`{"model":"gemini-3.8-flash","max_tokens":16,"messages":[{"role":"user","content":"hello"}]}`)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", bytes.NewReader(body))
result, err := svc.Forward(context.Background(), c, newVertexTokenTestAccount(t), body)
require.Nil(t, result)
requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo)
}
func TestGeminiForwardAsChatCompletions_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) {
gin.SetMode(gin.TestMode)
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
svc, httpStub, repo := newVertexTokenErrorService()
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
body := []byte(`{"model":"gemini-3.8-flash","messages":[{"role":"user","content":"hi"}]}`)
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body))
result, err := svc.ForwardAsChatCompletions(context.Background(), c, newVertexTokenTestAccount(t), body)
require.Nil(t, result)
requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo)
}
func TestGeminiForwardNative_OtherVertexTokenErrorKeepsAccountState(t *testing.T) {
gin.SetMode(gin.TestMode)
useVertexTokenEndpoint(t, http.StatusBadRequest, `{"error":"invalid_grant","error_description":"Invalid JWT Signature."}`)
svc, httpStub, repo := newVertexTokenErrorService()
c, rec := newGeminiNativeTestContext(t)
result, err := svc.ForwardNative(context.Background(), c, newVertexTokenTestAccount(t),
"gemini-3.8-flash", "generateContent", false, []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`))
require.Nil(t, result)
require.Error(t, err)
var failoverErr *UpstreamFailoverError
require.False(t, errors.As(err, &failoverErr), "未识别的换 token 失败不换号")
require.Zero(t, repo.setErrCalls, "未识别的换 token 失败不改账号状态")
require.Zero(t, httpStub.calls)
require.Equal(t, http.StatusBadGateway, rec.Code)
require.True(t, strings.Contains(rec.Body.String(), "Invalid JWT Signature."))
}
@@ -39,6 +39,9 @@ var (
vertexAnthropicAlreadyDatedIDPattern = regexp.MustCompile(`^.+@[0-9]{8}$`)
)
// vertexServiceAccountTokenURL 是换取 SA access token 的端点,测试中替换为本地服务。
var vertexServiceAccountTokenURL = vertexDefaultTokenURL
type vertexServiceAccountKey struct {
Type string `json:"type"`
ProjectID string `json:"project_id"`
@@ -138,7 +141,7 @@ func parseVertexServiceAccountJSON(raw []byte) (*vertexServiceAccountKey, error)
return nil, errors.New("service account json missing project_id")
}
// Always use the well-known Google token endpoint to prevent SSRF via crafted token_uri.
key.TokenURI = vertexDefaultTokenURL
key.TokenURI = vertexServiceAccountTokenURL
return &key, nil
}
@@ -277,7 +280,12 @@ func exchangeVertexServiceAccountToken(ctx context.Context, key *vertexServiceAc
if msg == "" {
msg = string(bytes.TrimSpace(body))
}
return "", 0, fmt.Errorf("service account token request returned %d: %s", resp.StatusCode, msg)
return "", 0, &vertexTokenRequestError{
StatusCode: resp.StatusCode,
Code: strings.TrimSpace(parsed.Error),
Description: strings.TrimSpace(parsed.ErrorDesc),
Message: msg,
}
}
if strings.TrimSpace(parsed.AccessToken) == "" {
return "", 0, errors.New("service account token response missing access_token")
@@ -292,6 +300,33 @@ func exchangeVertexServiceAccountToken(ctx context.Context, key *vertexServiceAc
return parsed.AccessToken, ttl, nil
}
// vertexTokenRequestError 是 token 端点对换取 SA access token 请求返回的非 2xx 响应。
type vertexTokenRequestError struct {
StatusCode int
Code string // 响应的 error 字段,如 invalid_grant
Description string // 响应的 error_description 字段
Message string
}
func (e *vertexTokenRequestError) Error() string {
return fmt.Sprintf("service account token request returned %d: %s", e.StatusCode, e.Message)
}
// vertexServiceAccountNotFoundError 识别 SA 已不存在:token 端点返回
// 400 {"error":"invalid_grant","error_description":"Invalid grant: account not found"}。
// 该凭据再也换不到 token,账号需要人工更换凭据。
func vertexServiceAccountNotFoundError(err error) (*vertexTokenRequestError, bool) {
var tokenErr *vertexTokenRequestError
if !errors.As(err, &tokenErr) {
return nil, false
}
if tokenErr.StatusCode != http.StatusBadRequest || tokenErr.Code != "invalid_grant" ||
!strings.Contains(strings.ToLower(tokenErr.Description), "account not found") {
return nil, false
}
return tokenErr, true
}
// buildVertexGeminiAccountURL 按账号构造 Vertex Gemini 请求地址:未配置 project_id 的 API Key 账号
// 走 Express mode 全局端点,其余账号走项目级端点。
func buildVertexGeminiAccountURL(account *Account, model, action string, stream bool) (string, error) {