mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-15 23:22:59 +08:00
fix(gemini): Vertex SA 已不存在时账号置为 error 并换号
- 换取 SA access token 时 token 端点返回 400
{"error":"invalid_grant","error_description":"Invalid grant: account not found"}
表示服务账号已被删除,凭据再也换不到 token;此前三条 Gemini 转发路径把它当作
本地构造请求失败直接回 502,不换号也不改账号状态,账号持续被调度
- 识别该错误后账号置为 error 并停止调度(Service account token rejected (400): …),
记录凭据获取阶段(stage=account_auth)的 Ops 事件,请求交给 handler 换号
- 只匹配 400 + invalid_grant + account not found:invalid_grant 还涵盖时钟偏差、
签名无效等情况,其他换 token 失败保持原行为
- token 端点地址改为包级变量,测试中替换为本地服务
This commit is contained in:
@@ -118,6 +118,9 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions(
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, err
|
||||
}
|
||||
if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil {
|
||||
return nil, failoverErr
|
||||
}
|
||||
return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", err.Error())
|
||||
}
|
||||
requestIDHeader = idHeader
|
||||
|
||||
@@ -716,6 +716,9 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, err
|
||||
}
|
||||
if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil {
|
||||
return nil, failoverErr
|
||||
}
|
||||
// Local build error: don't retry.
|
||||
if strings.Contains(err.Error(), "missing project_id") {
|
||||
return nil, s.writeClaudeError(c, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
@@ -1245,6 +1248,9 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin.
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, err
|
||||
}
|
||||
if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil {
|
||||
return nil, failoverErr
|
||||
}
|
||||
// Local build error: don't retry.
|
||||
if strings.Contains(err.Error(), "missing project_id") {
|
||||
return nil, s.writeGoogleError(c, http.StatusBadRequest, err.Error())
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// vertexServiceAccountNotFoundReason 标识换取 SA access token 时 SA 已不存在。
|
||||
const vertexServiceAccountNotFoundReason GatewayFailureReason = "vertex_service_account_not_found"
|
||||
|
||||
// geminiVertexCredentialFailoverBody 是 SA 凭据失效时挂在 failover 错误上的 Google 格式错误体,
|
||||
// 供 handler 在换号耗尽后做透传规则匹配与错误信息提取。
|
||||
var geminiVertexCredentialFailoverBody = []byte(`{"error":{"code":401,"message":"Upstream authentication failed","status":"UNAUTHENTICATED"}}`)
|
||||
|
||||
// handleVertexServiceAccountNotFound 处理 Gemini 三条转发路径(原生 / Claude 兼容 /
|
||||
// Chat Completions 兼容)构造上游请求时换取 SA access token 失败、且 SA 已不存在的情况:
|
||||
//
|
||||
// - 账号置为 error 并停止调度,错误信息带上 token 端点的 error_description;
|
||||
// - 记录凭据获取阶段(stage=account_auth)的 Ops 事件,token 端点的状态码只出现在 Message 中;
|
||||
// - 返回 *UpstreamFailoverError(401),由 handler 的 failover 循环换号。
|
||||
//
|
||||
// 不是该类失败时返回 nil,由调用方继续处理。本函数不写响应。
|
||||
func (s *GeminiMessagesCompatService) handleVertexServiceAccountNotFound(ctx context.Context, c *gin.Context, account *Account, err error) *UpstreamFailoverError {
|
||||
tokenErr, ok := vertexServiceAccountNotFoundError(err)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
msg := "Service account token rejected (400): " + tokenErr.Description
|
||||
stateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout)
|
||||
defer cancel()
|
||||
if s.rateLimitService != nil {
|
||||
s.rateLimitService.handleAuthError(stateCtx, account, msg)
|
||||
} else if s.accountRepo != nil {
|
||||
if setErr := s.accountRepo.SetError(stateCtx, account.ID, msg); setErr != nil {
|
||||
slog.Warn("account_set_error_failed", "account_id", account.ID, "error", setErr)
|
||||
}
|
||||
}
|
||||
|
||||
event := OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
Kind: "failover",
|
||||
Stage: string(GatewayFailureStageAccountAuth),
|
||||
Scope: string(GatewayFailureScopeAccount),
|
||||
Reason: string(vertexServiceAccountNotFoundReason),
|
||||
Message: sanitizeUpstreamErrorMessage(err.Error()),
|
||||
}
|
||||
event.ProxyID, event.ProxyName = opsUpstreamProxyAttribution(account)
|
||||
appendOpsUpstreamError(c, event)
|
||||
|
||||
return &UpstreamFailoverError{
|
||||
StatusCode: http.StatusUnauthorized,
|
||||
ResponseBody: geminiVertexCredentialFailoverBody,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 换取 SA access token 时 SA 已不存在(token 端点 400 invalid_grant
|
||||
// "Invalid grant: account not found")的契约,三条转发路径一致:
|
||||
// - 账号置为 error 并停止调度,错误信息带 token 端点的 error_description;
|
||||
// - 不发出上游请求,返回 *UpstreamFailoverError(401) 交给 handler 换号,service 不写响应;
|
||||
// - Ops 事件记为凭据获取阶段(stage=account_auth);
|
||||
// - 其他换 token 失败不改账号状态、不换号,直接回写 502。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const vertexAccountNotFoundTokenBody = `{"error":"invalid_grant","error_description":"Invalid grant: account not found"}`
|
||||
|
||||
func newVertexTokenTestAccount(t *testing.T) *Account {
|
||||
t.Helper()
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
require.NoError(t, err)
|
||||
pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)})
|
||||
saJSON, err := json.Marshal(map[string]string{
|
||||
"type": "service_account",
|
||||
"project_id": "proj-1",
|
||||
"private_key_id": "kid-1",
|
||||
"private_key": string(pemBytes),
|
||||
"client_email": "svc@proj-1.iam.gserviceaccount.com",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return &Account{
|
||||
ID: 820,
|
||||
Platform: PlatformGemini,
|
||||
Type: AccountTypeServiceAccount,
|
||||
Credentials: map[string]any{
|
||||
"service_account_json": string(saJSON),
|
||||
"project_id": "proj-1",
|
||||
"location": "global",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func useVertexTokenEndpoint(t *testing.T, status int, body string) {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(server.Close)
|
||||
original := vertexServiceAccountTokenURL
|
||||
vertexServiceAccountTokenURL = server.URL
|
||||
t.Cleanup(func() { vertexServiceAccountTokenURL = original })
|
||||
}
|
||||
|
||||
func newVertexTokenErrorService() (*GeminiMessagesCompatService, *geminiCompatHTTPUpstreamStub, *errorPolicyRepoStub) {
|
||||
repo := &errorPolicyRepoStub{}
|
||||
httpStub := &geminiCompatHTTPUpstreamStub{}
|
||||
svc := &GeminiMessagesCompatService{
|
||||
accountRepo: repo,
|
||||
httpUpstream: httpStub,
|
||||
tokenProvider: NewGeminiTokenProvider(repo, nil, nil),
|
||||
rateLimitService: NewRateLimitService(repo, nil, &config.Config{}, nil, nil),
|
||||
cfg: &config.Config{},
|
||||
}
|
||||
return svc, httpStub, repo
|
||||
}
|
||||
|
||||
func requireVertexAccountNotFoundHandled(t *testing.T, c *gin.Context, err error, httpStub *geminiCompatHTTPUpstreamStub, repo *errorPolicyRepoStub) {
|
||||
t.Helper()
|
||||
var failoverErr *UpstreamFailoverError
|
||||
require.True(t, errors.As(err, &failoverErr), "SA 不存在应换号,got %T: %v", err, err)
|
||||
require.Equal(t, http.StatusUnauthorized, failoverErr.StatusCode)
|
||||
require.Equal(t, string(geminiVertexCredentialFailoverBody), string(failoverErr.ResponseBody))
|
||||
require.True(t, failoverErr.ShouldRetryNextAccount())
|
||||
require.False(t, failoverErr.RetryableOnSameAccount)
|
||||
|
||||
require.Equal(t, 1, repo.setErrCalls, "SA 不存在应把账号置为 error")
|
||||
require.Equal(t, "Service account token rejected (400): Invalid grant: account not found", repo.lastErrorMsg)
|
||||
require.Zero(t, httpStub.calls, "换不到 token 时不应发出上游请求")
|
||||
require.False(t, c.Writer.Written(), "换号场景不应写客户端响应")
|
||||
|
||||
raw, ok := c.Get(OpsUpstreamErrorsKey)
|
||||
require.True(t, ok, "应记录 ops 上游错误事件")
|
||||
events := raw.([]*OpsUpstreamErrorEvent)
|
||||
require.Len(t, events, 1)
|
||||
require.Equal(t, "failover", events[0].Kind)
|
||||
require.Equal(t, int64(820), events[0].AccountID)
|
||||
require.Equal(t, string(GatewayFailureStageAccountAuth), events[0].Stage)
|
||||
require.Equal(t, string(GatewayFailureScopeAccount), events[0].Scope)
|
||||
require.Equal(t, string(vertexServiceAccountNotFoundReason), events[0].Reason)
|
||||
require.Zero(t, events[0].UpstreamStatusCode, "换 token 失败不占用上游状态码")
|
||||
require.Contains(t, events[0].Message, "service account token request returned 400: Invalid grant: account not found")
|
||||
}
|
||||
|
||||
func TestExchangeVertexServiceAccountToken_AccountNotFoundIsRecognized(t *testing.T) {
|
||||
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
|
||||
key, err := parseVertexServiceAccountKey(newVertexTokenTestAccount(t))
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, err = exchangeVertexServiceAccountToken(contextWithTestTimeout(t), key, "")
|
||||
|
||||
require.EqualError(t, err, "service account token request returned 400: Invalid grant: account not found")
|
||||
tokenErr, ok := vertexServiceAccountNotFoundError(fmt.Errorf("build request: %w", err))
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "invalid_grant", tokenErr.Code)
|
||||
require.Equal(t, "Invalid grant: account not found", tokenErr.Description)
|
||||
}
|
||||
|
||||
func TestVertexServiceAccountNotFoundError_OnlyMatchesAccountNotFound(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
}{
|
||||
{"other_invalid_grant", &vertexTokenRequestError{StatusCode: 400, Code: "invalid_grant", Description: "Invalid JWT Signature."}},
|
||||
{"not_400", &vertexTokenRequestError{StatusCode: 401, Code: "invalid_grant", Description: "Invalid grant: account not found"}},
|
||||
{"other_code", &vertexTokenRequestError{StatusCode: 400, Code: "invalid_request", Description: "Invalid grant: account not found"}},
|
||||
{"server_error", &vertexTokenRequestError{StatusCode: 500, Message: "backend error"}},
|
||||
{"plain_error", errors.New("service account token request returned 400: Invalid grant: account not found")},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, ok := vertexServiceAccountNotFoundError(tt.err)
|
||||
require.False(t, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGeminiForwardNative_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
|
||||
svc, httpStub, repo := newVertexTokenErrorService()
|
||||
c, _ := newGeminiNativeTestContext(t)
|
||||
|
||||
result, err := svc.ForwardNative(context.Background(), c, newVertexTokenTestAccount(t),
|
||||
"gemini-3.8-flash", "streamGenerateContent", true, []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`))
|
||||
|
||||
require.Nil(t, result)
|
||||
requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo)
|
||||
}
|
||||
|
||||
func TestGeminiForward_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
|
||||
svc, httpStub, repo := newVertexTokenErrorService()
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
body := []byte(`{"model":"gemini-3.8-flash","max_tokens":16,"messages":[{"role":"user","content":"hello"}]}`)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", bytes.NewReader(body))
|
||||
|
||||
result, err := svc.Forward(context.Background(), c, newVertexTokenTestAccount(t), body)
|
||||
|
||||
require.Nil(t, result)
|
||||
requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo)
|
||||
}
|
||||
|
||||
func TestGeminiForwardAsChatCompletions_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody)
|
||||
svc, httpStub, repo := newVertexTokenErrorService()
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
body := []byte(`{"model":"gemini-3.8-flash","messages":[{"role":"user","content":"hi"}]}`)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body))
|
||||
|
||||
result, err := svc.ForwardAsChatCompletions(context.Background(), c, newVertexTokenTestAccount(t), body)
|
||||
|
||||
require.Nil(t, result)
|
||||
requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo)
|
||||
}
|
||||
|
||||
func TestGeminiForwardNative_OtherVertexTokenErrorKeepsAccountState(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
useVertexTokenEndpoint(t, http.StatusBadRequest, `{"error":"invalid_grant","error_description":"Invalid JWT Signature."}`)
|
||||
svc, httpStub, repo := newVertexTokenErrorService()
|
||||
c, rec := newGeminiNativeTestContext(t)
|
||||
|
||||
result, err := svc.ForwardNative(context.Background(), c, newVertexTokenTestAccount(t),
|
||||
"gemini-3.8-flash", "generateContent", false, []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`))
|
||||
|
||||
require.Nil(t, result)
|
||||
require.Error(t, err)
|
||||
var failoverErr *UpstreamFailoverError
|
||||
require.False(t, errors.As(err, &failoverErr), "未识别的换 token 失败不换号")
|
||||
require.Zero(t, repo.setErrCalls, "未识别的换 token 失败不改账号状态")
|
||||
require.Zero(t, httpStub.calls)
|
||||
require.Equal(t, http.StatusBadGateway, rec.Code)
|
||||
require.True(t, strings.Contains(rec.Body.String(), "Invalid JWT Signature."))
|
||||
}
|
||||
@@ -39,6 +39,9 @@ var (
|
||||
vertexAnthropicAlreadyDatedIDPattern = regexp.MustCompile(`^.+@[0-9]{8}$`)
|
||||
)
|
||||
|
||||
// vertexServiceAccountTokenURL 是换取 SA access token 的端点,测试中替换为本地服务。
|
||||
var vertexServiceAccountTokenURL = vertexDefaultTokenURL
|
||||
|
||||
type vertexServiceAccountKey struct {
|
||||
Type string `json:"type"`
|
||||
ProjectID string `json:"project_id"`
|
||||
@@ -138,7 +141,7 @@ func parseVertexServiceAccountJSON(raw []byte) (*vertexServiceAccountKey, error)
|
||||
return nil, errors.New("service account json missing project_id")
|
||||
}
|
||||
// Always use the well-known Google token endpoint to prevent SSRF via crafted token_uri.
|
||||
key.TokenURI = vertexDefaultTokenURL
|
||||
key.TokenURI = vertexServiceAccountTokenURL
|
||||
return &key, nil
|
||||
}
|
||||
|
||||
@@ -277,7 +280,12 @@ func exchangeVertexServiceAccountToken(ctx context.Context, key *vertexServiceAc
|
||||
if msg == "" {
|
||||
msg = string(bytes.TrimSpace(body))
|
||||
}
|
||||
return "", 0, fmt.Errorf("service account token request returned %d: %s", resp.StatusCode, msg)
|
||||
return "", 0, &vertexTokenRequestError{
|
||||
StatusCode: resp.StatusCode,
|
||||
Code: strings.TrimSpace(parsed.Error),
|
||||
Description: strings.TrimSpace(parsed.ErrorDesc),
|
||||
Message: msg,
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(parsed.AccessToken) == "" {
|
||||
return "", 0, errors.New("service account token response missing access_token")
|
||||
@@ -292,6 +300,33 @@ func exchangeVertexServiceAccountToken(ctx context.Context, key *vertexServiceAc
|
||||
return parsed.AccessToken, ttl, nil
|
||||
}
|
||||
|
||||
// vertexTokenRequestError 是 token 端点对换取 SA access token 请求返回的非 2xx 响应。
|
||||
type vertexTokenRequestError struct {
|
||||
StatusCode int
|
||||
Code string // 响应的 error 字段,如 invalid_grant
|
||||
Description string // 响应的 error_description 字段
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *vertexTokenRequestError) Error() string {
|
||||
return fmt.Sprintf("service account token request returned %d: %s", e.StatusCode, e.Message)
|
||||
}
|
||||
|
||||
// vertexServiceAccountNotFoundError 识别 SA 已不存在:token 端点返回
|
||||
// 400 {"error":"invalid_grant","error_description":"Invalid grant: account not found"}。
|
||||
// 该凭据再也换不到 token,账号需要人工更换凭据。
|
||||
func vertexServiceAccountNotFoundError(err error) (*vertexTokenRequestError, bool) {
|
||||
var tokenErr *vertexTokenRequestError
|
||||
if !errors.As(err, &tokenErr) {
|
||||
return nil, false
|
||||
}
|
||||
if tokenErr.StatusCode != http.StatusBadRequest || tokenErr.Code != "invalid_grant" ||
|
||||
!strings.Contains(strings.ToLower(tokenErr.Description), "account not found") {
|
||||
return nil, false
|
||||
}
|
||||
return tokenErr, true
|
||||
}
|
||||
|
||||
// buildVertexGeminiAccountURL 按账号构造 Vertex Gemini 请求地址:未配置 project_id 的 API Key 账号
|
||||
// 走 Express mode 全局端点,其余账号走项目级端点。
|
||||
func buildVertexGeminiAccountURL(account *Account, model, action string, stream bool) (string, error) {
|
||||
|
||||
Reference in New Issue
Block a user