diff --git a/backend/internal/service/gemini_chat_completions_compat_service.go b/backend/internal/service/gemini_chat_completions_compat_service.go index 1ad2b487e8..669baff99d 100644 --- a/backend/internal/service/gemini_chat_completions_compat_service.go +++ b/backend/internal/service/gemini_chat_completions_compat_service.go @@ -118,6 +118,9 @@ func (s *GeminiMessagesCompatService) forwardClaudeBodyAsChatCompletions( if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil, err } + if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil { + return nil, failoverErr + } return nil, s.writeChatCompletionsError(c, http.StatusBadGateway, "upstream_error", err.Error()) } requestIDHeader = idHeader diff --git a/backend/internal/service/gemini_messages_compat_service.go b/backend/internal/service/gemini_messages_compat_service.go index 06b3e84306..4717b6404f 100644 --- a/backend/internal/service/gemini_messages_compat_service.go +++ b/backend/internal/service/gemini_messages_compat_service.go @@ -716,6 +716,9 @@ func (s *GeminiMessagesCompatService) Forward(ctx context.Context, c *gin.Contex if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil, err } + if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil { + return nil, failoverErr + } // Local build error: don't retry. if strings.Contains(err.Error(), "missing project_id") { return nil, s.writeClaudeError(c, http.StatusBadRequest, "invalid_request_error", err.Error()) @@ -1245,6 +1248,9 @@ func (s *GeminiMessagesCompatService) ForwardNative(ctx context.Context, c *gin. if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil, err } + if failoverErr := s.handleVertexServiceAccountNotFound(ctx, c, account, err); failoverErr != nil { + return nil, failoverErr + } // Local build error: don't retry. if strings.Contains(err.Error(), "missing project_id") { return nil, s.writeGoogleError(c, http.StatusBadRequest, err.Error()) diff --git a/backend/internal/service/gemini_vertex_token_error.go b/backend/internal/service/gemini_vertex_token_error.go new file mode 100644 index 0000000000..7577dcca5b --- /dev/null +++ b/backend/internal/service/gemini_vertex_token_error.go @@ -0,0 +1,60 @@ +package service + +import ( + "context" + "log/slog" + "net/http" + + "github.com/gin-gonic/gin" +) + +// vertexServiceAccountNotFoundReason 标识换取 SA access token 时 SA 已不存在。 +const vertexServiceAccountNotFoundReason GatewayFailureReason = "vertex_service_account_not_found" + +// geminiVertexCredentialFailoverBody 是 SA 凭据失效时挂在 failover 错误上的 Google 格式错误体， +// 供 handler 在换号耗尽后做透传规则匹配与错误信息提取。 +var geminiVertexCredentialFailoverBody = []byte(`{"error":{"code":401,"message":"Upstream authentication failed","status":"UNAUTHENTICATED"}}`) + +// handleVertexServiceAccountNotFound 处理 Gemini 三条转发路径（原生 / Claude 兼容 / +// Chat Completions 兼容）构造上游请求时换取 SA access token 失败、且 SA 已不存在的情况： +// +// - 账号置为 error 并停止调度，错误信息带上 token 端点的 error_description； +// - 记录凭据获取阶段（stage=account_auth）的 Ops 事件，token 端点的状态码只出现在 Message 中； +// - 返回 *UpstreamFailoverError(401)，由 handler 的 failover 循环换号。 +// +// 不是该类失败时返回 nil，由调用方继续处理。本函数不写响应。 +func (s *GeminiMessagesCompatService) handleVertexServiceAccountNotFound(ctx context.Context, c *gin.Context, account *Account, err error) *UpstreamFailoverError { + tokenErr, ok := vertexServiceAccountNotFoundError(err) + if !ok { + return nil + } + + msg := "Service account token rejected (400): " + tokenErr.Description + stateCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout) + defer cancel() + if s.rateLimitService != nil { + s.rateLimitService.handleAuthError(stateCtx, account, msg) + } else if s.accountRepo != nil { + if setErr := s.accountRepo.SetError(stateCtx, account.ID, msg); setErr != nil { + slog.Warn("account_set_error_failed", "account_id", account.ID, "error", setErr) + } + } + + event := OpsUpstreamErrorEvent{ + Platform: account.Platform, + AccountID: account.ID, + AccountName: account.Name, + Kind: "failover", + Stage: string(GatewayFailureStageAccountAuth), + Scope: string(GatewayFailureScopeAccount), + Reason: string(vertexServiceAccountNotFoundReason), + Message: sanitizeUpstreamErrorMessage(err.Error()), + } + event.ProxyID, event.ProxyName = opsUpstreamProxyAttribution(account) + appendOpsUpstreamError(c, event) + + return &UpstreamFailoverError{ + StatusCode: http.StatusUnauthorized, + ResponseBody: geminiVertexCredentialFailoverBody, + } +} diff --git a/backend/internal/service/gemini_vertex_token_error_test.go b/backend/internal/service/gemini_vertex_token_error_test.go new file mode 100644 index 0000000000..57e98f1da1 --- /dev/null +++ b/backend/internal/service/gemini_vertex_token_error_test.go @@ -0,0 +1,207 @@ +//go:build unit + +package service + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Wei-Shaw/sub2api/internal/config" + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// 换取 SA access token 时 SA 已不存在（token 端点 400 invalid_grant +// "Invalid grant: account not found"）的契约，三条转发路径一致： +// - 账号置为 error 并停止调度，错误信息带 token 端点的 error_description； +// - 不发出上游请求，返回 *UpstreamFailoverError(401) 交给 handler 换号，service 不写响应； +// - Ops 事件记为凭据获取阶段（stage=account_auth）； +// - 其他换 token 失败不改账号状态、不换号，直接回写 502。 +// --------------------------------------------------------------------------- + +const vertexAccountNotFoundTokenBody = `{"error":"invalid_grant","error_description":"Invalid grant: account not found"}` + +func newVertexTokenTestAccount(t *testing.T) *Account { + t.Helper() + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + pemBytes := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + saJSON, err := json.Marshal(map[string]string{ + "type": "service_account", + "project_id": "proj-1", + "private_key_id": "kid-1", + "private_key": string(pemBytes), + "client_email": "svc@proj-1.iam.gserviceaccount.com", + }) + require.NoError(t, err) + return &Account{ + ID: 820, + Platform: PlatformGemini, + Type: AccountTypeServiceAccount, + Credentials: map[string]any{ + "service_account_json": string(saJSON), + "project_id": "proj-1", + "location": "global", + }, + } +} + +func useVertexTokenEndpoint(t *testing.T, status int, body string) { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(server.Close) + original := vertexServiceAccountTokenURL + vertexServiceAccountTokenURL = server.URL + t.Cleanup(func() { vertexServiceAccountTokenURL = original }) +} + +func newVertexTokenErrorService() (*GeminiMessagesCompatService, *geminiCompatHTTPUpstreamStub, *errorPolicyRepoStub) { + repo := &errorPolicyRepoStub{} + httpStub := &geminiCompatHTTPUpstreamStub{} + svc := &GeminiMessagesCompatService{ + accountRepo: repo, + httpUpstream: httpStub, + tokenProvider: NewGeminiTokenProvider(repo, nil, nil), + rateLimitService: NewRateLimitService(repo, nil, &config.Config{}, nil, nil), + cfg: &config.Config{}, + } + return svc, httpStub, repo +} + +func requireVertexAccountNotFoundHandled(t *testing.T, c *gin.Context, err error, httpStub *geminiCompatHTTPUpstreamStub, repo *errorPolicyRepoStub) { + t.Helper() + var failoverErr *UpstreamFailoverError + require.True(t, errors.As(err, &failoverErr), "SA 不存在应换号，got %T: %v", err, err) + require.Equal(t, http.StatusUnauthorized, failoverErr.StatusCode) + require.Equal(t, string(geminiVertexCredentialFailoverBody), string(failoverErr.ResponseBody)) + require.True(t, failoverErr.ShouldRetryNextAccount()) + require.False(t, failoverErr.RetryableOnSameAccount) + + require.Equal(t, 1, repo.setErrCalls, "SA 不存在应把账号置为 error") + require.Equal(t, "Service account token rejected (400): Invalid grant: account not found", repo.lastErrorMsg) + require.Zero(t, httpStub.calls, "换不到 token 时不应发出上游请求") + require.False(t, c.Writer.Written(), "换号场景不应写客户端响应") + + raw, ok := c.Get(OpsUpstreamErrorsKey) + require.True(t, ok, "应记录 ops 上游错误事件") + events := raw.([]*OpsUpstreamErrorEvent) + require.Len(t, events, 1) + require.Equal(t, "failover", events[0].Kind) + require.Equal(t, int64(820), events[0].AccountID) + require.Equal(t, string(GatewayFailureStageAccountAuth), events[0].Stage) + require.Equal(t, string(GatewayFailureScopeAccount), events[0].Scope) + require.Equal(t, string(vertexServiceAccountNotFoundReason), events[0].Reason) + require.Zero(t, events[0].UpstreamStatusCode, "换 token 失败不占用上游状态码") + require.Contains(t, events[0].Message, "service account token request returned 400: Invalid grant: account not found") +} + +func TestExchangeVertexServiceAccountToken_AccountNotFoundIsRecognized(t *testing.T) { + useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody) + key, err := parseVertexServiceAccountKey(newVertexTokenTestAccount(t)) + require.NoError(t, err) + + _, _, err = exchangeVertexServiceAccountToken(contextWithTestTimeout(t), key, "") + + require.EqualError(t, err, "service account token request returned 400: Invalid grant: account not found") + tokenErr, ok := vertexServiceAccountNotFoundError(fmt.Errorf("build request: %w", err)) + require.True(t, ok) + require.Equal(t, "invalid_grant", tokenErr.Code) + require.Equal(t, "Invalid grant: account not found", tokenErr.Description) +} + +func TestVertexServiceAccountNotFoundError_OnlyMatchesAccountNotFound(t *testing.T) { + tests := []struct { + name string + err error + }{ + {"other_invalid_grant", &vertexTokenRequestError{StatusCode: 400, Code: "invalid_grant", Description: "Invalid JWT Signature."}}, + {"not_400", &vertexTokenRequestError{StatusCode: 401, Code: "invalid_grant", Description: "Invalid grant: account not found"}}, + {"other_code", &vertexTokenRequestError{StatusCode: 400, Code: "invalid_request", Description: "Invalid grant: account not found"}}, + {"server_error", &vertexTokenRequestError{StatusCode: 500, Message: "backend error"}}, + {"plain_error", errors.New("service account token request returned 400: Invalid grant: account not found")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, ok := vertexServiceAccountNotFoundError(tt.err) + require.False(t, ok) + }) + } +} + +func TestGeminiForwardNative_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) { + gin.SetMode(gin.TestMode) + useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody) + svc, httpStub, repo := newVertexTokenErrorService() + c, _ := newGeminiNativeTestContext(t) + + result, err := svc.ForwardNative(context.Background(), c, newVertexTokenTestAccount(t), + "gemini-3.8-flash", "streamGenerateContent", true, []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`)) + + require.Nil(t, result) + requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo) +} + +func TestGeminiForward_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) { + gin.SetMode(gin.TestMode) + useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody) + svc, httpStub, repo := newVertexTokenErrorService() + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := []byte(`{"model":"gemini-3.8-flash","max_tokens":16,"messages":[{"role":"user","content":"hello"}]}`) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", bytes.NewReader(body)) + + result, err := svc.Forward(context.Background(), c, newVertexTokenTestAccount(t), body) + + require.Nil(t, result) + requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo) +} + +func TestGeminiForwardAsChatCompletions_VertexServiceAccountNotFoundDisablesAndFailsOver(t *testing.T) { + gin.SetMode(gin.TestMode) + useVertexTokenEndpoint(t, http.StatusBadRequest, vertexAccountNotFoundTokenBody) + svc, httpStub, repo := newVertexTokenErrorService() + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + body := []byte(`{"model":"gemini-3.8-flash","messages":[{"role":"user","content":"hi"}]}`) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", bytes.NewReader(body)) + + result, err := svc.ForwardAsChatCompletions(context.Background(), c, newVertexTokenTestAccount(t), body) + + require.Nil(t, result) + requireVertexAccountNotFoundHandled(t, c, err, httpStub, repo) +} + +func TestGeminiForwardNative_OtherVertexTokenErrorKeepsAccountState(t *testing.T) { + gin.SetMode(gin.TestMode) + useVertexTokenEndpoint(t, http.StatusBadRequest, `{"error":"invalid_grant","error_description":"Invalid JWT Signature."}`) + svc, httpStub, repo := newVertexTokenErrorService() + c, rec := newGeminiNativeTestContext(t) + + result, err := svc.ForwardNative(context.Background(), c, newVertexTokenTestAccount(t), + "gemini-3.8-flash", "generateContent", false, []byte(`{"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`)) + + require.Nil(t, result) + require.Error(t, err) + var failoverErr *UpstreamFailoverError + require.False(t, errors.As(err, &failoverErr), "未识别的换 token 失败不换号") + require.Zero(t, repo.setErrCalls, "未识别的换 token 失败不改账号状态") + require.Zero(t, httpStub.calls) + require.Equal(t, http.StatusBadGateway, rec.Code) + require.True(t, strings.Contains(rec.Body.String(), "Invalid JWT Signature.")) +} diff --git a/backend/internal/service/vertex_service_account.go b/backend/internal/service/vertex_service_account.go index 2a19a50e21..fdc7d41e48 100644 --- a/backend/internal/service/vertex_service_account.go +++ b/backend/internal/service/vertex_service_account.go @@ -39,6 +39,9 @@ var ( vertexAnthropicAlreadyDatedIDPattern = regexp.MustCompile(`^.+@[0-9]{8}$`) ) +// vertexServiceAccountTokenURL 是换取 SA access token 的端点，测试中替换为本地服务。 +var vertexServiceAccountTokenURL = vertexDefaultTokenURL + type vertexServiceAccountKey struct { Type string `json:"type"` ProjectID string `json:"project_id"` @@ -138,7 +141,7 @@ func parseVertexServiceAccountJSON(raw []byte) (*vertexServiceAccountKey, error) return nil, errors.New("service account json missing project_id") } // Always use the well-known Google token endpoint to prevent SSRF via crafted token_uri. - key.TokenURI = vertexDefaultTokenURL + key.TokenURI = vertexServiceAccountTokenURL return &key, nil } @@ -277,7 +280,12 @@ func exchangeVertexServiceAccountToken(ctx context.Context, key *vertexServiceAc if msg == "" { msg = string(bytes.TrimSpace(body)) } - return "", 0, fmt.Errorf("service account token request returned %d: %s", resp.StatusCode, msg) + return "", 0, &vertexTokenRequestError{ + StatusCode: resp.StatusCode, + Code: strings.TrimSpace(parsed.Error), + Description: strings.TrimSpace(parsed.ErrorDesc), + Message: msg, + } } if strings.TrimSpace(parsed.AccessToken) == "" { return "", 0, errors.New("service account token response missing access_token") @@ -292,6 +300,33 @@ func exchangeVertexServiceAccountToken(ctx context.Context, key *vertexServiceAc return parsed.AccessToken, ttl, nil } +// vertexTokenRequestError 是 token 端点对换取 SA access token 请求返回的非 2xx 响应。 +type vertexTokenRequestError struct { + StatusCode int + Code string // 响应的 error 字段，如 invalid_grant + Description string // 响应的 error_description 字段 + Message string +} + +func (e *vertexTokenRequestError) Error() string { + return fmt.Sprintf("service account token request returned %d: %s", e.StatusCode, e.Message) +} + +// vertexServiceAccountNotFoundError 识别 SA 已不存在：token 端点返回 +// 400 {"error":"invalid_grant","error_description":"Invalid grant: account not found"}。 +// 该凭据再也换不到 token，账号需要人工更换凭据。 +func vertexServiceAccountNotFoundError(err error) (*vertexTokenRequestError, bool) { + var tokenErr *vertexTokenRequestError + if !errors.As(err, &tokenErr) { + return nil, false + } + if tokenErr.StatusCode != http.StatusBadRequest || tokenErr.Code != "invalid_grant" || + !strings.Contains(strings.ToLower(tokenErr.Description), "account not found") { + return nil, false + } + return tokenErr, true +} + // buildVertexGeminiAccountURL 按账号构造 Vertex Gemini 请求地址：未配置 project_id 的 API Key 账号 // 走 Express mode 全局端点，其余账号走项目级端点。 func buildVertexGeminiAccountURL(account *Account, model, action string, stream bool) (string, error) {