fix: 过滤入口拒绝日志并强化鉴权边界

This commit is contained in:
benjamin
2026-07-18 00:11:18 +08:00
parent 57914967cb
commit b92bbf0299
101 changed files with 5872 additions and 828 deletions
+3 -2
View File
@@ -103,8 +103,9 @@ func ProvideRouter(
func ProvideHTTPServer(cfg *config.Config, router *gin.Engine) *http.Server {
httpHandler := http.Handler(router)
server := &http.Server{
Addr: cfg.Server.Address(),
Handler: httpHandler,
Addr: cfg.Server.Address(),
Handler: httpHandler,
MaxHeaderBytes: cfg.Server.MaxHeaderBytes,
// ReadHeaderTimeout: 读取请求头的超时时间,防止慢速请求头攻击
ReadHeaderTimeout: time.Duration(cfg.Server.ReadHeaderTimeout) * time.Second,
// IdleTimeout: 空闲连接超时时间,释放不活跃的连接资源
@@ -0,0 +1,126 @@
//go:build unit
package server
import (
"bufio"
"context"
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func ingressTestConfig() *config.Config {
return &config.Config{
Server: config.ServerConfig{
Host: "127.0.0.1",
ReadHeaderTimeout: 1,
IdleTimeout: 5,
MaxHeaderBytes: 8 * 1024,
MaxRequestBodySize: 1024,
},
Gateway: config.GatewayConfig{MaxBodySize: 1024},
}
}
func TestProvideHTTPServerAppliesIngressLimits(t *testing.T) {
srv := ProvideHTTPServer(ingressTestConfig(), gin.New())
require.Equal(t, 8*1024, srv.MaxHeaderBytes)
require.Equal(t, time.Second, srv.ReadHeaderTimeout)
require.Equal(t, 5*time.Second, srv.IdleTimeout)
}
func TestProvideHTTPServerEnablesBoundedH2C(t *testing.T) {
cfg := ingressTestConfig()
cfg.Server.H2C = config.H2CConfig{
Enabled: true,
MaxConcurrentStreams: 25,
IdleTimeout: 30,
MaxReadFrameSize: 64 * 1024,
MaxUploadBufferPerConnection: 1024 * 1024,
MaxUploadBufferPerStream: 256 * 1024,
}
srv := ProvideHTTPServer(cfg, gin.New())
require.NotNil(t, srv.Protocols)
require.True(t, srv.Protocols.UnencryptedHTTP2())
require.True(t, srv.Protocols.HTTP1())
}
func TestHTTPServerRejectsOversizedHTTP1Header(t *testing.T) {
r := gin.New()
r.GET("/", func(c *gin.Context) { c.Status(http.StatusOK) })
srv := ProvideHTTPServer(ingressTestConfig(), r)
addr, stop := serveIngressTestServer(t, srv)
defer stop()
conn, err := net.DialTimeout("tcp", addr, time.Second)
require.NoError(t, err)
defer func() { _ = conn.Close() }()
_ = conn.SetDeadline(time.Now().Add(3 * time.Second))
_, err = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: test\r\nX-Fill: "+strings.Repeat("a", 32*1024)+"\r\n\r\n")
require.NoError(t, err)
resp, err := http.ReadResponse(bufio.NewReader(conn), nil)
require.NoError(t, err)
defer func() { _ = resp.Body.Close() }()
require.Equal(t, http.StatusRequestHeaderFieldsTooLarge, resp.StatusCode)
}
func TestHTTPServerClosesSlowIncompleteHeader(t *testing.T) {
r := gin.New()
r.GET("/", func(c *gin.Context) { c.Status(http.StatusOK) })
srv := ProvideHTTPServer(ingressTestConfig(), r)
addr, stop := serveIngressTestServer(t, srv)
defer stop()
conn, err := net.DialTimeout("tcp", addr, time.Second)
require.NoError(t, err)
defer func() { _ = conn.Close() }()
_, err = io.WriteString(conn, "GET / HTTP/1.1\r\nHost: test\r\nX-Slow:")
require.NoError(t, err)
time.Sleep(1200 * time.Millisecond)
_ = conn.SetReadDeadline(time.Now().Add(time.Second))
_, err = bufio.NewReader(conn).ReadByte()
require.Error(t, err)
}
func TestHTTPServerGlobalBodyLimit(t *testing.T) {
r := gin.New()
r.POST("/", func(c *gin.Context) {
_, err := io.ReadAll(c.Request.Body)
if err != nil {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
c.Status(http.StatusRequestEntityTooLarge)
return
}
}
c.Status(http.StatusOK)
})
srv := ProvideHTTPServer(ingressTestConfig(), r)
req, err := http.NewRequest(http.MethodPost, "/", strings.NewReader(strings.Repeat("x", 1025)))
require.NoError(t, err)
rec := httptest.NewRecorder()
srv.Handler.ServeHTTP(rec, req)
require.Equal(t, http.StatusRequestEntityTooLarge, rec.Code)
}
func serveIngressTestServer(t *testing.T, srv *http.Server) (string, func()) {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
go func() { _ = srv.Serve(ln) }()
return ln.Addr().String(), func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = srv.Shutdown(ctx)
}
}
@@ -15,6 +15,8 @@ import (
"github.com/gin-gonic/gin"
)
const maxAPIKeyAuthorizationHeaderBytes = service.MaxAPIKeyCredentialBytes + 128
// NewAPIKeyAuthMiddleware 创建 API Key 认证中间件
func NewAPIKeyAuthMiddleware(apiKeyService *service.APIKeyService, subscriptionService *service.SubscriptionService, cfg *config.Config) APIKeyAuthMiddleware {
return APIKeyAuthMiddleware(apiKeyAuthWithSubscription(apiKeyService, subscriptionService, cfg))
@@ -32,10 +34,23 @@ func NewAPIKeyAuthMiddleware(apiKeyService *service.APIKeyService, subscriptionS
func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscriptionService *service.SubscriptionService, cfg *config.Config) gin.HandlerFunc {
return func(c *gin.Context) {
// ── 1. 提取 API Key ──────────────────────────────────────────
if rejectInvalidAuthAbuse(c, apiKeyService) {
AbortWithError(c, http.StatusTooManyRequests, "INVALID_AUTH_RATE_LIMITED", "Too many invalid authentication attempts; retry later")
return
}
if apiKeyHeadersTooLarge(c) {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
AbortWithError(c, http.StatusUnauthorized, "INVALID_API_KEY", "Invalid API key")
return
}
queryKey := strings.TrimSpace(c.Query("key"))
queryApiKey := strings.TrimSpace(c.Query("api_key"))
if queryKey != "" || queryApiKey != "" {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectQueryAPIKeyDeprecated)
AbortWithError(c, 400, "api_key_in_query_deprecated", "API key in query parameter is deprecated. Please use Authorization header instead.")
return
}
@@ -56,6 +71,12 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
if apiKeyString == "" {
apiKeyString = c.GetHeader("x-api-key")
}
if len(apiKeyString) > service.MaxAPIKeyCredentialBytes {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
AbortWithError(c, http.StatusUnauthorized, "INVALID_API_KEY", "Invalid API key")
return
}
// 如果x-api-key header中没有,尝试从x-goog-api-key header中提取(Gemini CLI兼容)
if apiKeyString == "" {
@@ -64,6 +85,12 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
// 如果所有header都没有API key
if apiKeyString == "" {
recordInvalidAuthFailure(c, apiKeyService)
if hasAPIKeyCredentialInput(c) {
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
} else {
MarkIngressRejected(c, IngressRejectAPIKeyRequired)
}
AbortWithError(c, 401, "API_KEY_REQUIRED", "API key is required in Authorization header (Bearer scheme), x-api-key header, or x-goog-api-key header")
return
}
@@ -73,9 +100,16 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
apiKey, err := apiKeyService.GetByKey(c.Request.Context(), apiKeyString)
if err != nil {
if errors.Is(err, service.ErrAPIKeyNotFound) {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
AbortWithError(c, 401, "INVALID_API_KEY", "Invalid API key")
return
}
if errors.Is(err, service.ErrAPIKeyAuthOverloaded) {
MarkIngressRejected(c, IngressRejectAPIKeyAuthOverloaded)
AbortWithError(c, http.StatusServiceUnavailable, "API_KEY_AUTH_OVERLOADED", "API key authentication is temporarily unavailable")
return
}
AbortWithError(c, 500, "INTERNAL_ERROR", "Failed to validate API key")
return
}
@@ -90,6 +124,7 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
if !apiKey.IsActive() &&
apiKey.Status != service.StatusAPIKeyExpired &&
apiKey.Status != service.StatusAPIKeyQuotaExhausted {
MarkIngressRejected(c, IngressRejectAPIKeyDisabled)
AbortWithError(c, 401, "API_KEY_DISABLED", "API key is disabled")
return
}
@@ -104,6 +139,7 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
clientIP = "unknown"
}
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonIPRestriction)
MarkIngressRejected(c, IngressRejectIPRestricted)
AbortWithError(c, 403, "ACCESS_DENIED", fmt.Sprintf("Access denied. Your IP is %s", clientIP))
return
}
@@ -117,6 +153,7 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
// 检查用户状态
if !apiKey.User.IsActive() {
MarkIngressRejected(c, IngressRejectUserInactive)
AbortWithError(c, 401, "USER_INACTIVE", "User account is not active")
return
}
@@ -250,6 +287,24 @@ func apiKeyAuthWithSubscription(apiKeyService *service.APIKeyService, subscripti
}
}
func apiKeyHeadersTooLarge(c *gin.Context) bool {
if c == nil {
return false
}
return len(c.GetHeader("Authorization")) > maxAPIKeyAuthorizationHeaderBytes ||
len(c.GetHeader("x-api-key")) > service.MaxAPIKeyCredentialBytes ||
len(c.GetHeader("x-goog-api-key")) > service.MaxAPIKeyCredentialBytes
}
func hasAPIKeyCredentialInput(c *gin.Context) bool {
if c == nil {
return false
}
return c.GetHeader("Authorization") != "" ||
c.GetHeader("x-api-key") != "" ||
c.GetHeader("x-goog-api-key") != ""
}
func isAsyncImageTaskRead(method, path string) bool {
if method != http.MethodGet {
return false
@@ -321,6 +376,11 @@ func abortIfAPIKeyGroupUnavailable(c *gin.Context, apiKey *service.APIKey) bool
return false
}
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
if code == "GROUP_DELETED" {
MarkIngressRejected(c, IngressRejectGroupDeleted)
} else {
MarkIngressRejected(c, IngressRejectGroupDisabled)
}
AbortWithError(c, 403, code, message)
return true
}
@@ -330,6 +390,7 @@ func abortIfAPIKeyGroupNotAllowed(c *gin.Context, apiKey *service.APIKey) bool {
return false
}
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
MarkIngressRejected(c, IngressRejectGroupNotAllowed)
AbortWithError(c, 403, "GROUP_NOT_ALLOWED", "API Key 所属专属分组不再允许当前用户使用")
return true
}
@@ -24,22 +24,53 @@ func APIKeyAuthGoogle(apiKeyService *service.APIKeyService, cfg *config.Config)
// It is intended for Gemini native endpoints (/v1beta) to match Gemini SDK expectations.
func APIKeyAuthWithSubscriptionGoogle(apiKeyService *service.APIKeyService, subscriptionService *service.SubscriptionService, cfg *config.Config) gin.HandlerFunc {
return func(c *gin.Context) {
if rejectInvalidAuthAbuse(c, apiKeyService) {
abortWithGoogleError(c, 429, "Too many invalid authentication attempts; retry later")
return
}
if apiKeyHeadersTooLarge(c) {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
abortWithGoogleError(c, 401, "Invalid API key")
return
}
if v := strings.TrimSpace(c.Query("api_key")); v != "" {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectQueryAPIKeyDeprecated)
abortWithGoogleError(c, 400, "Query parameter api_key is deprecated. Use Authorization header or key instead.")
return
}
apiKeyString := extractAPIKeyForGoogle(c)
if apiKeyString == "" {
recordInvalidAuthFailure(c, apiKeyService)
if hasAPIKeyCredentialInput(c) {
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
} else {
MarkIngressRejected(c, IngressRejectAPIKeyRequired)
}
abortWithGoogleError(c, 401, "API key is required")
return
}
if len(apiKeyString) > service.MaxAPIKeyCredentialBytes {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
abortWithGoogleError(c, 401, "Invalid API key")
return
}
apiKey, err := apiKeyService.GetByKey(c.Request.Context(), apiKeyString)
if err != nil {
if errors.Is(err, service.ErrAPIKeyNotFound) {
recordInvalidAuthFailure(c, apiKeyService)
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
abortWithGoogleError(c, 401, "Invalid API key")
return
}
if errors.Is(err, service.ErrAPIKeyAuthOverloaded) {
MarkIngressRejected(c, IngressRejectAPIKeyAuthOverloaded)
abortWithGoogleError(c, 503, "API key authentication is temporarily unavailable")
return
}
abortWithGoogleError(c, 500, "Failed to validate API key")
return
}
@@ -53,6 +84,7 @@ func APIKeyAuthWithSubscriptionGoogle(apiKeyService *service.APIKeyService, subs
if !apiKey.IsActive() &&
apiKey.Status != service.StatusAPIKeyExpired &&
apiKey.Status != service.StatusAPIKeyQuotaExhausted {
MarkIngressRejected(c, IngressRejectAPIKeyDisabled)
abortWithGoogleError(c, 401, "API key is disabled")
return
}
@@ -66,6 +98,7 @@ func APIKeyAuthWithSubscriptionGoogle(apiKeyService *service.APIKeyService, subs
clientIP = "unknown"
}
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonIPRestriction)
MarkIngressRejected(c, IngressRejectIPRestricted)
abortWithGoogleError(c, 403, fmt.Sprintf("Access denied. Your IP is %s", clientIP))
return
}
@@ -76,17 +109,24 @@ func APIKeyAuthWithSubscriptionGoogle(apiKeyService *service.APIKeyService, subs
return
}
if !apiKey.User.IsActive() {
MarkIngressRejected(c, IngressRejectUserInactive)
abortWithGoogleError(c, 401, "User account is not active")
return
}
if _, message, ok := validateAPIKeyGroupAvailable(apiKey); !ok {
if code, message, ok := validateAPIKeyGroupAvailable(apiKey); !ok {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
if code == "GROUP_DELETED" {
MarkIngressRejected(c, IngressRejectGroupDeleted)
} else {
MarkIngressRejected(c, IngressRejectGroupDisabled)
}
abortWithGoogleError(c, 403, message)
return
}
// 专属分组授权校验:用户对该专属分组的授权被撤销后应拒绝(与主中间件一致,防止越权)。
if !validateAPIKeyGroupAllowed(apiKey) {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
MarkIngressRejected(c, IngressRejectGroupNotAllowed)
abortWithGoogleError(c, 403, "API Key 所属专属分组不再允许当前用户使用")
return
}
@@ -6,6 +6,8 @@ import (
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
@@ -18,6 +20,59 @@ import (
"github.com/stretchr/testify/require"
)
func TestGoogleAPIKeyAuthRejectsOversizedCredentialsBeforeLookup(t *testing.T) {
gin.SetMode(gin.TestMode)
var calls atomic.Int32
repo := fakeAPIKeyRepo{getByKey: func(context.Context, string) (*service.APIKey, error) {
calls.Add(1)
return nil, service.ErrAPIKeyNotFound
}}
cfg := &config.Config{RunMode: config.RunModeSimple}
svc := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
r := gin.New()
var reason IngressRejectReason
var rejected bool
r.Use(func(c *gin.Context) {
c.Next()
reason, rejected = GetIngressRejectReason(c)
})
r.Use(APIKeyAuthGoogle(svc, cfg))
r.GET("/v1beta/models", func(c *gin.Context) { c.Status(http.StatusOK) })
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/v1beta/models", nil)
req.Header.Set("x-goog-api-key", strings.Repeat("x", service.MaxAPIKeyCredentialBytes+1))
r.ServeHTTP(w, req)
require.Equal(t, http.StatusUnauthorized, w.Code)
require.Zero(t, calls.Load())
require.True(t, rejected)
require.Equal(t, IngressRejectInvalidAPIKey, reason)
}
func TestGoogleAPIKeyAuthMarksLookupBulkheadRejection(t *testing.T) {
gin.SetMode(gin.TestMode)
repo := fakeAPIKeyRepo{getByKey: func(context.Context, string) (*service.APIKey, error) {
return nil, service.ErrAPIKeyAuthOverloaded
}}
cfg := &config.Config{RunMode: config.RunModeSimple}
svc := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
r := gin.New()
var reason IngressRejectReason
var rejected bool
r.Use(func(c *gin.Context) {
c.Next()
reason, rejected = GetIngressRejectReason(c)
})
r.Use(APIKeyAuthGoogle(svc, cfg))
r.GET("/v1beta/models", func(c *gin.Context) { c.Status(http.StatusOK) })
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/v1beta/models", nil)
req.Header.Set("x-goog-api-key", "valid-shape")
r.ServeHTTP(w, req)
require.Equal(t, http.StatusServiceUnavailable, w.Code)
require.True(t, rejected)
require.Equal(t, IngressRejectAPIKeyAuthOverloaded, reason)
}
type fakeAPIKeyRepo struct {
getByKey func(ctx context.Context, key string) (*service.APIKey, error)
updateLastUsed func(ctx context.Context, id int64, usedAt time.Time) error
@@ -376,6 +431,12 @@ func TestApiKeyAuthWithSubscriptionGoogle_InvalidKey(t *testing.T) {
return nil, service.ErrAPIKeyNotFound
},
})
var rejectReason IngressRejectReason
var rejected bool
r.Use(func(c *gin.Context) {
c.Next()
rejectReason, rejected = GetIngressRejectReason(c)
})
r.Use(APIKeyAuthWithSubscriptionGoogle(apiKeyService, nil, &config.Config{}))
r.GET("/v1beta/test", func(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) })
@@ -390,6 +451,8 @@ func TestApiKeyAuthWithSubscriptionGoogle_InvalidKey(t *testing.T) {
require.Equal(t, http.StatusUnauthorized, resp.Error.Code)
require.Equal(t, "Invalid API key", resp.Error.Message)
require.Equal(t, "UNAUTHENTICATED", resp.Error.Status)
require.True(t, rejected)
require.Equal(t, IngressRejectInvalidAPIKey, rejectReason)
}
func TestApiKeyAuthWithSubscriptionGoogle_MarksUnavailableGroupBusinessLimited(t *testing.T) {
@@ -422,9 +485,12 @@ func TestApiKeyAuthWithSubscriptionGoogle_MarksUnavailableGroupBusinessLimited(t
r := gin.New()
var markedBusinessLimited bool
var businessLimitedReason string
var rejectReason IngressRejectReason
var rejected bool
r.Use(func(c *gin.Context) {
c.Next()
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
rejectReason, rejected = GetIngressRejectReason(c)
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
businessLimitedReason, _ = v.(string)
}
@@ -452,6 +518,8 @@ func TestApiKeyAuthWithSubscriptionGoogle_MarksUnavailableGroupBusinessLimited(t
require.Equal(t, "API Key 所属分组已删除", resp.Error.Message)
require.True(t, markedBusinessLimited)
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable, businessLimitedReason)
require.True(t, rejected)
require.Equal(t, IngressRejectGroupDeleted, rejectReason)
}
func TestApiKeyAuthWithSubscriptionGoogle_RepoError(t *testing.T) {
@@ -8,6 +8,8 @@ import (
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
@@ -19,6 +21,35 @@ import (
"github.com/stretchr/testify/require"
)
func TestAPIKeyAuthRejectsOversizedCredentialsBeforeLookup(t *testing.T) {
gin.SetMode(gin.TestMode)
var calls atomic.Int32
repo := &stubApiKeyRepo{getByKey: func(context.Context, string) (*service.APIKey, error) {
calls.Add(1)
return nil, service.ErrAPIKeyNotFound
}}
cfg := &config.Config{RunMode: config.RunModeSimple}
svc := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
for _, headers := range []map[string]string{
{"x-api-key": strings.Repeat("x", service.MaxAPIKeyCredentialBytes+1)},
{"Authorization": "Bearer " + strings.Repeat("x", service.MaxAPIKeyCredentialBytes+1)},
{"Authorization": strings.Repeat("x", maxAPIKeyAuthorizationHeaderBytes+1)},
} {
r := gin.New()
r.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(svc, nil, cfg)))
r.GET("/t", func(c *gin.Context) { c.Status(http.StatusOK) })
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/t", nil)
for name, value := range headers {
req.Header.Set(name, value)
}
r.ServeHTTP(w, req)
require.Equal(t, http.StatusUnauthorized, w.Code)
}
require.Zero(t, calls.Load())
}
func TestSimpleModeBypassesQuotaCheck(t *testing.T) {
gin.SetMode(gin.TestMode)
@@ -436,6 +467,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
wantStatus int
wantCode string
wantMarked bool
wantReject IngressRejectReason
}{
{
name: "active group passes",
@@ -460,6 +492,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
wantStatus: http.StatusForbidden,
wantCode: "GROUP_DISABLED",
wantMarked: true,
wantReject: IngressRejectGroupDisabled,
},
{
name: "deleted status group is forbidden",
@@ -473,6 +506,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
wantStatus: http.StatusForbidden,
wantCode: "GROUP_DELETED",
wantMarked: true,
wantReject: IngressRejectGroupDeleted,
},
{
name: "missing group edge is forbidden",
@@ -480,6 +514,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
wantStatus: http.StatusForbidden,
wantCode: "GROUP_DELETED",
wantMarked: true,
wantReject: IngressRejectGroupDeleted,
},
}
@@ -508,9 +543,12 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
router := gin.New()
var markedBusinessLimited bool
var businessLimitedReason string
var rejectReason IngressRejectReason
var rejected bool
router.Use(func(c *gin.Context) {
c.Next()
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
rejectReason, rejected = GetIngressRejectReason(c)
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
businessLimitedReason, _ = v.(string)
}
@@ -530,6 +568,8 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
require.Contains(t, w.Body.String(), tt.wantCode)
}
require.Equal(t, tt.wantMarked, markedBusinessLimited)
require.Equal(t, tt.wantReject != "", rejected)
require.Equal(t, tt.wantReject, rejectReason)
if tt.wantMarked {
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable, businessLimitedReason)
}
@@ -537,6 +577,112 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
}
}
func TestAPIKeyAuthMarksOnlyExpectedIngressRejections(t *testing.T) {
gin.SetMode(gin.TestMode)
tests := []struct {
name string
path string
key string
authHeader string
repoErr error
wantStatus int
wantCode string
wantReason IngressRejectReason
}{
{
name: "query key deprecated",
path: "/t?key=legacy",
wantStatus: http.StatusBadRequest,
wantCode: "api_key_in_query_deprecated",
wantReason: IngressRejectQueryAPIKeyDeprecated,
},
{
name: "missing key",
path: "/t",
wantStatus: http.StatusUnauthorized,
wantCode: "API_KEY_REQUIRED",
wantReason: IngressRejectAPIKeyRequired,
},
{
name: "malformed authorization",
path: "/t",
authHeader: "Basic not-a-bearer-key",
wantStatus: http.StatusUnauthorized,
wantCode: "API_KEY_REQUIRED",
wantReason: IngressRejectInvalidAPIKey,
},
{
name: "oversized key",
path: "/t",
key: strings.Repeat("x", service.MaxAPIKeyCredentialBytes+1),
wantStatus: http.StatusUnauthorized,
wantCode: "INVALID_API_KEY",
wantReason: IngressRejectInvalidAPIKey,
},
{
name: "invalid key",
path: "/t",
key: "invalid",
repoErr: service.ErrAPIKeyNotFound,
wantStatus: http.StatusUnauthorized,
wantCode: "INVALID_API_KEY",
wantReason: IngressRejectInvalidAPIKey,
},
{
name: "repository failure remains operational error",
path: "/t",
key: "valid-shape",
repoErr: errors.New("database unavailable"),
wantStatus: http.StatusInternalServerError,
wantCode: "INTERNAL_ERROR",
},
{
name: "auth lookup bulkhead rejection is an admission rejection",
path: "/t",
key: "valid-shape",
repoErr: service.ErrAPIKeyAuthOverloaded,
wantStatus: http.StatusServiceUnavailable,
wantCode: "API_KEY_AUTH_OVERLOADED",
wantReason: IngressRejectAPIKeyAuthOverloaded,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
repo := &stubApiKeyRepo{getByKey: func(context.Context, string) (*service.APIKey, error) {
return nil, tt.repoErr
}}
cfg := &config.Config{RunMode: config.RunModeSimple}
apiKeyService := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
router := gin.New()
var reason IngressRejectReason
var rejected bool
router.Use(func(c *gin.Context) {
c.Next()
reason, rejected = GetIngressRejectReason(c)
})
router.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(apiKeyService, nil, cfg)))
router.GET("/t", func(c *gin.Context) { c.Status(http.StatusOK) })
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, tt.path, nil)
if tt.key != "" {
req.Header.Set("x-api-key", tt.key)
}
if tt.authHeader != "" {
req.Header.Set("Authorization", tt.authHeader)
}
router.ServeHTTP(w, req)
require.Equal(t, tt.wantStatus, w.Code)
require.Contains(t, w.Body.String(), tt.wantCode)
require.Equal(t, tt.wantReason != "", rejected)
require.Equal(t, tt.wantReason, reason)
})
}
}
func TestAPIKeyAuthSetsOpsFallbackKeyOnEarlyAbort(t *testing.T) {
gin.SetMode(gin.TestMode)
@@ -686,9 +832,12 @@ func TestRequireGroupAssignmentMarksUngroupedKeyBusinessLimited(t *testing.T) {
router := gin.New()
var markedBusinessLimited bool
var businessLimitedReason string
var rejectReason IngressRejectReason
var rejected bool
router.Use(func(c *gin.Context) {
c.Next()
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
rejectReason, rejected = GetIngressRejectReason(c)
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
businessLimitedReason, _ = v.(string)
}
@@ -708,6 +857,8 @@ func TestRequireGroupAssignmentMarksUngroupedKeyBusinessLimited(t *testing.T) {
require.Equal(t, http.StatusForbidden, w.Code)
require.Contains(t, w.Body.String(), "not assigned to any group")
require.True(t, rejected)
require.Equal(t, IngressRejectGroupUnassigned, rejectReason)
require.True(t, markedBusinessLimited)
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned, businessLimitedReason)
}
@@ -822,7 +973,7 @@ func TestAPIKeyAuthIPRestrictionIncludesClientIPForBlacklistDenial(t *testing.T)
requireAPIKeyAuthError(t, w, "ACCESS_DENIED", "Access denied. Your IP is 9.9.9.9")
}
func TestAPIKeyAuthIPRestrictionCanTrustForwardedClientIPForReverseProxy(t *testing.T) {
func TestAPIKeyAuthIPRestrictionUsesConfiguredTrustedProxy(t *testing.T) {
gin.SetMode(gin.TestMode)
user := &service.User{
@@ -855,7 +1006,7 @@ func TestAPIKeyAuthIPRestrictionCanTrustForwardedClientIPForReverseProxy(t *test
cfg.SetTrustForwardedIPForAPIKeyACL(true)
apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg)
router := gin.New()
require.NoError(t, router.SetTrustedProxies(nil))
require.NoError(t, router.SetTrustedProxies([]string{"9.9.9.9"}))
router.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(apiKeyService, nil, cfg)))
router.GET("/t", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"ok": true})
@@ -906,7 +1057,7 @@ func TestAPIKeyAuthIPRestrictionUsesForwardedClientIPInDenialWhenTrusted(t *test
cfg.SetTrustForwardedIPForAPIKeyACL(true)
apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg)
router := gin.New()
require.NoError(t, router.SetTrustedProxies(nil))
require.NoError(t, router.SetTrustedProxies([]string{"9.9.9.9"}))
router.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(apiKeyService, nil, cfg)))
router.GET("/t", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"ok": true})
@@ -24,7 +24,14 @@ func ClientRequestID() gin.HandlerFunc {
}
if v, _ := c.Request.Context().Value(ctxkey.ClientRequestID).(string); strings.TrimSpace(v) != "" {
c.Header(clientRequestIDHeader, strings.TrimSpace(v))
var valid bool
v, valid = normalizeCorrelationID(v)
if !valid {
v = uuid.New().String()
}
c.Header(clientRequestIDHeader, v)
ctx := context.WithValue(c.Request.Context(), ctxkey.ClientRequestID, v)
c.Request = c.Request.WithContext(ctx)
c.Next()
return
}
@@ -4,6 +4,7 @@ import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/pkg/ctxkey"
@@ -30,6 +31,23 @@ func TestClientRequestIDGeneratesAndExposesID(t *testing.T) {
require.Equal(t, w.Body.String(), w.Header().Get(clientRequestIDHeader))
}
func TestClientRequestIDBoundsExistingContextID(t *testing.T) {
gin.SetMode(gin.TestMode)
router := gin.New()
router.Use(ClientRequestID())
router.GET("/", func(c *gin.Context) {
value, _ := c.Request.Context().Value(ctxkey.ClientRequestID).(string)
c.String(http.StatusOK, value)
})
req := httptest.NewRequest(http.MethodGet, "/", nil)
req = req.WithContext(context.WithValue(req.Context(), ctxkey.ClientRequestID, strings.Repeat("x", 200)))
w := httptest.NewRecorder()
router.ServeHTTP(w, req)
require.Len(t, w.Body.String(), 36)
require.NotEqual(t, strings.Repeat("x", maxPersistentRequestIDBytes), w.Body.String())
require.Equal(t, w.Body.String(), w.Header().Get(clientRequestIDHeader))
}
func TestClientRequestIDPreservesExistingContextID(t *testing.T) {
gin.SetMode(gin.TestMode)
router := gin.New()
@@ -0,0 +1,166 @@
package middleware
import (
"math"
"net/netip"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/gin-gonic/gin"
)
// IngressRejectReason identifies expected gateway admission failures that must
// not be treated as operational request errors.
type IngressRejectReason string
const (
IngressRejectQueryAPIKeyDeprecated IngressRejectReason = "query_api_key_deprecated"
IngressRejectAPIKeyRequired IngressRejectReason = "api_key_required"
IngressRejectInvalidAPIKey IngressRejectReason = "invalid_api_key"
IngressRejectAPIKeyDisabled IngressRejectReason = "api_key_disabled"
IngressRejectIPRestricted IngressRejectReason = "ip_restricted"
IngressRejectUserInactive IngressRejectReason = "user_inactive"
IngressRejectGroupDeleted IngressRejectReason = "group_deleted"
IngressRejectGroupDisabled IngressRejectReason = "group_disabled"
IngressRejectGroupNotAllowed IngressRejectReason = "group_not_allowed"
IngressRejectGroupUnassigned IngressRejectReason = "group_unassigned"
IngressRejectInvalidAuthRateLimited IngressRejectReason = "invalid_auth_rate_limited"
IngressRejectAPIKeyAuthOverloaded IngressRejectReason = "api_key_auth_overloaded"
)
const ingressRejectReasonContextKey = "ingress_reject_reason"
type IngressRejectRecorder interface {
RecordIngressReject(reason, routeFamily, protocol, clientIP string, userID, apiKeyID int64)
}
func invalidAuthClientKey(c *gin.Context) string {
return normalizeIngressRejectIP(SecurityClientIP(c))
}
func rejectInvalidAuthAbuse(c *gin.Context, apiKeyService interface {
CheckInvalidAuthAbuse(string) (time.Duration, bool)
}) bool {
if c == nil || apiKeyService == nil {
return false
}
retry, blocked := apiKeyService.CheckInvalidAuthAbuse(invalidAuthClientKey(c))
if !blocked {
return false
}
retrySeconds := int(math.Ceil(retry.Seconds()))
if retrySeconds < 1 {
retrySeconds = 1
}
c.Header("Retry-After", strconv.Itoa(retrySeconds))
MarkIngressRejected(c, IngressRejectInvalidAuthRateLimited)
return true
}
func recordInvalidAuthFailure(c *gin.Context, apiKeyService interface {
RecordInvalidAuthFailure(string)
}) {
if c == nil || apiKeyService == nil {
return
}
apiKeyService.RecordInvalidAuthFailure(invalidAuthClientKey(c))
}
type ingressRejectRecorderHolder struct{ recorder IngressRejectRecorder }
var activeIngressRejectRecorder atomic.Pointer[ingressRejectRecorderHolder]
func SetIngressRejectRecorder(recorder IngressRejectRecorder) {
if recorder == nil {
activeIngressRejectRecorder.Store(nil)
return
}
activeIngressRejectRecorder.Store(&ingressRejectRecorderHolder{recorder: recorder})
}
// MarkIngressRejected marks a request as rejected before gateway admission.
func MarkIngressRejected(c *gin.Context, reason IngressRejectReason) {
if c == nil || reason == "" {
return
}
c.Set(ingressRejectReasonContextKey, reason)
}
// GetIngressRejectReason returns the admission rejection reason, if any.
func GetIngressRejectReason(c *gin.Context) (IngressRejectReason, bool) {
if c == nil {
return "", false
}
value, exists := c.Get(ingressRejectReasonContextKey)
if !exists {
return "", false
}
reason, ok := value.(IngressRejectReason)
return reason, ok && reason != ""
}
func recordIngressReject(c *gin.Context, reason IngressRejectReason) {
holder := activeIngressRejectRecorder.Load()
if holder == nil || holder.recorder == nil || c == nil || c.Request == nil {
return
}
routeFamily, protocol := ingressRejectRoute(c.Request.URL.Path)
clientIP := normalizeIngressRejectIP(SecurityClientIP(c))
var userID, apiKeyID int64
if apiKey, ok := GetAPIKeyFromContext(c); ok && apiKey != nil {
apiKeyID = apiKey.ID
if apiKey.User != nil {
userID = apiKey.User.ID
}
} else if apiKey, ok := GetOpsFallbackAPIKey(c); ok && apiKey != nil {
apiKeyID = apiKey.ID
if apiKey.User != nil {
userID = apiKey.User.ID
}
}
holder.recorder.RecordIngressReject(string(reason), routeFamily, protocol, clientIP, userID, apiKeyID)
}
func normalizeIngressRejectIP(raw string) string {
addr, err := netip.ParseAddr(strings.TrimSpace(raw))
if err != nil {
return "0.0.0.0"
}
addr = addr.Unmap()
if addr.Is6() {
return netip.PrefixFrom(addr, 64).Masked().Addr().String()
}
return addr.String()
}
func ingressRejectRoute(path string) (string, string) {
path = strings.ToLower(strings.TrimSpace(path))
switch {
case strings.HasPrefix(path, "/antigravity/v1beta"):
return "antigravity", "google"
case strings.HasPrefix(path, "/v1beta"):
return "gemini", "google"
case strings.HasPrefix(path, "/backend-api/codex"):
return "codex", "openai"
case strings.HasPrefix(path, "/antigravity"):
return "antigravity", "anthropic"
case strings.Contains(path, "/messages"):
return "messages", "anthropic"
case strings.Contains(path, "/responses"):
return "responses", "openai"
case strings.Contains(path, "/chat/completions"):
return "chat_completions", "openai"
case strings.Contains(path, "/images"):
return "images", "openai"
case strings.Contains(path, "/videos"):
return "videos", "openai"
case strings.Contains(path, "/embeddings"):
return "embeddings", "openai"
case strings.Contains(path, "/models"):
return "models", "openai"
default:
return "other", "gateway"
}
}
@@ -0,0 +1,58 @@
package middleware
import (
"sync"
"time"
)
const (
ingressRejectAccessLogLimit = 20
ingressRejectAccessLogWindow = time.Second
ingressRejectDroppedSummaryPeriod = 30 * time.Second
)
type ingressRejectAccessSampler struct {
mu sync.Mutex
limit int
window time.Duration
summaryPeriod time.Duration
windowStart time.Time
emitted int
dropped uint64
lastSummary time.Time
}
func newIngressRejectAccessSampler(limit int, window, summaryPeriod time.Duration) *ingressRejectAccessSampler {
return &ingressRejectAccessSampler{limit: limit, window: window, summaryPeriod: summaryPeriod}
}
// allow applies one process-wide fixed-window budget. It stores no attacker
// dimensions, so memory remains constant even for rotating keys and addresses.
func (s *ingressRejectAccessSampler) allow(now time.Time) (allowed bool, droppedSummary uint64) {
if s == nil || s.limit <= 0 || s.window <= 0 {
return false, 0
}
s.mu.Lock()
defer s.mu.Unlock()
if s.windowStart.IsZero() || now.Sub(s.windowStart) >= s.window || now.Before(s.windowStart) {
s.windowStart = now
s.emitted = 0
}
if s.emitted < s.limit {
s.emitted++
return true, 0
}
s.dropped++
if s.summaryPeriod > 0 && (s.lastSummary.IsZero() || now.Sub(s.lastSummary) >= s.summaryPeriod) {
droppedSummary = s.dropped
s.dropped = 0
s.lastSummary = now
}
return false, droppedSummary
}
var globalIngressRejectAccessSampler = newIngressRejectAccessSampler(
ingressRejectAccessLogLimit,
ingressRejectAccessLogWindow,
ingressRejectDroppedSummaryPeriod,
)
@@ -0,0 +1,81 @@
package middleware
import (
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func TestIngressRejectAccessSamplerConcurrentGlobalLimit(t *testing.T) {
sampler := newIngressRejectAccessSampler(10, time.Hour, time.Minute)
now := time.Now()
var allowed atomic.Int64
var wg sync.WaitGroup
for i := 0; i < 200; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if ok, _ := sampler.allow(now); ok {
allowed.Add(1)
}
}()
}
wg.Wait()
require.Equal(t, int64(10), allowed.Load())
}
func TestLoggerIngressRejectSamplingIsBoundedAndSummarySkipsOpsSink(t *testing.T) {
gin.SetMode(gin.TestMode)
original := globalIngressRejectAccessSampler
globalIngressRejectAccessSampler = newIngressRejectAccessSampler(2, time.Hour, time.Hour)
t.Cleanup(func() { globalIngressRejectAccessSampler = original })
sink := initMiddlewareTestLogger(t)
router := gin.New()
router.Use(Logger())
router.GET("/v1/messages", func(c *gin.Context) {
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
c.Status(http.StatusUnauthorized)
})
for i := 0; i < 20; i++ {
router.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/v1/messages", nil))
}
var accessEvents, summaries int
for _, event := range sink.list() {
switch event.Message {
case "http request completed":
accessEvents++
case "ingress rejection access logs dropped":
summaries++
if skipped, _ := event.Fields[logger.OpsSystemLogSkipField].(bool); !skipped {
t.Fatalf("dropped summary must skip ops system log sink")
}
}
}
require.Equal(t, 2, accessEvents)
require.Equal(t, 1, summaries)
}
func TestIngressRejectAccessSamplerDroppedSummaryIsLowFrequency(t *testing.T) {
sampler := newIngressRejectAccessSampler(1, time.Hour, time.Second)
now := time.Now()
allowed, summary := sampler.allow(now)
require.True(t, allowed)
require.Zero(t, summary)
allowed, summary = sampler.allow(now.Add(100 * time.Millisecond))
require.False(t, allowed)
require.Equal(t, uint64(1), summary)
allowed, summary = sampler.allow(now.Add(200 * time.Millisecond))
require.False(t, allowed)
require.Zero(t, summary)
allowed, summary = sampler.allow(now.Add(2 * time.Second))
require.False(t, allowed)
require.Equal(t, uint64(2), summary)
}
@@ -0,0 +1,50 @@
package middleware
import (
"net/http"
"net/http/httptest"
"sync"
"testing"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
type ingressRejectRecorderStub struct {
mu sync.Mutex
calls int
clientIP string
}
func (r *ingressRejectRecorderStub) RecordIngressReject(_, _, _, clientIP string, _, _ int64) {
r.mu.Lock()
defer r.mu.Unlock()
r.calls++
r.clientIP = clientIP
}
func TestNormalizeIngressRejectIP(t *testing.T) {
require.Equal(t, "2001:db8:abcd:1234::", normalizeIngressRejectIP("2001:db8:abcd:1234:ffff::1"))
require.Equal(t, "192.0.2.4", normalizeIngressRejectIP("::ffff:192.0.2.4"))
require.Equal(t, "0.0.0.0", normalizeIngressRejectIP("not-an-ip"))
}
func TestLoggerRecordsIngressRejectOnce(t *testing.T) {
gin.SetMode(gin.TestMode)
recorder := &ingressRejectRecorderStub{}
SetIngressRejectRecorder(recorder)
t.Cleanup(func() { SetIngressRejectRecorder(nil) })
router := gin.New()
router.Use(Logger())
router.GET("/v1/messages", func(c *gin.Context) {
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
c.Status(http.StatusUnauthorized)
})
request := httptest.NewRequest(http.MethodGet, "/v1/messages", nil)
request.RemoteAddr = "[2001:db8:abcd:1234:ffff::1]:1234"
router.ServeHTTP(httptest.NewRecorder(), request)
recorder.mu.Lock()
require.Equal(t, 1, recorder.calls)
require.Equal(t, "2001:db8:abcd:1234::", recorder.clientIP)
recorder.mu.Unlock()
}
@@ -0,0 +1,147 @@
//go:build unit
package middleware
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/Wei-Shaw/sub2api/internal/config"
"github.com/Wei-Shaw/sub2api/internal/service"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func invalidAuthAbuseTestConfig(threshold int) *config.Config {
return &config.Config{
RunMode: config.RunModeSimple,
APIKeyAuth: config.APIKeyAuthCacheConfig{InvalidAbuse: config.InvalidAuthAbuseConfig{
Enabled: true, Threshold: threshold, WindowSeconds: 60, BlockSeconds: 60, Capacity: 256,
}},
}
}
func TestAPIKeyAuthInvalidAbuseReturns429BeforeRepository(t *testing.T) {
gin.SetMode(gin.TestMode)
repoCalls := 0
repo := &stubApiKeyRepo{getByKey: func(context.Context, string) (*service.APIKey, error) {
repoCalls++
return nil, service.ErrAPIKeyNotFound
}}
cfg := invalidAuthAbuseTestConfig(3)
svc := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
r := gin.New()
var reason IngressRejectReason
r.Use(func(c *gin.Context) { c.Next(); reason, _ = GetIngressRejectReason(c) })
r.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(svc, nil, cfg)))
r.POST("/v1/messages", func(c *gin.Context) { c.Status(http.StatusOK) })
requests := []*http.Request{
httpRequest(t, "/v1/messages", "", ""),
httpRequest(t, "/v1/messages", "Basic malformed", ""),
httpRequest(t, "/v1/messages", "", "random-invalid-key"),
}
for _, req := range requests {
w := httptest.NewRecorder()
r.ServeHTTP(w, req)
require.NotEqual(t, http.StatusTooManyRequests, w.Code)
}
w := httptest.NewRecorder()
r.ServeHTTP(w, httpRequest(t, "/v1/messages", "", "another-random-key"))
require.Equal(t, http.StatusTooManyRequests, w.Code)
require.Equal(t, "60", w.Header().Get("Retry-After"))
require.Contains(t, w.Body.String(), "INVALID_AUTH_RATE_LIMITED")
require.Equal(t, IngressRejectInvalidAuthRateLimited, reason)
require.Equal(t, 1, repoCalls, "rate-limited request must not reach the repository")
}
func TestGoogleAPIKeyAuthInvalidAbuseReturnsProtocol429(t *testing.T) {
gin.SetMode(gin.TestMode)
repoCalls := 0
repo := fakeAPIKeyRepo{getByKey: func(context.Context, string) (*service.APIKey, error) {
repoCalls++
return nil, service.ErrAPIKeyNotFound
}}
cfg := invalidAuthAbuseTestConfig(2)
svc := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
r := gin.New()
var reason IngressRejectReason
r.Use(func(c *gin.Context) { c.Next(); reason, _ = GetIngressRejectReason(c) })
r.Use(APIKeyAuthGoogle(svc, cfg))
r.POST("/v1beta/models/test:generateContent", func(c *gin.Context) { c.Status(http.StatusOK) })
for _, key := range []string{"random-1", "random-2"} {
w := httptest.NewRecorder()
req := httpRequest(t, "/v1beta/models/test:generateContent", "", key)
req.Header.Del("x-api-key")
req.Header.Set("x-goog-api-key", key)
r.ServeHTTP(w, req)
require.Equal(t, http.StatusUnauthorized, w.Code)
}
w := httptest.NewRecorder()
req := httpRequest(t, "/v1beta/models/test:generateContent", "", "random-3")
req.Header.Del("x-api-key")
req.Header.Set("x-goog-api-key", "random-3")
r.ServeHTTP(w, req)
require.Equal(t, http.StatusTooManyRequests, w.Code)
require.Equal(t, "60", w.Header().Get("Retry-After"))
require.Contains(t, w.Body.String(), "RESOURCE_EXHAUSTED")
require.Equal(t, IngressRejectInvalidAuthRateLimited, reason)
require.Equal(t, 2, repoCalls)
}
func TestInvalidAuthAbuseDoesNotCountValidOrOperationalFailures(t *testing.T) {
gin.SetMode(gin.TestMode)
user := &service.User{ID: 1, Status: service.StatusActive, Role: service.RoleUser, Balance: 1}
repo := &stubApiKeyRepo{getByKey: func(_ context.Context, key string) (*service.APIKey, error) {
switch key {
case "valid-key":
return &service.APIKey{ID: 1, UserID: 1, Key: key, Status: service.StatusActive, User: user}, nil
case "db-error":
return nil, errors.New("database unavailable")
default:
return nil, service.ErrAPIKeyNotFound
}
}}
cfg := invalidAuthAbuseTestConfig(10)
svc := service.NewAPIKeyService(repo, nil, nil, nil, nil, nil, cfg)
r := gin.New()
r.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(svc, nil, cfg)))
r.POST("/t", func(c *gin.Context) { c.Status(http.StatusOK) })
for _, tc := range []struct {
key string
want int
}{{"invalid", 401}, {"valid-key", 200}, {"db-error", 500}, {"db-error", 500}} {
w := httptest.NewRecorder()
r.ServeHTTP(w, httpRequest(t, "/t", "", tc.key))
require.Equal(t, tc.want, w.Code)
}
w := httptest.NewRecorder()
req := httpRequest(t, "/t", "", "")
req.Header.Set("x-goog-api-key", "valid-key")
r.ServeHTTP(w, req)
require.Equal(t, http.StatusOK, w.Code)
require.Equal(t, uint64(1), svc.InvalidAuthAbuseHealth().Recorded)
}
func TestNormalizeIngressRejectIPGroupsIPv6By64(t *testing.T) {
require.Equal(t, "2001:db8:abcd:1234::", normalizeIngressRejectIP("2001:db8:abcd:1234:1111::1"))
require.Equal(t, normalizeIngressRejectIP("2001:db8:abcd:1234:1111::1"), normalizeIngressRejectIP("2001:db8:abcd:1234:ffff::2"))
}
func httpRequest(t *testing.T, path, authorization, apiKey string) *http.Request {
t.Helper()
req := httptest.NewRequest(http.MethodPost, path, nil)
req.RemoteAddr = "203.0.113.10:12345"
if authorization != "" {
req.Header.Set("Authorization", authorization)
}
if apiKey != "" {
req.Header.Set("x-api-key", apiKey)
}
return req
}
@@ -37,6 +37,21 @@ func Logger() gin.HandlerFunc {
accountID, hasAccountID := c.Request.Context().Value(ctxkey.AccountID).(int64)
platform, _ := c.Request.Context().Value(ctxkey.Platform).(string)
model, _ := c.Request.Context().Value(ctxkey.Model).(string)
reason, rejected := GetIngressRejectReason(c)
if rejected {
recordIngressReject(c, reason)
allowed, droppedSummary := globalIngressRejectAccessSampler.allow(endTime)
if droppedSummary > 0 {
logger.FromContext(c.Request.Context()).Info("ingress rejection access logs dropped",
zap.String("component", "http.access"),
zap.Uint64("dropped_count", droppedSummary),
zap.Bool(logger.OpsSystemLogSkipField, true),
)
}
if !allowed {
return
}
}
fields := []zap.Field{
zap.String("component", "http.access"),
@@ -47,6 +62,12 @@ func Logger() gin.HandlerFunc {
zap.String("method", method),
zap.String("path", path),
}
if rejected {
fields = append(fields,
zap.String("ingress_reject_reason", string(reason)),
zap.Bool(logger.OpsSystemLogSkipField, true),
)
}
if hasAccountID && accountID > 0 {
fields = append(fields, zap.Int64("account_id", accountID))
}
@@ -121,6 +121,7 @@ func RequireGroupAssignment(settingService *service.SettingService, writeError G
return
}
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned)
MarkIngressRejected(c, IngressRejectGroupUnassigned)
writeError(c, http.StatusForbidden, "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.")
c.Abort()
}
@@ -112,6 +112,26 @@ func TestRequestLogger_KeepIncomingRequestID(t *testing.T) {
}
}
func TestRequestLoggerBoundsIncomingRequestID(t *testing.T) {
gin.SetMode(gin.TestMode)
r := gin.New()
r.Use(RequestLogger())
r.GET("/t", func(c *gin.Context) {
reqID, _ := c.Request.Context().Value(ctxkey.RequestID).(string)
if len(reqID) != 36 {
t.Fatalf("request_id length=%d", len(reqID))
}
c.Status(http.StatusOK)
})
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/t", nil)
req.Header.Set(requestIDHeader, strings.Repeat("r", 1024))
r.ServeHTTP(w, req)
if got := len(w.Header().Get(requestIDHeader)); got != 36 {
t.Fatalf("response request_id length=%d", got)
}
}
func TestLogger_AccessLogIncludesCoreFields(t *testing.T) {
gin.SetMode(gin.TestMode)
sink := initMiddlewareTestLogger(t)
@@ -180,11 +200,43 @@ func TestLogger_AccessLogIncludesCoreFields(t *testing.T) {
}
}
func TestLogger_AccessLogUsesForwardedClientIP(t *testing.T) {
func TestLogger_IngressRejectRemainsInStandardAccessLog(t *testing.T) {
gin.SetMode(gin.TestMode)
sink := initMiddlewareTestLogger(t)
r := gin.New()
r.Use(Logger())
r.GET("/v1/messages", func(c *gin.Context) {
MarkIngressRejected(c, IngressRejectInvalidAPIKey)
c.Status(http.StatusUnauthorized)
})
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/v1/messages", nil)
r.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized {
t.Fatalf("status=%d", w.Code)
}
events := sink.list()
if len(events) != 1 {
t.Fatalf("events=%d, want 1", len(events))
}
if got := events[0].Fields["ingress_reject_reason"]; got != string(IngressRejectInvalidAPIKey) {
t.Fatalf("ingress_reject_reason=%v", got)
}
if got, _ := events[0].Fields[logger.OpsSystemLogSkipField].(bool); !got {
t.Fatalf("%s must be true", logger.OpsSystemLogSkipField)
}
}
func TestLogger_AccessLogUsesForwardedClientIPFromTrustedProxy(t *testing.T) {
gin.SetMode(gin.TestMode)
sink := initMiddlewareTestLogger(t)
r := gin.New()
if err := r.SetTrustedProxies([]string{"104.23.251.120"}); err != nil {
t.Fatalf("set trusted proxies: %v", err)
}
r.Use(Logger())
r.GET("/api/test", func(c *gin.Context) {
c.Status(http.StatusOK)
@@ -193,7 +245,7 @@ func TestLogger_AccessLogUsesForwardedClientIP(t *testing.T) {
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/test", nil)
req.RemoteAddr = "104.23.251.120:443"
req.Header.Set("CF-Connecting-IP", "203.0.113.42")
req.Header.Set("X-Forwarded-For", "203.0.113.42")
r.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("status=%d", w.Code)
@@ -21,14 +21,15 @@ func RequestLogger() gin.HandlerFunc {
return
}
requestID := strings.TrimSpace(c.GetHeader(requestIDHeader))
if requestID == "" {
requestID, validRequestID := normalizeCorrelationID(c.GetHeader(requestIDHeader))
if !validRequestID {
requestID = uuid.NewString()
}
c.Header(requestIDHeader, requestID)
ctx := context.WithValue(c.Request.Context(), ctxkey.RequestID, requestID)
clientRequestID, _ := ctx.Value(ctxkey.ClientRequestID).(string)
clientRequestID, _ = normalizeCorrelationID(clientRequestID)
requestLogger := logger.With(
zap.String("component", "http"),
@@ -0,0 +1,30 @@
package middleware
import (
"strings"
"unicode/utf8"
)
const (
maxPersistentRequestIDBytes = 64
maxPersistentUserAgentBytes = 512
)
// normalizePersistentText bounds attacker-controlled metadata before it reaches
// logs or database columns while preserving valid UTF-8 content.
func normalizePersistentText(value string, maxBytes int) string {
value = strings.TrimSpace(strings.ToValidUTF8(value, ""))
if maxBytes <= 0 || len(value) <= maxBytes {
return value
}
value = value[:maxBytes]
for !utf8.ValidString(value) {
value = value[:len(value)-1]
}
return value
}
func normalizeCorrelationID(value string) (string, bool) {
value = strings.TrimSpace(strings.ToValidUTF8(value, ""))
return value, value != "" && len(value) <= maxPersistentRequestIDBytes
}
@@ -13,14 +13,15 @@ import (
// SessionBindingContext 全局中间件:将请求的客户端 IP 与 User-Agent 注入
// request context,供 token 签发路径(登录 / 刷新 / OAuth 回调)读取并写入会话绑定,
// 同时作为审计日志、会话绑定校验的统一客户端 IP 来源。
// IP 取值与 API Key IP 限制共用「信任反代传递的客户端 IP」系统开关:
// 开启时信任反代转发头(CF-Connecting-IP / X-Real-IP / X-Forwarded-For),
// 关闭时走 trusted_proxies 解析链,避免不可信头伪造绕过绑定。
// IP 取值与 API Key IP 限制共用 Gin trusted_proxies 解析链;旧设置开关
// 仅为配置兼容保留,不能单独使直连请求的转发头变为可信。
func SessionBindingContext(cfg *config.Config) gin.HandlerFunc {
return func(c *gin.Context) {
userAgent := normalizePersistentText(c.Request.UserAgent(), maxPersistentUserAgentBytes)
c.Request.Header.Set("User-Agent", userAgent)
binding := &service.SessionBinding{
IP: ip.GetSecurityClientIP(c, cfg.TrustForwardedIPForAPIKeyACL()),
UserAgent: c.Request.UserAgent(),
UserAgent: userAgent,
}
c.Request = c.Request.WithContext(service.WithSessionBinding(c.Request.Context(), binding))
c.Next()
@@ -36,7 +37,7 @@ func requestSessionBinding(c *gin.Context) *service.SessionBinding {
}
return &service.SessionBinding{
IP: ip.GetTrustedClientIP(c),
UserAgent: c.Request.UserAgent(),
UserAgent: normalizePersistentText(c.Request.UserAgent(), maxPersistentUserAgentBytes),
}
}
@@ -93,7 +94,7 @@ func enforceSessionBinding(
Method: c.Request.Method,
Path: path,
ClientIP: binding.IP,
UserAgent: c.Request.UserAgent(),
UserAgent: normalizePersistentText(c.Request.UserAgent(), maxPersistentUserAgentBytes),
StatusCode: 401,
})
}
@@ -4,6 +4,7 @@ package middleware
import (
"net/http/httptest"
"strings"
"testing"
"github.com/Wei-Shaw/sub2api/internal/config"
@@ -13,9 +14,7 @@ import (
"github.com/stretchr/testify/require"
)
// 反代场景:RemoteAddr 为 127.0.0.1,真实客户端 IP 在 X-Real-IP 中。
// 会话绑定注入与审计 IP 必须与 API Key IP 限制共用「信任反代传递的客户端 IP」开关语义。
func TestSessionBindingContextHonorsTrustForwardedToggle(t *testing.T) {
func TestSessionBindingContextDoesNotTrustHeadersWithoutTrustedProxy(t *testing.T) {
gin.SetMode(gin.TestMode)
for _, tc := range []struct {
@@ -24,7 +23,7 @@ func TestSessionBindingContextHonorsTrustForwardedToggle(t *testing.T) {
wantIP string
}{
{name: "trust disabled records proxy address", trustForwarded: false, wantIP: "127.0.0.1"},
{name: "trust enabled records forwarded client IP", trustForwarded: true, wantIP: "1.2.3.4"},
{name: "legacy trust toggle cannot bypass trusted proxies", trustForwarded: true, wantIP: "127.0.0.1"},
} {
t.Run(tc.name, func(t *testing.T) {
cfg := &config.Config{}
@@ -54,6 +53,23 @@ func TestSessionBindingContextHonorsTrustForwardedToggle(t *testing.T) {
}
}
func TestSessionBindingContextBoundsPersistedUserAgent(t *testing.T) {
cfg := &config.Config{}
r := gin.New()
r.Use(SessionBindingContext(cfg))
r.GET("/t", func(c *gin.Context) {
binding := service.SessionBindingFromContext(c.Request.Context())
require.Len(t, binding.UserAgent, maxPersistentUserAgentBytes)
require.Equal(t, binding.UserAgent, c.Request.UserAgent())
c.Status(200)
})
w := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/t", nil)
req.Header.Set("User-Agent", strings.Repeat("u", 2048))
r.ServeHTTP(w, req)
require.Equal(t, 200, w.Code)
}
// 未经过 SessionBindingContext 注入时(异常挂载顺序/单测直调),回退 trusted_proxies 链,
// 等价于开关关闭时的历史行为。
func TestSecurityClientIPFallsBackWithoutInjectedBinding(t *testing.T) {
@@ -75,8 +91,6 @@ func TestSecurityClientIPFallsBackWithoutInjectedBinding(t *testing.T) {
require.Equal(t, "9.9.9.9", w.Body.String())
}
// requestSessionBinding 优先取注入值:开关开启时校验哈希必须基于注入的转发 IP 计算,
// 与 token 签发路径取值一致,否则同一客户端会被误判为指纹变化。
func TestRequestSessionBindingPrefersInjectedBinding(t *testing.T) {
gin.SetMode(gin.TestMode)
@@ -84,7 +98,7 @@ func TestRequestSessionBindingPrefersInjectedBinding(t *testing.T) {
cfg.SetTrustForwardedIPForAPIKeyACL(true)
r := gin.New()
require.NoError(t, r.SetTrustedProxies(nil))
require.NoError(t, r.SetTrustedProxies([]string{"127.0.0.1"}))
r.Use(SessionBindingContext(cfg))
r.GET("/t", func(c *gin.Context) {
issued := &service.SessionBinding{IP: "1.2.3.4", UserAgent: "test-agent"}
+2 -1
View File
@@ -35,6 +35,7 @@ func SetupRouter(
cfg *config.Config,
redisClient *redis.Client,
) *gin.Engine {
middleware2.SetIngressRejectRecorder(opsService)
// 缓存 iframe 页面的 origin 列表,用于动态注入 CSP frame-src
var cachedFrameOrigins atomic.Pointer[[]string]
emptyOrigins := []string{}
@@ -55,7 +56,7 @@ func SetupRouter(
// 应用中间件
r.Use(middleware2.RequestLogger())
// 将客户端 IP + UA 注入 request context,供 token 签发/会话绑定/审计日志统一读取。
// IP 取值与 API Key IP 限制共用「信任反代传递的客户端 IP」系统开关。
// IP 取值与 API Key IP 限制共用 server.trusted_proxies 信任链。
r.Use(middleware2.SessionBindingContext(cfg))
r.Use(middleware2.Logger())
r.Use(middleware2.CORS(cfg.CORS))
+5
View File
@@ -235,6 +235,11 @@ func registerOpsRoutes(admin *gin.RouterGroup, h *handler.Handlers) {
ops.GET("/request-errors/:id/upstream-errors", h.Admin.Ops.ListRequestErrorUpstreamErrors)
ops.PUT("/request-errors/:id/resolve", h.Admin.Ops.ResolveRequestError)
// Bounded ingress-admission rejection aggregates.
ops.GET("/ingress-rejections", h.Admin.Ops.ListIngressRejects)
ops.GET("/ingress-rejections/health", h.Admin.Ops.GetIngressRejectHealth)
ops.GET("/auth-cache-invalidation/health", h.Admin.Ops.GetAuthCacheInvalidationHealth)
// Upstream errors (independent upstream failures)
ops.GET("/upstream-errors", h.Admin.Ops.ListUpstreamErrors)
ops.GET("/upstream-errors/:id", h.Admin.Ops.GetUpstreamError)
+6 -5
View File
@@ -23,6 +23,7 @@ func RegisterGatewayRoutes(
cfg *config.Config,
) {
bodyLimit := middleware.RequestBodyLimit(cfg.Gateway.MaxBodySize)
textBodyLimit := middleware.RequestBodyLimit(cfg.Gateway.TextMaxBodySize)
clientRequestID := middleware.ClientRequestID()
opsErrorLogger := handler.OpsErrorLoggerMiddleware(opsService)
endpointNorm := handler.InboundEndpointMiddleware()
@@ -165,7 +166,7 @@ func RegisterGatewayRoutes(
}
h.Gateway.Responses(c)
})
gateway.POST("/alpha/search", h.OpenAIGateway.AlphaSearch)
gateway.POST("/alpha/search", textBodyLimit, h.OpenAIGateway.AlphaSearch)
gateway.GET("/responses", func(c *gin.Context) {
h.OpenAIGateway.ResponsesWebSocket(c)
})
@@ -177,7 +178,7 @@ func RegisterGatewayRoutes(
}
h.Gateway.ChatCompletions(c)
})
gateway.POST("/embeddings", func(c *gin.Context) {
gateway.POST("/embeddings", textBodyLimit, func(c *gin.Context) {
if getGroupPlatform(c) != service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{
@@ -236,7 +237,7 @@ func RegisterGatewayRoutes(
}
r.POST("/responses", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, responsesHandler)
r.POST("/responses/*subpath", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, responsesHandler)
r.POST("/alpha/search", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, h.OpenAIGateway.AlphaSearch)
r.POST("/alpha/search", textBodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, h.OpenAIGateway.AlphaSearch)
r.GET("/responses", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
h.OpenAIGateway.ResponsesWebSocket(c)
})
@@ -246,7 +247,7 @@ func RegisterGatewayRoutes(
{
codexDirect.POST("/responses", responsesHandler)
codexDirect.POST("/responses/*subpath", responsesHandler)
codexDirect.POST("/alpha/search", h.OpenAIGateway.AlphaSearch)
codexDirect.POST("/alpha/search", textBodyLimit, h.OpenAIGateway.AlphaSearch)
codexDirect.GET("/responses", func(c *gin.Context) {
h.OpenAIGateway.ResponsesWebSocket(c)
})
@@ -260,7 +261,7 @@ func RegisterGatewayRoutes(
}
h.Gateway.ChatCompletions(c)
})
r.POST("/embeddings", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
r.POST("/embeddings", textBodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
if getGroupPlatform(c) != service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{
@@ -0,0 +1,53 @@
package routes
import (
"net/http"
"net/http/httptest"
"testing"
"github.com/Wei-Shaw/sub2api/internal/handler"
adminhandler "github.com/Wei-Shaw/sub2api/internal/handler/admin"
servermiddleware "github.com/Wei-Shaw/sub2api/internal/server/middleware"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func TestIngressRejectAdminRoutesRequireAdminAuthentication(t *testing.T) {
gin.SetMode(gin.TestMode)
router := gin.New()
handlers := &handler.Handlers{Admin: &handler.AdminHandlers{Ops: adminhandler.NewOpsHandler(nil)}}
adminAuth := servermiddleware.AdminAuthMiddleware(func(c *gin.Context) {
if c.GetHeader("Authorization") == "" {
servermiddleware.AbortWithError(c, http.StatusUnauthorized, "UNAUTHORIZED", "Authorization required")
return
}
servermiddleware.AbortWithError(c, http.StatusForbidden, "FORBIDDEN", "Admin access required")
})
auditLog := servermiddleware.AuditLogMiddleware(func(c *gin.Context) { c.Next() })
stepUp := servermiddleware.StepUpAuthMiddleware(func(c *gin.Context) { c.Next() })
RegisterAdminRoutes(router.Group("/api/v1"), handlers, adminAuth, auditLog, stepUp, nil)
for _, path := range []string{
"/api/v1/admin/ops/ingress-rejections",
"/api/v1/admin/ops/ingress-rejections/health",
} {
for _, tc := range []struct {
name string
auth string
wantStatus int
}{
{name: "unauthenticated", wantStatus: http.StatusUnauthorized},
{name: "non-admin", auth: "Bearer user-token", wantStatus: http.StatusForbidden},
} {
t.Run(path+"/"+tc.name, func(t *testing.T) {
recorder := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodGet, path, nil)
if tc.auth != "" {
request.Header.Set("Authorization", tc.auth)
}
router.ServeHTTP(recorder, request)
require.Equal(t, tc.wantStatus, recorder.Code)
})
}
}
}