mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 12:02:27 +08:00
fix(ops): 记录固化 200 SSE 流上的就地错误,修复流内限流不进错误看板
流式请求一旦 flush 了 keepalive ping,HTTP 状态码即固化为 200;此后 出现的错误(等待并发槽位超时后回退的限流、Wait 后二次计费校验失败、 流开始后才无可用账号等)只能就地以 SSE error 帧回传。而 ops_error_logger 以 status>=400 为采集触发条件,这类挂在 200 流上的失败此前会在错误看板里 完全隐形——客户端能收到 rate_limit_error,但网关侧没有任何错误记录可供排障。 - service: 新增 OpsStreamError 上下文 + MarkOpsStreamError/GetOpsStreamError, 采用「首个标记生效」保留根因错误,避免被后续通用兜底帧覆盖。 - handler: handleStreamingAwareError 在 streamStarted 分支标记流内错误。 - handler: OpsErrorLoggerMiddleware 在 status<400 且无上游错误上下文时, 据标记补记一条错误日志;分级用 IntendedStatus(如并发限流 429), StatusCode 仍记 wire 的 200。上游透传错误已由 upstream-context 分支落库, 故此路径不重复记录。 - 补充单测覆盖补记、no-op、skip_monitoring 跳过与首个标记生效。
This commit is contained in:
@@ -590,6 +590,10 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
if !hasUpstreamContext {
|
||||
// 没有上游错误上下文,但网关可能在已固化的 200 流上就地补发了 SSE 错误帧
|
||||
// (如 ping 等待后并发超限、Wait 后二次计费校验失败)。这类失败若不在此补记,
|
||||
// 会因 wire 状态码为 200 而在错误看板里彻底隐形。
|
||||
logOpsStreamError(c, ops, status)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -999,6 +1003,138 @@ func OpsErrorLoggerMiddleware(ops *service.OpsService) gin.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// logOpsStreamError 记录一次挂在已固化 HTTP 200 SSE 流上的就地错误。
|
||||
// 由于 wire 状态码停留在 200,常规的 status>=400 捕获路径永远不会触发;
|
||||
// handleStreamingAwareError 通过 service.MarkOpsStreamError 标记这类错误,
|
||||
// 此函数据此补记一条错误日志,让并发限流/流内失败在错误看板里可见。
|
||||
//
|
||||
// 仅在 status<400 且不存在上游错误上下文时调用:上游透传错误已由中间件的
|
||||
// upstream-context 分支落库,无需在此重复记录。
|
||||
func logOpsStreamError(c *gin.Context, ops *service.OpsService, wireStatus int) {
|
||||
streamErr, ok := service.GetOpsStreamError(c)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// 命中 skip_monitoring=true 透传规则的请求跳过落库,与其它分支一致。
|
||||
if v, ok := c.Get(service.OpsSkipPassthroughKey); ok {
|
||||
if skip, _ := v.(bool); skip {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 复用与 status>=400 分支相同的设置过滤(context canceled / 无可用账号等)。
|
||||
if shouldSkipOpsErrorLog(c.Request.Context(), ops, streamErr.Message, streamErr.Message, c.Request.URL.Path) {
|
||||
return
|
||||
}
|
||||
|
||||
// 分级用「本应返回的状态码」(如并发限流 429),wire 状态码缺省时回退。
|
||||
classifyStatus := streamErr.IntendedStatus
|
||||
if classifyStatus <= 0 {
|
||||
classifyStatus = wireStatus
|
||||
}
|
||||
normalizedType := normalizeOpsErrorType(streamErr.ErrType, "")
|
||||
phase, isBusinessLimited, errorOwner, errorSource := classifyOpsErrorLog(c, normalizedType, streamErr.Message, "", classifyStatus)
|
||||
|
||||
apiKey := getOpsAPIKey(c)
|
||||
clientRequestID, _ := c.Request.Context().Value(ctxkey.ClientRequestID).(string)
|
||||
|
||||
model, _ := c.Get(opsModelKey)
|
||||
var modelName string
|
||||
if s, ok := model.(string); ok {
|
||||
modelName = s
|
||||
}
|
||||
accountIDV, _ := c.Get(opsAccountIDKey)
|
||||
var accountID *int64
|
||||
if v, ok := accountIDV.(int64); ok && v > 0 {
|
||||
accountID = &v
|
||||
}
|
||||
|
||||
fallbackPlatform := guessPlatformFromPath(c.Request.URL.Path)
|
||||
platform := resolveOpsPlatform(apiKey, fallbackPlatform)
|
||||
|
||||
requestID := c.Writer.Header().Get("X-Request-Id")
|
||||
if requestID == "" {
|
||||
requestID = c.Writer.Header().Get("x-request-id")
|
||||
}
|
||||
|
||||
entry := &service.OpsInsertErrorLogInput{
|
||||
RequestID: requestID,
|
||||
ClientRequestID: clientRequestID,
|
||||
|
||||
AccountID: accountID,
|
||||
Platform: platform,
|
||||
Model: modelName,
|
||||
RequestPath: func() string {
|
||||
if c.Request != nil && c.Request.URL != nil {
|
||||
return c.Request.URL.Path
|
||||
}
|
||||
return ""
|
||||
}(),
|
||||
// 就地 SSE 错误只出现在流式请求上。
|
||||
Stream: true,
|
||||
InboundEndpoint: GetInboundEndpoint(c),
|
||||
UpstreamEndpoint: GetUpstreamEndpoint(c, platform),
|
||||
RequestedModel: modelName,
|
||||
UpstreamModel: func() string {
|
||||
if v, ok := c.Get(opsUpstreamModelKey); ok {
|
||||
if s, ok := v.(string); ok {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}(),
|
||||
RequestType: func() *int16 {
|
||||
if v, ok := c.Get(opsRequestTypeKey); ok {
|
||||
switch t := v.(type) {
|
||||
case int16:
|
||||
return &t
|
||||
case int:
|
||||
v16 := int16(t)
|
||||
return &v16
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}(),
|
||||
UserAgent: c.GetHeader("User-Agent"),
|
||||
|
||||
ErrorPhase: phase,
|
||||
ErrorType: normalizedType,
|
||||
Severity: classifyOpsSeverity(normalizedType, classifyStatus),
|
||||
StatusCode: wireStatus,
|
||||
IsBusinessLimited: isBusinessLimited,
|
||||
IsCountTokens: isCountTokensRequest(c),
|
||||
|
||||
ErrorMessage: streamErr.Message,
|
||||
ErrorBody: "",
|
||||
ErrorSource: errorSource,
|
||||
ErrorOwner: errorOwner,
|
||||
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
applyOpsLatencyFieldsFromContext(c, entry)
|
||||
|
||||
if apiKey != nil {
|
||||
entry.APIKeyID = &apiKey.ID
|
||||
entry.APIKeyPrefix = keyPrefix(apiKey.Key, 8)
|
||||
if apiKey.User != nil {
|
||||
entry.UserID = &apiKey.User.ID
|
||||
}
|
||||
if apiKey.GroupID != nil {
|
||||
entry.GroupID = apiKey.GroupID
|
||||
}
|
||||
if apiKey.Group != nil && apiKey.Group.Platform != "" {
|
||||
entry.Platform = apiKey.Group.Platform
|
||||
}
|
||||
}
|
||||
|
||||
if clientIP := strings.TrimSpace(ip.GetClientIP(c)); clientIP != "" {
|
||||
entry.ClientIP = &clientIP
|
||||
}
|
||||
|
||||
enqueueOpsErrorLog(ops, entry)
|
||||
}
|
||||
|
||||
// isCountTokensRequest checks if the request is a count_tokens request
|
||||
func isCountTokensRequest(c *gin.Context) bool {
|
||||
if c == nil || c.Request == nil || c.Request.URL == nil {
|
||||
|
||||
Reference in New Issue
Block a user