sub2api/backend/internal/service/gateway_upstream_transport_error.go

package service

import (
	"context"
	"errors"
	"net/http"
	"time"

	"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
	"github.com/gin-gonic/gin"
	"go.uber.org/zap"
)

// gatewayTransportErrorTempUnschedDuration is how long an account is temporarily
// unscheduled after a durable transport failure (matches the OpenAI-side
// openAITransportErrorTempUnschedDuration).
const gatewayTransportErrorTempUnschedDuration = 10 * time.Minute

// gatewayTransportFailoverBody is the Anthropic-format error body attached to
// the failover error for a transport-level failure. Kept identical to the
// legacy inline 502 body so the client-visible payload is unchanged if
// failover is ultimately exhausted.
var gatewayTransportFailoverBody = []byte(`{"type":"error","error":{"type":"upstream_error","message":"Upstream request failed"}}`)

// handleUpstreamTransportError handles a transport-level upstream failure on
// the Anthropic/Bedrock forward paths (Do/DoWithTLS returned a non-HTTP error:
// proxy / DNS / TCP / TLS). It:
//  1. records the failure in Ops error logs (status 0, kind=request_error) —
//     the caller passes path-specific fields (UpstreamURL, Passthrough) via
//     event; identity, proxy attribution and classification fields are
//     filled here from the same account snapshot that built the transport.
//     A client disconnect is not recorded (see isClientCanceledTransportError);
//  2. for durable faults (expired/rejected proxy creds, dead proxy,
//     DNS/routing) temporarily unschedules the account and logs a stable warn
//     event that alert rules can key on;
//  3. returns an error that is *UpstreamFailoverError (so the handler fails
//     over to a healthy account) for all non-canceled errors, or the original
//     error for context.Canceled (client gone — no failover, no eviction).
//
// It deliberately does NOT write to the response: the handler owns the
// response (failover, or a protocol-correct error once failover is exhausted).
func (s *GatewayService) handleUpstreamTransportError(ctx context.Context, c *gin.Context, account *Account, err error, event OpsUpstreamErrorEvent) error {
	if isClientCanceledTransportError(ctx, err) {
		return err
	}
	safeErr := sanitizeUpstreamErrorMessage(err.Error())
	setOpsUpstreamError(c, 0, safeErr, "")
	event.ProxyID, event.ProxyName = opsUpstreamProxyAttribution(account)
	event.Platform = account.Platform
	event.AccountID = account.ID
	event.AccountName = account.Name
	event.UpstreamStatusCode = 0
	event.Kind = "request_error"
	event.Message = safeErr
	appendOpsUpstreamError(c, event)

	// Client disconnected: do NOT fail over to another account and do NOT
	// evict this one — the upstream never had a chance to exhibit a fault.
	if errors.Is(err, context.Canceled) || (errors.Is(err, context.DeadlineExceeded) && errors.Is(ctx.Err(), context.DeadlineExceeded)) {
		return err
	}

	// Transport attempt left local validation; count Ollama Cloud activity.
	scheduleOllamaCloudUsageActivity(s.deferredService, account)

	if classifyUpstreamTransportError(err).Persistent {
		s.tempUnscheduleTransportError(ctx, account, safeErr)
	}

	return &UpstreamFailoverError{
		StatusCode:   http.StatusBadGateway,
		ResponseBody: gatewayTransportFailoverBody,
	}
}

// tempUnscheduleTransportError marks an account temporarily unschedulable
// after a durable transport failure. Unlike the OpenAI side there is no
// in-memory scheduler block on this path: the Anthropic/Bedrock scheduler
// reads the persisted temp-unschedulable state, so the DB write is the single
// source of truth (same as tempUnscheduleGoogleConfigError /
// tempUnscheduleEmptyResponse).
//
// Log semantics:
//   - "gateway.account_temp_unscheduled_transport" — DB write succeeded.
//   - "gateway.account_temp_unschedule_transport_failed" — DB write attempted
//     but returned an error (the account remains schedulable).
func (s *GatewayService) tempUnscheduleTransportError(ctx context.Context, account *Account, safeErr string) {
	if s == nil {
		return
	}
	tempUnscheduleAccountForTransportError(ctx, s.accountRepo, account, safeErr)
}

// tempUnscheduleAccountForTransportError is the repo-level implementation
// shared by every forward path whose scheduler reads the persisted
// temp-unschedulable state (Anthropic/Bedrock and Gemini).
func tempUnscheduleAccountForTransportError(ctx context.Context, repo AccountRepository, account *Account, safeErr string) {
	if account == nil || repo == nil {
		return
	}
	until := time.Now().Add(gatewayTransportErrorTempUnschedDuration)
	reason := "upstream transport error (proxy/network): " + safeErr

	bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout)
	defer cancel()
	if err := repo.SetTempUnschedulable(bgCtx, account.ID, until, reason); err != nil {
		logger.L().With(zap.String("component", "service.gateway")).Warn(
			"gateway.account_temp_unschedule_transport_failed",
			zap.Int64("account_id", account.ID),
			zap.Error(err),
		)
		return
	}
	logger.L().With(zap.String("component", "service.gateway")).Warn(
		"gateway.account_temp_unscheduled_transport",
		zap.Int64("account_id", account.ID),
		zap.String("account_name", account.Name),
		zap.String("platform", account.Platform),
		zap.Time("until", until),
		zap.String("reason", reason),
	)
}