Files
sub2api/backend/internal/service/batch_image_billing_recovery_test.go
T
shaw 80a229bce5 fix(batch-image): 修复审计发现的计费死锁、状态机与队列原子性缺陷
修复 PR #3768 批量图像 MVP 合并后审计报告中的全部问题:

结算与计费(高危):
- 所有 SETTLEMENT_* 失败(超冻结/计数非法/manifest 冲突/定价缺失/扣费失败)
  统一计入 retry_count 并在耗尽时释放冻结转 failed,消灭 settling 无限
  requeue 导致的冻结余额永久锁死
- 耗尽出口的释放指纹统一为 RequestHash,与 processor/Cancel/recovery 一致;
  release 遇同 request id 指纹冲突视为幂等成功,治愈历史毒消息
- 管理端校验 hold_multiplier >= discount_multiplier,定价快照对存量脏数据钳制
- 释放前校验 per-job hold claim(dedup+归档表),杜绝幻影释放

索引对账(高危):
- provider 输出与提交 custom_id 集对账:未知条目丢弃并记事件,
  漏项补 PROVIDER_RESULT_MISSING 失败行,保证 success+fail == item_count

提交与恢复(高危):
- 提交前转 uploading 并在 provider.Submit 期间心跳刷新 updated_at;
  恢复扫描改为原子复核(FailStaleUnsubmittedBatchImageJob),
  消灭慢提交被误杀退款而上游任务照常计费的孤儿场景
- 上游任务创建成功但本地状态推进失败时,尽力取消上游并清理输入
- recovery 释放失败时入队交由 worker releaseTerminalHold 兜底重试

队列与并发(中危):
- Enqueue(SetNX+LPush)与 Reserve(BRPop+ZAdd)均改为 Lua 原子脚本,
  消灭崩溃窗口导致 job 脱离队列、被 7 天 inflight 键锁死
- 锁冲突按 LockConflictDelay 重新入队(原直接丢弃需等 10 分钟 stale 恢复)
- 处理期间心跳:active zset 续期(ZAddXX 防幽灵成员)+ 锁 TTL 续期
- ReplaceBatchImageItemsForJob 增加 indexing 状态守卫,防掉队 worker 重写账目

存量回归(中危):
- image-only 定价条目(仅图片价无 token 价)恢复 token 计费 fail-closed,
  不再按 $0 计费;图片计费路径不受影响
- 鉴权余额门槛恢复 balance <= 0 语义,MinimumBalanceReserve 不再作硬 403

加固:
- ZIP max_items 钳制到管理员上限;Submit 补齐 Platform==Gemini 校验;
  gemini downloadUri 跟随前做 host 白名单校验
- 批量客户端改用共享 httpclient(拨号/TLS/响应头超时有界)
- 审计点名的忽略错误(MarkDownloaded/SettlementFailed/AppendEvent 等)改为记日志
2026-07-07 18:53:56 +08:00

118 lines
4.1 KiB
Go

//go:build unit
package service
import (
"context"
"errors"
"testing"
"time"
"github.com/stretchr/testify/require"
)
type recordingBatchImageQueue struct {
*fakeBatchImageQueue
enqueued []string
}
func (q *recordingBatchImageQueue) Enqueue(_ context.Context, batchID string) error {
q.enqueued = append(q.enqueued, batchID)
return nil
}
func TestBatchImageBillingRecoveryService_ReleasesStaleUnsubmittedHold(t *testing.T) {
repo := newFakeBatchImageRepository()
apiKeyID := int64(22)
holdAmount := 0.5
stale := &BatchImageJob{
BatchID: "imgbatch_stale_created",
UserID: 11,
APIKeyID: &apiKeyID,
Status: BatchImageJobStatusCreated,
EstimatedCost: holdAmount,
HoldAmount: &holdAmount,
CreatedAt: time.Now().Add(-time.Hour),
UpdatedAt: time.Now().Add(-time.Hour),
}
activeProviderName := "providers/job"
active := &BatchImageJob{
BatchID: "imgbatch_has_provider",
UserID: 11,
APIKeyID: &apiKeyID,
Status: BatchImageJobStatusSubmitted,
ProviderJobName: &activeProviderName,
EstimatedCost: holdAmount,
HoldAmount: &holdAmount,
CreatedAt: time.Now().Add(-time.Hour),
UpdatedAt: time.Now().Add(-time.Hour),
}
repo.jobs[stale.BatchID] = stale
repo.jobs[active.BatchID] = active
billing := &fakeBatchImageBillingRepo{}
svc := &BatchImageBillingRecoveryService{Repo: repo, Billing: billing, StaleAfter: time.Minute, Limit: 10}
released, err := svc.ReleaseStaleUnsubmittedOnce(context.Background())
require.NoError(t, err)
require.Equal(t, 1, released)
require.Equal(t, BatchImageJobStatusFailed, repo.jobs[stale.BatchID].Status)
require.Equal(t, "SUBMIT_STALE_BEFORE_PROVIDER", batchImageDerefString(repo.jobs[stale.BatchID].LastErrorCode))
require.Len(t, billing.releases, 1)
require.Equal(t, BatchImageReleaseRequestID(stale.BatchID), billing.releases[0].RequestID)
require.Equal(t, BatchImageJobStatusSubmitted, repo.jobs[active.BatchID].Status)
}
func TestBatchImageBillingRecoveryService_SkipsJobRefreshedByHeartbeat(t *testing.T) {
repo := newFakeBatchImageRepository()
apiKeyID := int64(22)
holdAmount := 0.5
// updated_at 在 cutoff 之后(慢提交心跳持续续期):不得误杀退款。
fresh := &BatchImageJob{
BatchID: "imgbatch_fresh_uploading",
UserID: 11,
APIKeyID: &apiKeyID,
Status: BatchImageJobStatusUploading,
EstimatedCost: holdAmount,
HoldAmount: &holdAmount,
CreatedAt: time.Now().Add(-time.Hour),
UpdatedAt: time.Now(),
}
repo.jobs[fresh.BatchID] = fresh
billing := &fakeBatchImageBillingRepo{}
svc := &BatchImageBillingRecoveryService{Repo: repo, Billing: billing, StaleAfter: time.Minute, Limit: 10}
released, err := svc.ReleaseStaleUnsubmittedOnce(context.Background())
require.NoError(t, err)
require.Equal(t, 0, released)
require.Equal(t, BatchImageJobStatusUploading, repo.jobs[fresh.BatchID].Status)
require.Empty(t, billing.releases)
}
func TestBatchImageBillingRecoveryService_EnqueuesRetryWhenReleaseFails(t *testing.T) {
repo := newFakeBatchImageRepository()
apiKeyID := int64(22)
holdAmount := 0.5
stale := &BatchImageJob{
BatchID: "imgbatch_stale_release_fail",
UserID: 11,
APIKeyID: &apiKeyID,
Status: BatchImageJobStatusCreated,
EstimatedCost: holdAmount,
HoldAmount: &holdAmount,
CreatedAt: time.Now().Add(-time.Hour),
UpdatedAt: time.Now().Add(-time.Hour),
}
repo.jobs[stale.BatchID] = stale
billing := &fakeBatchImageBillingRepo{releaseErr: errors.New("billing db down")}
queue := &recordingBatchImageQueue{fakeBatchImageQueue: newFakeBatchImageQueue("")}
svc := &BatchImageBillingRecoveryService{Repo: repo, Billing: billing, Queue: queue, StaleAfter: time.Minute, Limit: 10}
released, err := svc.ReleaseStaleUnsubmittedOnce(context.Background())
// job 已转 failed、不会再出现在 stale 列表:释放失败必须入队重试
//(由 worker 的 releaseTerminalHold 兜底),否则冻结余额永久泄漏。
require.Error(t, err)
require.Equal(t, 0, released)
require.Equal(t, BatchImageJobStatusFailed, repo.jobs[stale.BatchID].Status)
require.Equal(t, []string{stale.BatchID}, queue.enqueued)
}