mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 12:02:27 +08:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7d036c0863 | ||
|
|
b5e83156d9 | ||
|
|
fddc806db2 | ||
|
|
bbc8b6e906 | ||
|
|
48eb3766d2 | ||
|
|
e01c917a97 |
@@ -1 +1 @@
|
||||
0.1.172
|
||||
0.1.173
|
||||
|
||||
@@ -153,12 +153,12 @@ func (h *BackupHandler) GetDownloadURL(c *gin.Context) {
|
||||
response.BadRequest(c, "backup ID is required")
|
||||
return
|
||||
}
|
||||
url, err := h.backupService.GetBackupDownloadURL(c.Request.Context(), backupID)
|
||||
download, err := h.backupService.GetBackupDownloadURL(c.Request.Context(), backupID)
|
||||
if err != nil {
|
||||
response.ErrorFrom(c, err)
|
||||
return
|
||||
}
|
||||
response.Success(c, gin.H{"url": url})
|
||||
response.Success(c, download)
|
||||
}
|
||||
|
||||
// ─── 恢复操作(需要重新输入管理员密码) ───
|
||||
|
||||
@@ -47,14 +47,16 @@ type contentModerationConfigRequest struct {
|
||||
ViolationWindowHours *int `json:"violation_window_hours"`
|
||||
// cyber_policy 命中是否排除出自动封号计数;前端 RiskControlView 已发送该字段,
|
||||
// service.UpdateContentModerationConfigInput 已支持,此前 handler 层缺透传导致开关静默失效。
|
||||
CyberPolicyExcludeFromBanCount *bool `json:"cyber_policy_exclude_from_ban_count"`
|
||||
RetryCount *int `json:"retry_count"`
|
||||
HitRetentionDays *int `json:"hit_retention_days"`
|
||||
NonHitRetentionDays *int `json:"non_hit_retention_days"`
|
||||
PreHashCheckEnabled *bool `json:"pre_hash_check_enabled"`
|
||||
BlockedKeywords *[]string `json:"blocked_keywords"`
|
||||
KeywordBlockingMode *string `json:"keyword_blocking_mode"`
|
||||
ModelFilter *service.ContentModerationModelFilter `json:"model_filter"`
|
||||
CyberPolicyExcludeFromBanCount *bool `json:"cyber_policy_exclude_from_ban_count"`
|
||||
// 审计链路自身异常(审计 API 失败、无可用审计 Key)时是否拒绝请求。
|
||||
FailClosedOnError *bool `json:"fail_closed_on_error"`
|
||||
RetryCount *int `json:"retry_count"`
|
||||
HitRetentionDays *int `json:"hit_retention_days"`
|
||||
NonHitRetentionDays *int `json:"non_hit_retention_days"`
|
||||
PreHashCheckEnabled *bool `json:"pre_hash_check_enabled"`
|
||||
BlockedKeywords *[]string `json:"blocked_keywords"`
|
||||
KeywordBlockingMode *string `json:"keyword_blocking_mode"`
|
||||
ModelFilter *service.ContentModerationModelFilter `json:"model_filter"`
|
||||
}
|
||||
|
||||
type contentModerationAPIKeyTestRequest struct {
|
||||
@@ -112,6 +114,7 @@ func (h *ContentModerationHandler) UpdateConfig(c *gin.Context) {
|
||||
BanThreshold: req.BanThreshold,
|
||||
ViolationWindowHours: req.ViolationWindowHours,
|
||||
CyberPolicyExcludeFromBanCount: req.CyberPolicyExcludeFromBanCount,
|
||||
FailClosedOnError: req.FailClosedOnError,
|
||||
RetryCount: req.RetryCount,
|
||||
HitRetentionDays: req.HitRetentionDays,
|
||||
NonHitRetentionDays: req.NonHitRetentionDays,
|
||||
|
||||
@@ -19,7 +19,15 @@ func contentModerationStatus(decision *service.ContentModerationDecision) int {
|
||||
return decision.StatusCode
|
||||
}
|
||||
|
||||
// contentModerationErrorCode 优先使用决策自带的错误码,只有内容策略拦截才回退到
|
||||
// content_policy_violation。审计链路故障(fail-closed)会带
|
||||
// content_moderation_unavailable,不能让客户端以为自己触犯了内容策略。
|
||||
func contentModerationErrorCode(decision *service.ContentModerationDecision) string {
|
||||
if decision != nil {
|
||||
if code := strings.TrimSpace(decision.ErrorCode); code != "" {
|
||||
return code
|
||||
}
|
||||
}
|
||||
return "content_policy_violation"
|
||||
}
|
||||
|
||||
|
||||
@@ -65,10 +65,10 @@ func runSecurityAudit(c *gin.Context, reqLog *zap.Logger, coordinator *securitya
|
||||
decision.Legacy = &securityaudit.LegacyDecision{
|
||||
Allowed: legacyDecision.Allowed, Blocked: legacyDecision.Blocked, Flagged: legacyDecision.Flagged,
|
||||
Message: legacyDecision.Message, StatusCode: legacyDecision.StatusCode,
|
||||
ErrorCode: "content_policy_violation", Action: legacyDecision.Action,
|
||||
ErrorCode: contentModerationErrorCode(legacyDecision), Action: legacyDecision.Action,
|
||||
}
|
||||
if legacyDecision.Blocked {
|
||||
decision.Kind, decision.HTTPStatus, decision.ErrorCode, decision.ClientMessage, decision.AllowNextStage = securityaudit.DecisionBlock, contentModerationStatus(legacyDecision), "content_policy_violation", legacyDecision.Message, false
|
||||
decision.Kind, decision.HTTPStatus, decision.ErrorCode, decision.ClientMessage, decision.AllowNextStage = securityaudit.DecisionBlock, contentModerationStatus(legacyDecision), contentModerationErrorCode(legacyDecision), legacyDecision.Message, false
|
||||
}
|
||||
if decision.AllowNextStage && cacheCompletion {
|
||||
c.Set(securityAuditCompletedContextKey, true)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
@@ -59,6 +60,34 @@ func (s *S3BackupStore) Upload(ctx context.Context, key string, body io.Reader,
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) UploadFile(ctx context.Context, key string, filePath string, contentType string) (int64, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("open upload file: %w", err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stat upload file: %w", err)
|
||||
}
|
||||
sizeBytes := info.Size()
|
||||
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
|
||||
Bucket: &s.bucket,
|
||||
Key: &key,
|
||||
Body: file,
|
||||
ContentLength: &sizeBytes,
|
||||
ContentType: &contentType,
|
||||
})
|
||||
finish()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("S3 PutObject file: %w", err)
|
||||
}
|
||||
return sizeBytes, nil
|
||||
}
|
||||
|
||||
func (s *S3BackupStore) Download(ctx context.Context, key string) (io.ReadCloser, error) {
|
||||
finish := servertiming.ObserveDependency(ctx, "s3")
|
||||
result, err := s.client.GetObject(ctx, &s3.GetObjectInput{
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
//go:build unit
|
||||
|
||||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestS3BackupStore_UploadFile(t *testing.T) {
|
||||
var received []byte
|
||||
var receivedLength int64
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
require.Equal(t, http.MethodPut, r.Method)
|
||||
receivedLength = r.ContentLength
|
||||
var err error
|
||||
received, err = io.ReadAll(r.Body)
|
||||
require.NoError(t, err)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := newS3Client(context.Background(), s3ClientParams{
|
||||
Endpoint: server.URL,
|
||||
Region: "auto",
|
||||
AccessKeyID: "test-ak",
|
||||
SecretAccessKey: "test-sk",
|
||||
ForcePathStyle: true,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
content := []byte("streamed backup payload")
|
||||
filePath := t.TempDir() + "/part.gz"
|
||||
require.NoError(t, os.WriteFile(filePath, content, 0o600))
|
||||
|
||||
store := &S3BackupStore{client: client, bucket: "backup-bucket"}
|
||||
size, err := store.UploadFile(context.Background(), "backup/part-1", filePath, "application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(len(content)), size)
|
||||
require.Equal(t, int64(len(content)), receivedLength)
|
||||
require.Equal(t, content, received)
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package securityaudit
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
)
|
||||
@@ -27,9 +28,24 @@ func (a *LegacyModerationAdapter) Check(ctx context.Context, req Request) (*Lega
|
||||
if err != nil || decision == nil {
|
||||
return nil, err
|
||||
}
|
||||
return legacyDecisionFromModeration(decision), nil
|
||||
}
|
||||
|
||||
// legacyDecisionFromModeration 把内容审计决策映射成协调器决策。
|
||||
//
|
||||
// 审计链路故障(fail-closed)自带 content_moderation_unavailable;内容策略拦截不带
|
||||
// 错误码,回退到 content_policy_violation。
|
||||
func legacyDecisionFromModeration(decision *service.ContentModerationDecision) *LegacyDecision {
|
||||
if decision == nil {
|
||||
return nil
|
||||
}
|
||||
errorCode := strings.TrimSpace(decision.ErrorCode)
|
||||
if errorCode == "" {
|
||||
errorCode = "content_policy_violation"
|
||||
}
|
||||
return &LegacyDecision{
|
||||
Allowed: decision.Allowed, Blocked: decision.Blocked, Flagged: decision.Flagged,
|
||||
Message: decision.Message, StatusCode: decision.StatusCode,
|
||||
ErrorCode: "content_policy_violation", Action: decision.Action,
|
||||
}, nil
|
||||
ErrorCode: errorCode, Action: decision.Action,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package securityaudit
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/service"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// LegacyModerationAdapter 必须透传决策自带的错误码。审计链路故障(fail-closed)带
|
||||
// content_moderation_unavailable,不能被压成 content_policy_violation —— 否则客户端
|
||||
// 会把一次系统不可用当成自己触犯了内容策略。
|
||||
func TestLegacyModerationAdapter_PropagatesDecisionErrorCode(t *testing.T) {
|
||||
t.Run("内容策略拦截回退到 content_policy_violation", func(t *testing.T) {
|
||||
decision := &service.ContentModerationDecision{
|
||||
Blocked: true, Flagged: true, StatusCode: http.StatusForbidden,
|
||||
Message: "blocked", Action: service.ContentModerationActionBlock,
|
||||
}
|
||||
got := legacyDecisionFromModeration(decision)
|
||||
require.Equal(t, "content_policy_violation", got.ErrorCode)
|
||||
})
|
||||
|
||||
t.Run("审计链路故障保留独立错误码", func(t *testing.T) {
|
||||
decision := &service.ContentModerationDecision{
|
||||
Blocked: true, Flagged: false, StatusCode: http.StatusServiceUnavailable,
|
||||
Message: service.ContentModerationUnavailableMessage,
|
||||
Action: service.ContentModerationActionError,
|
||||
ErrorCode: service.ContentModerationErrorCodeUnavailable,
|
||||
}
|
||||
got := legacyDecisionFromModeration(decision)
|
||||
require.Equal(t, service.ContentModerationErrorCodeUnavailable, got.ErrorCode)
|
||||
require.False(t, got.Flagged, "系统故障不是内容违规")
|
||||
})
|
||||
}
|
||||
|
||||
// prioritize 必须把 legacy 的 503 与错误码原样带到网关响应上,而不是压成 403。
|
||||
func TestPrioritize_KeepsModerationUnavailableStatusAndCode(t *testing.T) {
|
||||
legacy := &LegacyDecision{
|
||||
Blocked: true, StatusCode: http.StatusServiceUnavailable,
|
||||
Message: service.ContentModerationUnavailableMessage,
|
||||
ErrorCode: service.ContentModerationErrorCodeUnavailable,
|
||||
Action: service.ContentModerationActionError,
|
||||
}
|
||||
|
||||
decision := prioritize(legacy, nil)
|
||||
|
||||
require.Equal(t, DecisionBlock, decision.Kind)
|
||||
require.Equal(t, http.StatusServiceUnavailable, decision.HTTPStatus)
|
||||
require.Equal(t, service.ContentModerationErrorCodeUnavailable, decision.ErrorCode)
|
||||
require.False(t, decision.AllowNextStage)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
const defaultBackupPartSizeBytes int64 = 4 * 1024 * 1024 * 1024
|
||||
|
||||
// BackupPart 描述一个 gzip 字节分卷。
|
||||
type BackupPart struct {
|
||||
Index int `json:"index"`
|
||||
S3Key string `json:"s3_key"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256,omitempty"`
|
||||
}
|
||||
|
||||
type localBackupPart struct {
|
||||
Index int
|
||||
Path string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
func splitBackupFile(srcPath string, partSize int64) (parts []localBackupPart, err error) {
|
||||
if partSize <= 0 {
|
||||
return nil, fmt.Errorf("backup part size must be positive")
|
||||
}
|
||||
|
||||
src, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open backup archive: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := src.Close(); err == nil && closeErr != nil {
|
||||
err = fmt.Errorf("close backup archive: %w", closeErr)
|
||||
}
|
||||
if err != nil {
|
||||
paths := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
paths = append(paths, part.Path)
|
||||
}
|
||||
_ = cleanupBackupFiles(paths...)
|
||||
}
|
||||
}()
|
||||
|
||||
info, err := src.Stat()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat backup archive: %w", err)
|
||||
}
|
||||
if info.Size() <= 0 {
|
||||
return nil, errors.New("backup archive is empty")
|
||||
}
|
||||
|
||||
remaining := info.Size()
|
||||
for index := 1; remaining > 0; index++ {
|
||||
partFile, createErr := os.CreateTemp("", "sub2api-backup-part-*")
|
||||
if createErr != nil {
|
||||
return nil, fmt.Errorf("create backup part: %w", createErr)
|
||||
}
|
||||
partPath := partFile.Name()
|
||||
partBytes := partSize
|
||||
if remaining < partBytes {
|
||||
partBytes = remaining
|
||||
}
|
||||
|
||||
hash := sha256.New()
|
||||
written, copyErr := io.CopyN(io.MultiWriter(partFile, hash), src, partBytes)
|
||||
closeErr := partFile.Close()
|
||||
if copyErr != nil {
|
||||
_ = os.Remove(partPath)
|
||||
return nil, fmt.Errorf("write backup part %d: %w", index, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(partPath)
|
||||
return nil, fmt.Errorf("close backup part %d: %w", index, closeErr)
|
||||
}
|
||||
|
||||
parts = append(parts, localBackupPart{
|
||||
Index: index,
|
||||
Path: partPath,
|
||||
SizeBytes: written,
|
||||
SHA256: hex.EncodeToString(hash.Sum(nil)),
|
||||
})
|
||||
remaining -= written
|
||||
}
|
||||
|
||||
return parts, nil
|
||||
}
|
||||
|
||||
func cleanupBackupFiles(paths ...string) error {
|
||||
var errs []error
|
||||
for _, path := range paths {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
errs = append(errs, fmt.Errorf("remove %s: %w", path, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSplitBackupFile_ReassemblesExactBytes(t *testing.T) {
|
||||
src := writeBackupArchiveFixture(t, []byte("0123456789abcdefg"))
|
||||
parts, err := splitBackupFile(src, 5)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, parts, 4)
|
||||
|
||||
var got bytes.Buffer
|
||||
for i, part := range parts {
|
||||
require.Equal(t, i+1, part.Index)
|
||||
require.LessOrEqual(t, part.SizeBytes, int64(5))
|
||||
data, readErr := os.ReadFile(part.Path)
|
||||
require.NoError(t, readErr)
|
||||
require.Equal(t, fmt.Sprintf("%x", sha256.Sum256(data)), part.SHA256)
|
||||
got.Write(data)
|
||||
}
|
||||
require.Equal(t, []byte("0123456789abcdefg"), got.Bytes())
|
||||
}
|
||||
|
||||
func TestSplitBackupFile_RejectsInvalidInput(t *testing.T) {
|
||||
src := writeBackupArchiveFixture(t, []byte("data"))
|
||||
|
||||
_, err := splitBackupFile(src, 0)
|
||||
require.Error(t, err)
|
||||
|
||||
empty := writeBackupArchiveFixture(t, nil)
|
||||
_, err = splitBackupFile(empty, 5)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = splitBackupFile(filepathForMissingBackupArchive(t), 5)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func writeBackupArchiveFixture(t *testing.T, content []byte) string {
|
||||
t.Helper()
|
||||
path := filepathForBackupArchive(t)
|
||||
require.NoError(t, os.WriteFile(path, content, 0o600))
|
||||
return path
|
||||
}
|
||||
|
||||
func filepathForBackupArchive(t *testing.T) string {
|
||||
t.Helper()
|
||||
return t.TempDir() + "/archive.gz"
|
||||
}
|
||||
|
||||
func filepathForMissingBackupArchive(t *testing.T) string {
|
||||
t.Helper()
|
||||
return t.TempDir() + "/missing.gz"
|
||||
}
|
||||
@@ -3,10 +3,13 @@ package service
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -26,7 +29,8 @@ const (
|
||||
settingKeyBackupSchedule = "backup_schedule"
|
||||
settingKeyBackupRecords = "backup_records"
|
||||
|
||||
maxBackupRecords = 100
|
||||
maxBackupRecords = 100
|
||||
backupObjectCleanupTimeout = 2 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -61,6 +65,7 @@ type DBDumper interface {
|
||||
// BackupObjectStore abstracts object storage for backup files
|
||||
type BackupObjectStore interface {
|
||||
Upload(ctx context.Context, key string, body io.Reader, contentType string) (sizeBytes int64, err error)
|
||||
UploadFile(ctx context.Context, key string, filePath string, contentType string) (sizeBytes int64, err error)
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, error)
|
||||
Delete(ctx context.Context, key string) error
|
||||
PresignURL(ctx context.Context, key string, expiry time.Duration) (string, error)
|
||||
@@ -98,21 +103,35 @@ type BackupScheduleConfig struct {
|
||||
|
||||
// BackupRecord 备份记录
|
||||
type BackupRecord struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"` // pending, running, completed, failed
|
||||
BackupType string `json:"backup_type"` // postgres
|
||||
FileName string `json:"file_name"`
|
||||
S3Key string `json:"s3_key"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
TriggeredBy string `json:"triggered_by"` // manual, scheduled
|
||||
ErrorMsg string `json:"error_message,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
ExpiresAt string `json:"expires_at,omitempty"` // 过期时间
|
||||
Progress string `json:"progress,omitempty"` // "dumping", "uploading", ""
|
||||
RestoreStatus string `json:"restore_status,omitempty"` // "", "running", "completed", "failed"
|
||||
RestoreError string `json:"restore_error,omitempty"`
|
||||
RestoredAt string `json:"restored_at,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"` // pending, running, completed, failed
|
||||
BackupType string `json:"backup_type"` // postgres
|
||||
FileName string `json:"file_name"`
|
||||
S3Key string `json:"s3_key"`
|
||||
Parts []BackupPart `json:"parts,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
TriggeredBy string `json:"triggered_by"` // manual, scheduled
|
||||
ErrorMsg string `json:"error_message,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
ExpiresAt string `json:"expires_at,omitempty"` // 过期时间
|
||||
Progress string `json:"progress,omitempty"` // "dumping", "uploading", ""
|
||||
RestoreStatus string `json:"restore_status,omitempty"` // "", "running", "completed", "failed"
|
||||
RestoreError string `json:"restore_error,omitempty"`
|
||||
RestoredAt string `json:"restored_at,omitempty"`
|
||||
}
|
||||
|
||||
// BackupDownloadPart 描述一个可下载的备份分卷。
|
||||
type BackupDownloadPart struct {
|
||||
Index int `json:"index"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
// BackupDownloadResponse 是单文件和分卷下载响应的兼容表示。
|
||||
type BackupDownloadResponse struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
Parts []BackupDownloadPart `json:"parts,omitempty"`
|
||||
}
|
||||
|
||||
// BackupService 数据库备份恢复服务
|
||||
@@ -142,10 +161,11 @@ type BackupService struct {
|
||||
cronSched *cron.Cron
|
||||
cronEntryID cron.EntryID
|
||||
|
||||
wg sync.WaitGroup // 追踪活跃的备份/恢复 goroutine
|
||||
shuttingDown atomic.Bool // 阻止新备份启动
|
||||
bgCtx context.Context // 所有后台操作的 parent context
|
||||
bgCancel context.CancelFunc // 取消所有活跃后台操作
|
||||
wg sync.WaitGroup // 追踪活跃的备份/恢复 goroutine
|
||||
shuttingDown atomic.Bool // 阻止新备份启动
|
||||
bgCtx context.Context // 所有后台操作的 parent context
|
||||
bgCancel context.CancelFunc // 取消所有活跃后台操作
|
||||
partSizeBytes int64 // 分卷阈值;生产使用 4 GiB,测试可注入更小值
|
||||
}
|
||||
|
||||
func NewBackupService(
|
||||
@@ -165,6 +185,7 @@ func NewBackupService(
|
||||
dumper: dumper,
|
||||
bgCtx: bgCtx,
|
||||
bgCancel: bgCancel,
|
||||
partSizeBytes: defaultBackupPartSizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -191,33 +212,57 @@ func (s *BackupService) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
// recoverStaleRecords 启动时将孤立的 running 记录标记为 failed
|
||||
// recoverStaleRecords 启动时将孤立的 running 记录标记为 failed,并清理已上传对象。
|
||||
func (s *BackupService) recoverStaleRecords() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
loadCtx, loadCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer loadCancel()
|
||||
|
||||
records, err := s.loadRecords(ctx)
|
||||
records, err := s.loadRecords(loadCtx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for i := range records {
|
||||
if records[i].Status == "running" {
|
||||
staleRecord := records[i]
|
||||
records[i].Status = "failed"
|
||||
records[i].ErrorMsg = "interrupted by server restart"
|
||||
records[i].Progress = ""
|
||||
records[i].FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = s.saveRecord(ctx, &records[i])
|
||||
s.saveRecoveredRecord(&records[i])
|
||||
|
||||
if cleanupErr := s.cleanupStaleBackupObjects(&staleRecord); cleanupErr != nil {
|
||||
records[i].ErrorMsg = fmt.Sprintf("interrupted by server restart; cleanup failed, manual deletion may be required: %v", cleanupErr)
|
||||
s.saveRecoveredRecord(&records[i])
|
||||
logger.LegacyPrintf("service.backup", "[Backup] failed to clean stale backup objects for %s: %v", records[i].ID, cleanupErr)
|
||||
}
|
||||
logger.LegacyPrintf("service.backup", "[Backup] recovered stale running record: %s", records[i].ID)
|
||||
}
|
||||
if records[i].RestoreStatus == "running" {
|
||||
records[i].RestoreStatus = "failed"
|
||||
records[i].RestoreError = "interrupted by server restart"
|
||||
_ = s.saveRecord(ctx, &records[i])
|
||||
s.saveRecoveredRecord(&records[i])
|
||||
logger.LegacyPrintf("service.backup", "[Backup] recovered stale restoring record: %s", records[i].ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BackupService) saveRecoveredRecord(record *BackupRecord) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.saveRecord(ctx, record); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存恢复后的备份记录失败 %s: %v", record.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BackupService) cleanupStaleBackupObjects(record *BackupRecord) error {
|
||||
if len(backupObjectKeys(record)) == 0 {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), backupObjectCleanupTimeout)
|
||||
defer cancel()
|
||||
return s.deleteBackupObjects(ctx, record)
|
||||
}
|
||||
|
||||
// Stop 停止定时备份并等待活跃操作完成
|
||||
func (s *BackupService) Stop() {
|
||||
s.shuttingDown.Store(true)
|
||||
@@ -452,7 +497,7 @@ func (s *BackupService) runScheduledBackup() {
|
||||
|
||||
// ─── 备份/恢复核心 ───
|
||||
|
||||
// CreateBackup 创建全量数据库备份并上传到 S3(流式处理)
|
||||
// CreateBackup 创建全量数据库备份并上传到 S3。
|
||||
// expireDays: 备份过期天数,0=永不过期,默认14天
|
||||
func (s *BackupService) CreateBackup(ctx context.Context, triggeredBy string, expireDays int) (*BackupRecord, error) {
|
||||
if s.shuttingDown.Load() {
|
||||
@@ -506,61 +551,27 @@ func (s *BackupService) CreateBackup(ctx context.Context, triggeredBy string, ex
|
||||
ExpiresAt: expiresAt,
|
||||
}
|
||||
|
||||
// 流式执行: pg_dump -> gzip -> S3 upload
|
||||
dumpReader, err := s.dumper.Dump(ctx)
|
||||
archivePath, sizeBytes, err := s.createCompressedBackupFile(ctx)
|
||||
if err != nil {
|
||||
record.Status = "failed"
|
||||
record.ErrorMsg = fmt.Sprintf("pg_dump failed: %v", err)
|
||||
record.ErrorMsg = err.Error()
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = s.saveRecord(ctx, record)
|
||||
return record, fmt.Errorf("pg_dump: %w", err)
|
||||
return record, err
|
||||
}
|
||||
|
||||
// 使用 io.Pipe 将 gzip 压缩数据流式传递给 S3 上传
|
||||
pr, pw := io.Pipe()
|
||||
gzipDone := make(chan error, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
pw.CloseWithError(fmt.Errorf("gzip goroutine panic: %v", r)) //nolint:errcheck
|
||||
gzipDone <- fmt.Errorf("gzip goroutine panic: %v", r)
|
||||
}
|
||||
}()
|
||||
gzWriter := gzip.NewWriter(pw)
|
||||
var gzErr error
|
||||
_, gzErr = io.Copy(gzWriter, dumpReader)
|
||||
if closeErr := gzWriter.Close(); closeErr != nil && gzErr == nil {
|
||||
gzErr = closeErr
|
||||
}
|
||||
if closeErr := dumpReader.Close(); closeErr != nil && gzErr == nil {
|
||||
gzErr = closeErr
|
||||
}
|
||||
if gzErr != nil {
|
||||
_ = pw.CloseWithError(gzErr)
|
||||
} else {
|
||||
_ = pw.Close()
|
||||
}
|
||||
gzipDone <- gzErr
|
||||
}()
|
||||
|
||||
contentType := "application/gzip"
|
||||
sizeBytes, err := objectStore.Upload(ctx, s3Key, pr, contentType)
|
||||
if err != nil {
|
||||
_ = pr.CloseWithError(err) // 确保 gzip goroutine 不会悬挂
|
||||
gzErr := <-gzipDone // 安全等待 gzip goroutine 完成
|
||||
record.Status = "failed"
|
||||
errMsg := fmt.Sprintf("S3 upload failed: %v", err)
|
||||
if gzErr != nil {
|
||||
errMsg = fmt.Sprintf("gzip/dump failed: %v", gzErr)
|
||||
}
|
||||
record.ErrorMsg = errMsg
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = s.saveRecord(ctx, record)
|
||||
return record, fmt.Errorf("backup upload: %w", err)
|
||||
}
|
||||
<-gzipDone // 确保 gzip goroutine 已退出
|
||||
|
||||
defer func() { _ = cleanupBackupFiles(archivePath) }()
|
||||
record.SizeBytes = sizeBytes
|
||||
if err := s.saveRecord(ctx, record); err != nil {
|
||||
return nil, fmt.Errorf("save initial record: %w", err)
|
||||
}
|
||||
if err := s.uploadBackupArchive(ctx, record, objectStore, s3Cfg, archivePath); err != nil {
|
||||
record.Status = "failed"
|
||||
record.ErrorMsg = err.Error()
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = s.saveRecord(ctx, record)
|
||||
return record, err
|
||||
}
|
||||
|
||||
record.Status = "completed"
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
if err := s.saveRecord(ctx, record); err != nil {
|
||||
@@ -656,78 +667,43 @@ func (s *BackupService) StartBackup(ctx context.Context, triggeredBy string, exp
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
}
|
||||
}()
|
||||
s.executeBackup(record, objectStore)
|
||||
s.executeBackup(record, objectStore, s3Cfg)
|
||||
}()
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// executeBackup 后台执行备份(独立于 HTTP context)
|
||||
func (s *BackupService) executeBackup(record *BackupRecord, objectStore BackupObjectStore) {
|
||||
func (s *BackupService) executeBackup(record *BackupRecord, objectStore BackupObjectStore, s3Cfg *BackupS3Config) {
|
||||
ctx, cancel := context.WithTimeout(s.bgCtx, 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
// 阶段1: pg_dump
|
||||
// 阶段1: pg_dump -> gzip 临时文件
|
||||
record.Progress = "dumping"
|
||||
_ = s.saveRecord(ctx, record)
|
||||
|
||||
dumpReader, err := s.dumper.Dump(ctx)
|
||||
archivePath, sizeBytes, err := s.createCompressedBackupFile(ctx)
|
||||
if err != nil {
|
||||
record.Status = "failed"
|
||||
record.ErrorMsg = fmt.Sprintf("pg_dump failed: %v", err)
|
||||
record.ErrorMsg = err.Error()
|
||||
record.Progress = ""
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
defer func() { _ = cleanupBackupFiles(archivePath) }()
|
||||
record.SizeBytes = sizeBytes
|
||||
|
||||
// 阶段2: gzip + upload
|
||||
// 阶段2: 单对象或分卷上传
|
||||
record.Progress = "uploading"
|
||||
_ = s.saveRecord(ctx, record)
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
gzipDone := make(chan error, 1)
|
||||
go func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
pw.CloseWithError(fmt.Errorf("gzip goroutine panic: %v", r)) //nolint:errcheck
|
||||
gzipDone <- fmt.Errorf("gzip goroutine panic: %v", r)
|
||||
}
|
||||
}()
|
||||
gzWriter := gzip.NewWriter(pw)
|
||||
var gzErr error
|
||||
_, gzErr = io.Copy(gzWriter, dumpReader)
|
||||
if closeErr := gzWriter.Close(); closeErr != nil && gzErr == nil {
|
||||
gzErr = closeErr
|
||||
}
|
||||
if closeErr := dumpReader.Close(); closeErr != nil && gzErr == nil {
|
||||
gzErr = closeErr
|
||||
}
|
||||
if gzErr != nil {
|
||||
_ = pw.CloseWithError(gzErr)
|
||||
} else {
|
||||
_ = pw.Close()
|
||||
}
|
||||
gzipDone <- gzErr
|
||||
}()
|
||||
|
||||
contentType := "application/gzip"
|
||||
sizeBytes, err := objectStore.Upload(ctx, record.S3Key, pr, contentType)
|
||||
if err != nil {
|
||||
_ = pr.CloseWithError(err) // 确保 gzip goroutine 不会悬挂
|
||||
gzErr := <-gzipDone // 安全等待 gzip goroutine 完成
|
||||
if err := s.uploadBackupArchive(ctx, record, objectStore, s3Cfg, archivePath); err != nil {
|
||||
record.Status = "failed"
|
||||
errMsg := fmt.Sprintf("S3 upload failed: %v", err)
|
||||
if gzErr != nil {
|
||||
errMsg = fmt.Sprintf("gzip/dump failed: %v", gzErr)
|
||||
}
|
||||
record.ErrorMsg = errMsg
|
||||
record.ErrorMsg = err.Error()
|
||||
record.Progress = ""
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
<-gzipDone // 确保 gzip goroutine 已退出
|
||||
|
||||
record.SizeBytes = sizeBytes
|
||||
record.Status = "completed"
|
||||
@@ -738,6 +714,108 @@ func (s *BackupService) executeBackup(record *BackupRecord, objectStore BackupOb
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BackupService) createCompressedBackupFile(ctx context.Context) (string, int64, error) {
|
||||
dumpReader, err := s.dumper.Dump(ctx)
|
||||
if err != nil {
|
||||
return "", 0, fmt.Errorf("pg_dump: %w", err)
|
||||
}
|
||||
archive, err := os.CreateTemp("", "sub2api-backup-*.sql.gz")
|
||||
if err != nil {
|
||||
_ = dumpReader.Close()
|
||||
return "", 0, fmt.Errorf("create backup archive: %w", err)
|
||||
}
|
||||
archivePath := archive.Name()
|
||||
|
||||
gzWriter := gzip.NewWriter(archive)
|
||||
_, copyErr := io.Copy(gzWriter, dumpReader)
|
||||
if closeErr := gzWriter.Close(); copyErr == nil && closeErr != nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if closeErr := dumpReader.Close(); copyErr == nil && closeErr != nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if closeErr := archive.Close(); copyErr == nil && closeErr != nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
_ = cleanupBackupFiles(archivePath)
|
||||
return "", 0, fmt.Errorf("gzip/dump failed: %w", copyErr)
|
||||
}
|
||||
|
||||
info, err := os.Stat(archivePath)
|
||||
if err != nil {
|
||||
_ = cleanupBackupFiles(archivePath)
|
||||
return "", 0, fmt.Errorf("stat backup archive: %w", err)
|
||||
}
|
||||
return archivePath, info.Size(), nil
|
||||
}
|
||||
|
||||
func (s *BackupService) uploadBackupArchive(ctx context.Context, record *BackupRecord, objectStore BackupObjectStore, cfg *BackupS3Config, archivePath string) error {
|
||||
info, err := os.Stat(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("stat backup archive: %w", err)
|
||||
}
|
||||
partSize := s.partSizeBytes
|
||||
if partSize <= 0 {
|
||||
partSize = defaultBackupPartSizeBytes
|
||||
}
|
||||
if info.Size() <= partSize {
|
||||
if _, err := objectStore.UploadFile(ctx, record.S3Key, archivePath, "application/gzip"); err != nil {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), backupObjectCleanupTimeout)
|
||||
cleanupErr := deleteBackupObjectKeys(cleanupCtx, objectStore, record)
|
||||
cleanupCancel()
|
||||
return errors.Join(fmt.Errorf("backup upload: %w", err), cleanupErr)
|
||||
}
|
||||
record.Parts = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
localParts, err := splitBackupFile(archivePath, partSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("split backup archive: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
paths := make([]string, 0, len(localParts))
|
||||
for _, part := range localParts {
|
||||
paths = append(paths, part.Path)
|
||||
}
|
||||
_ = cleanupBackupFiles(paths...)
|
||||
}()
|
||||
if cfg == nil {
|
||||
return errors.New("backup S3 config is unavailable for split upload")
|
||||
}
|
||||
|
||||
record.S3Key = ""
|
||||
record.Parts = make([]BackupPart, 0, len(localParts))
|
||||
partRoot := strings.TrimRight(s.buildS3Key(cfg, record.ID), "/")
|
||||
for _, part := range localParts {
|
||||
record.Parts = append(record.Parts, BackupPart{
|
||||
Index: part.Index,
|
||||
S3Key: s.buildBackupPartKey(partRoot, part.Index),
|
||||
SizeBytes: part.SizeBytes,
|
||||
SHA256: part.SHA256,
|
||||
})
|
||||
}
|
||||
if err := s.saveRecord(ctx, record); err != nil {
|
||||
return fmt.Errorf("save split backup plan: %w", err)
|
||||
}
|
||||
for i, part := range localParts {
|
||||
if _, err := objectStore.UploadFile(ctx, record.Parts[i].S3Key, part.Path, "application/octet-stream"); err != nil {
|
||||
// PUT 可能已经在对象存储端成功、但客户端因超时收到错误;
|
||||
// 因此失败时清理整份分卷计划,而不只清理此前返回成功的卷。
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), backupObjectCleanupTimeout)
|
||||
cleanupErr := deleteBackupObjectKeys(cleanupCtx, objectStore, record)
|
||||
cleanupCancel()
|
||||
return errors.Join(fmt.Errorf("upload backup part %d: %w", part.Index, err), cleanupErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BackupService) buildBackupPartKey(root string, index int) string {
|
||||
return fmt.Sprintf("%s/payload.part-%06d", strings.TrimRight(root, "/"), index)
|
||||
}
|
||||
|
||||
// RestoreBackup 从 S3 下载备份并流式恢复到数据库
|
||||
func (s *BackupService) RestoreBackup(ctx context.Context, backupID string) error {
|
||||
s.opMu.Lock()
|
||||
@@ -770,7 +848,16 @@ func (s *BackupService) RestoreBackup(ctx context.Context, backupID string) erro
|
||||
return fmt.Errorf("init object store: %w", err)
|
||||
}
|
||||
|
||||
// 从 S3 流式下载
|
||||
if len(record.Parts) > 0 {
|
||||
archivePath, err := s.downloadBackupParts(ctx, objectStore, record.Parts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = cleanupBackupFiles(archivePath) }()
|
||||
return s.restoreArchive(ctx, archivePath)
|
||||
}
|
||||
|
||||
// 旧记录从 S3 流式下载
|
||||
body, err := objectStore.Download(ctx, record.S3Key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("S3 download failed: %w", err)
|
||||
@@ -866,6 +953,29 @@ func (s *BackupService) executeRestore(record *BackupRecord, objectStore BackupO
|
||||
ctx, cancel := context.WithTimeout(s.bgCtx, 30*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
if len(record.Parts) > 0 {
|
||||
archivePath, err := s.downloadBackupParts(ctx, objectStore, record.Parts)
|
||||
if err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = err.Error()
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
defer func() { _ = cleanupBackupFiles(archivePath) }()
|
||||
if err := s.restoreArchive(ctx, archivePath); err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = fmt.Sprintf("pg restore: %v", err)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
record.RestoreStatus = "completed"
|
||||
record.RestoredAt = time.Now().Format(time.RFC3339)
|
||||
if err := s.saveRecord(context.Background(), record); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存恢复记录失败: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
body, err := objectStore.Download(ctx, record.S3Key)
|
||||
if err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
@@ -898,6 +1008,79 @@ func (s *BackupService) executeRestore(record *BackupRecord, objectStore BackupO
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BackupService) downloadBackupParts(ctx context.Context, objectStore BackupObjectStore, parts []BackupPart) (path string, err error) {
|
||||
if len(parts) == 0 {
|
||||
return "", errors.New("backup parts are empty")
|
||||
}
|
||||
ordered := append([]BackupPart(nil), parts...)
|
||||
sort.Slice(ordered, func(i, j int) bool { return ordered[i].Index < ordered[j].Index })
|
||||
for i, part := range ordered {
|
||||
if part.Index != i+1 || part.S3Key == "" || part.SizeBytes <= 0 {
|
||||
return "", fmt.Errorf("invalid backup part metadata at index %d", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
archive, err := os.CreateTemp("", "sub2api-restore-*.sql.gz")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("create restore archive: %w", err)
|
||||
}
|
||||
path = archive.Name()
|
||||
cleanup := func() {
|
||||
_ = archive.Close()
|
||||
_ = cleanupBackupFiles(path)
|
||||
}
|
||||
|
||||
for _, part := range ordered {
|
||||
body, downloadErr := objectStore.Download(ctx, part.S3Key)
|
||||
if downloadErr != nil {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("download backup part %d: %w", part.Index, downloadErr)
|
||||
}
|
||||
hash := sha256.New()
|
||||
written, copyErr := io.Copy(io.MultiWriter(archive, hash), body)
|
||||
closeErr := body.Close()
|
||||
if copyErr != nil {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("read backup part %d: %w", part.Index, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("close backup part %d: %w", part.Index, closeErr)
|
||||
}
|
||||
if written != part.SizeBytes {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("backup part %d size mismatch: got %d, want %d", part.Index, written, part.SizeBytes)
|
||||
}
|
||||
if part.SHA256 != "" && !strings.EqualFold(part.SHA256, hex.EncodeToString(hash.Sum(nil))) {
|
||||
cleanup()
|
||||
return "", fmt.Errorf("backup part %d checksum mismatch", part.Index)
|
||||
}
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
_ = cleanupBackupFiles(path)
|
||||
return "", fmt.Errorf("close restore archive: %w", err)
|
||||
}
|
||||
return path, nil
|
||||
}
|
||||
|
||||
func (s *BackupService) restoreArchive(ctx context.Context, archivePath string) error {
|
||||
archive, err := os.Open(archivePath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open restore archive: %w", err)
|
||||
}
|
||||
defer func() { _ = archive.Close() }()
|
||||
|
||||
gzReader, err := gzip.NewReader(archive)
|
||||
if err != nil {
|
||||
return fmt.Errorf("gzip reader: %w", err)
|
||||
}
|
||||
defer func() { _ = gzReader.Close() }()
|
||||
if err := s.dumper.Restore(ctx, gzReader); err != nil {
|
||||
return fmt.Errorf("pg restore: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ─── 备份记录管理 ───
|
||||
|
||||
func (s *BackupService) ListBackups(ctx context.Context) ([]BackupRecord, error) {
|
||||
@@ -946,45 +1129,67 @@ func (s *BackupService) DeleteBackup(ctx context.Context, backupID string) error
|
||||
if found == nil {
|
||||
return ErrBackupNotFound
|
||||
}
|
||||
if found.Status == "running" {
|
||||
// 后台上传仍可能依赖 Parts 计划;删除对象会让随后完成的记录引用失效卷。
|
||||
return ErrBackupInProgress
|
||||
}
|
||||
|
||||
// 从 S3 删除
|
||||
if found.S3Key != "" && found.Status == "completed" {
|
||||
s3Cfg, err := s.loadS3Config(ctx)
|
||||
if err == nil && s3Cfg != nil && s3Cfg.IsConfigured() {
|
||||
objectStore, err := s.getOrCreateStore(ctx, s3Cfg)
|
||||
if err == nil {
|
||||
_ = objectStore.Delete(ctx, found.S3Key)
|
||||
}
|
||||
}
|
||||
// 从对象存储删除所有单文件或分卷对象。删除不完整时保留记录,便于重试。
|
||||
if err := s.deleteBackupObjects(ctx, found); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.saveRecordsLocked(ctx, remaining)
|
||||
}
|
||||
|
||||
// GetBackupDownloadURL 获取备份文件预签名下载 URL
|
||||
func (s *BackupService) GetBackupDownloadURL(ctx context.Context, backupID string) (string, error) {
|
||||
func (s *BackupService) GetBackupDownloadURL(ctx context.Context, backupID string) (BackupDownloadResponse, error) {
|
||||
var download BackupDownloadResponse
|
||||
record, err := s.GetBackupRecord(ctx, backupID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return download, err
|
||||
}
|
||||
if record.Status != "completed" {
|
||||
return "", infraerrors.BadRequest("BACKUP_NOT_COMPLETED", "backup is not completed")
|
||||
return download, infraerrors.BadRequest("BACKUP_NOT_COMPLETED", "backup is not completed")
|
||||
}
|
||||
|
||||
s3Cfg, err := s.loadS3Config(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return download, err
|
||||
}
|
||||
objectStore, err := s.getOrCreateStore(ctx, s3Cfg)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return download, err
|
||||
}
|
||||
|
||||
if len(record.Parts) > 0 {
|
||||
parts := append([]BackupPart(nil), record.Parts...)
|
||||
sort.Slice(parts, func(i, j int) bool { return parts[i].Index < parts[j].Index })
|
||||
for i, part := range parts {
|
||||
if part.Index != i+1 || part.S3Key == "" || part.SizeBytes <= 0 {
|
||||
return download, fmt.Errorf("invalid backup part metadata at index %d", i+1)
|
||||
}
|
||||
url, presignErr := objectStore.PresignURL(ctx, part.S3Key, 1*time.Hour)
|
||||
if presignErr != nil {
|
||||
return download, fmt.Errorf("presign backup part %d: %w", part.Index, presignErr)
|
||||
}
|
||||
download.Parts = append(download.Parts, BackupDownloadPart{
|
||||
Index: part.Index,
|
||||
SizeBytes: part.SizeBytes,
|
||||
URL: url,
|
||||
})
|
||||
}
|
||||
return download, nil
|
||||
}
|
||||
if record.S3Key == "" {
|
||||
return download, errors.New("backup object key is empty")
|
||||
}
|
||||
url, err := objectStore.PresignURL(ctx, record.S3Key, 1*time.Hour)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("presign url: %w", err)
|
||||
return download, fmt.Errorf("presign url: %w", err)
|
||||
}
|
||||
return url, nil
|
||||
download.URL = url
|
||||
return download, nil
|
||||
}
|
||||
|
||||
// ─── 内部方法 ───
|
||||
@@ -1141,28 +1346,87 @@ func (s *BackupService) cleanupOldBackups(ctx context.Context, schedule *BackupS
|
||||
}
|
||||
}
|
||||
|
||||
// 删除 S3 上的文件
|
||||
var cleanupErrs []error
|
||||
deletedCount := 0
|
||||
for _, r := range toDelete {
|
||||
if r.S3Key != "" {
|
||||
_ = s.deleteS3Object(ctx, r.S3Key)
|
||||
if err := s.deleteBackupObjects(ctx, &r); err != nil {
|
||||
// 对象删除失败时保留记录,避免丢失后续重试所需的 key。
|
||||
toKeep = append(toKeep, r)
|
||||
cleanupErrs = append(cleanupErrs, fmt.Errorf("cleanup backup %s: %w", r.ID, err))
|
||||
continue
|
||||
}
|
||||
deletedCount++
|
||||
}
|
||||
|
||||
if len(toDelete) > 0 {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 自动清理了 %d 个过期备份", len(toDelete))
|
||||
return s.saveRecordsLocked(ctx, toKeep)
|
||||
if err := s.saveRecordsLocked(ctx, toKeep); err != nil {
|
||||
cleanupErrs = append(cleanupErrs, fmt.Errorf("save backup records after cleanup: %w", err))
|
||||
}
|
||||
if deletedCount > 0 {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 自动清理了 %d 个过期备份", deletedCount)
|
||||
}
|
||||
return errors.Join(cleanupErrs...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *BackupService) deleteS3Object(ctx context.Context, key string) error {
|
||||
// backupObjectKeys 返回一条备份记录关联的全部对象 key。
|
||||
// 新记录使用 Parts,旧记录使用 S3Key;两者同时存在时也全部返回,便于清理异常残留对象。
|
||||
func backupObjectKeys(record *BackupRecord) []string {
|
||||
if record == nil {
|
||||
return nil
|
||||
}
|
||||
keys := make([]string, 0, len(record.Parts)+1)
|
||||
seen := make(map[string]struct{}, len(record.Parts)+1)
|
||||
appendKey := func(key string) {
|
||||
if key == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
return
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
appendKey(record.S3Key)
|
||||
parts := append([]BackupPart(nil), record.Parts...)
|
||||
sort.Slice(parts, func(i, j int) bool { return parts[i].Index < parts[j].Index })
|
||||
for _, part := range parts {
|
||||
appendKey(part.S3Key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// deleteBackupObjects 尝试删除记录关联的所有对象,并聚合删除错误。
|
||||
func (s *BackupService) deleteBackupObjects(ctx context.Context, record *BackupRecord) error {
|
||||
if len(backupObjectKeys(record)) == 0 {
|
||||
return nil
|
||||
}
|
||||
s3Cfg, err := s.loadS3Config(ctx)
|
||||
if err != nil || s3Cfg == nil {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s3Cfg == nil || !s3Cfg.IsConfigured() {
|
||||
// 兼容没有配置对象存储的旧记录:记录仍可被删除。
|
||||
return nil
|
||||
}
|
||||
objectStore, err := s.getOrCreateStore(ctx, s3Cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return objectStore.Delete(ctx, key)
|
||||
return deleteBackupObjectKeys(ctx, objectStore, record)
|
||||
}
|
||||
|
||||
func deleteBackupObjectKeys(ctx context.Context, objectStore BackupObjectStore, record *BackupRecord) error {
|
||||
keys := backupObjectKeys(record)
|
||||
if len(keys) == 0 {
|
||||
return nil
|
||||
}
|
||||
var errs []error
|
||||
for _, key := range keys {
|
||||
if deleteErr := objectStore.Delete(ctx, key); deleteErr != nil {
|
||||
errs = append(errs, fmt.Errorf("delete backup object %q: %w", key, deleteErr))
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
@@ -4,10 +4,13 @@ package service
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -159,12 +162,49 @@ func (d *blockingDumper) Restore(_ context.Context, data io.Reader) error {
|
||||
}
|
||||
|
||||
type mockObjectStore struct {
|
||||
objects map[string][]byte
|
||||
mu sync.Mutex
|
||||
objects map[string][]byte
|
||||
mu sync.Mutex
|
||||
failUploadFileAt int
|
||||
uploadFileCalls int
|
||||
deletedKeys []string
|
||||
failDeleteKeys map[string]error
|
||||
}
|
||||
|
||||
type cancelingUploadFailureStore struct {
|
||||
*mockObjectStore
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (m *cancelingUploadFailureStore) UploadFile(_ context.Context, key string, filePath string, _ string) (int64, error) {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
data, readErr := io.ReadAll(file)
|
||||
closeErr := file.Close()
|
||||
if readErr != nil {
|
||||
return 0, readErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return 0, closeErr
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.objects[key] = data
|
||||
m.mu.Unlock()
|
||||
m.cancel()
|
||||
return 0, fmt.Errorf("injected upload failure after object landed")
|
||||
}
|
||||
|
||||
func (m *cancelingUploadFailureStore) Delete(ctx context.Context, key string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.mockObjectStore.Delete(ctx, key)
|
||||
}
|
||||
|
||||
func newMockObjectStore() *mockObjectStore {
|
||||
return &mockObjectStore{objects: make(map[string][]byte)}
|
||||
return &mockObjectStore{objects: make(map[string][]byte), failDeleteKeys: make(map[string]error)}
|
||||
}
|
||||
|
||||
func (m *mockObjectStore) Upload(_ context.Context, key string, body io.Reader, _ string) (int64, error) {
|
||||
@@ -178,6 +218,23 @@ func (m *mockObjectStore) Upload(_ context.Context, key string, body io.Reader,
|
||||
return int64(len(data)), nil
|
||||
}
|
||||
|
||||
func (m *mockObjectStore) UploadFile(ctx context.Context, key string, filePath string, contentType string) (int64, error) {
|
||||
m.mu.Lock()
|
||||
m.uploadFileCalls++
|
||||
call := m.uploadFileCalls
|
||||
failAt := m.failUploadFileAt
|
||||
m.mu.Unlock()
|
||||
if failAt > 0 && call == failAt {
|
||||
return 0, fmt.Errorf("injected upload failure at call %d", call)
|
||||
}
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
return m.Upload(ctx, key, file, contentType)
|
||||
}
|
||||
|
||||
func (m *mockObjectStore) Download(_ context.Context, key string) (io.ReadCloser, error) {
|
||||
m.mu.Lock()
|
||||
data, ok := m.objects[key]
|
||||
@@ -190,6 +247,11 @@ func (m *mockObjectStore) Download(_ context.Context, key string) (io.ReadCloser
|
||||
|
||||
func (m *mockObjectStore) Delete(_ context.Context, key string) error {
|
||||
m.mu.Lock()
|
||||
m.deletedKeys = append(m.deletedKeys, key)
|
||||
if err, ok := m.failDeleteKeys[key]; ok {
|
||||
m.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
delete(m.objects, key)
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
@@ -405,6 +467,122 @@ func TestBackupService_CreateBackup_Streaming(t *testing.T) {
|
||||
store.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestBackupService_CreateBackup_SplitsCompressedArchive(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
dumpContent := entropyBackupFixture(512)
|
||||
dumper := &mockDumper{dumpData: dumpContent}
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, dumper, store)
|
||||
svc.partSizeBytes = 32
|
||||
|
||||
record, err := svc.CreateBackup(context.Background(), "manual", 14)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "completed", record.Status)
|
||||
require.Greater(t, len(record.Parts), 1)
|
||||
require.Empty(t, record.S3Key)
|
||||
|
||||
var compressed bytes.Buffer
|
||||
store.mu.Lock()
|
||||
for _, part := range record.Parts {
|
||||
data, ok := store.objects[part.S3Key]
|
||||
require.True(t, ok)
|
||||
require.LessOrEqual(t, len(data), 32)
|
||||
compressed.Write(data)
|
||||
}
|
||||
store.mu.Unlock()
|
||||
|
||||
gzReader, err := gzip.NewReader(bytes.NewReader(compressed.Bytes()))
|
||||
require.NoError(t, err)
|
||||
decompressed, err := io.ReadAll(gzReader)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gzReader.Close())
|
||||
require.Equal(t, dumpContent, decompressed)
|
||||
}
|
||||
|
||||
func TestBackupService_StartBackup_SplitsCompressedArchive(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{dumpData: entropyBackupFixture(512)}, store)
|
||||
svc.partSizeBytes = 32
|
||||
|
||||
record, err := svc.StartBackup(context.Background(), "manual", 14)
|
||||
require.NoError(t, err)
|
||||
svc.wg.Wait()
|
||||
|
||||
final, err := svc.GetBackupRecord(context.Background(), record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "completed", final.Status)
|
||||
require.Greater(t, len(final.Parts), 1)
|
||||
require.Empty(t, final.S3Key)
|
||||
}
|
||||
|
||||
func TestBackupService_StartBackup_UploadFailureCleansUploadedParts(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
store.failUploadFileAt = 2
|
||||
svc := newTestBackupService(repo, &mockDumper{dumpData: entropyBackupFixture(512)}, store)
|
||||
svc.partSizeBytes = 32
|
||||
|
||||
record, err := svc.StartBackup(context.Background(), "manual", 14)
|
||||
require.NoError(t, err)
|
||||
svc.wg.Wait()
|
||||
|
||||
final, err := svc.GetBackupRecord(context.Background(), record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "failed", final.Status)
|
||||
require.NotEmpty(t, final.Parts)
|
||||
store.mu.Lock()
|
||||
deletedKeys := append([]string(nil), store.deletedKeys...)
|
||||
store.mu.Unlock()
|
||||
for _, part := range final.Parts {
|
||||
require.Contains(t, deletedKeys, part.S3Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupService_UploadFailureCleanupUsesDetachedContext(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
svc.partSizeBytes = 4
|
||||
|
||||
archive, err := os.CreateTemp("", "backup-upload-context-*.gz")
|
||||
require.NoError(t, err)
|
||||
archivePath := archive.Name()
|
||||
defer func() { _ = os.Remove(archivePath) }()
|
||||
_, err = archive.Write([]byte("0123456789"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, archive.Close())
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
store := &cancelingUploadFailureStore{
|
||||
mockObjectStore: newMockObjectStore(),
|
||||
cancel: cancel,
|
||||
}
|
||||
record := &BackupRecord{ID: "cancel-cleanup", S3Key: "backups/cancel-cleanup.sql.gz"}
|
||||
|
||||
err = svc.uploadBackupArchive(ctx, record, store, &BackupS3Config{Prefix: "backups"}, archivePath)
|
||||
require.Error(t, err)
|
||||
require.NotContains(t, err.Error(), "context canceled")
|
||||
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
for _, part := range record.Parts {
|
||||
require.Contains(t, store.deletedKeys, part.S3Key)
|
||||
require.NotContains(t, store.objects, part.S3Key)
|
||||
}
|
||||
}
|
||||
|
||||
func entropyBackupFixture(size int) []byte {
|
||||
data := make([]byte, size)
|
||||
for i := range data {
|
||||
data[i] = byte((i*31 + 17) % 251)
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func TestBackupService_CreateBackup_DumpFailure(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
@@ -466,6 +644,123 @@ func TestBackupService_RestoreBackup_Streaming(t *testing.T) {
|
||||
require.Equal(t, dumpContent, string(dumper.restored))
|
||||
}
|
||||
|
||||
func TestBackupService_RestoreBackup_SplitParts(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
dumpContent := entropyBackupFixture(512)
|
||||
dumper := &mockDumper{}
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, dumper, store)
|
||||
|
||||
compressed := gzipBackupBytes(t, dumpContent)
|
||||
parts := splitBackupBytes(compressed, 11)
|
||||
recordParts := make([]BackupPart, 0, len(parts))
|
||||
for i, data := range parts {
|
||||
key := fmt.Sprintf("backups/split-1/payload.part-%06d", i+1)
|
||||
store.objects[key] = data
|
||||
recordParts = append(recordParts, BackupPart{
|
||||
Index: i + 1,
|
||||
S3Key: key,
|
||||
SizeBytes: int64(len(data)),
|
||||
SHA256: fmt.Sprintf("%x", sha256.Sum256(data)),
|
||||
})
|
||||
}
|
||||
record := &BackupRecord{
|
||||
ID: "split-1",
|
||||
Status: "completed",
|
||||
Parts: recordParts,
|
||||
SizeBytes: int64(len(compressed)),
|
||||
}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), record))
|
||||
|
||||
require.NoError(t, svc.RestoreBackup(context.Background(), record.ID))
|
||||
require.Equal(t, dumpContent, dumper.restored)
|
||||
}
|
||||
|
||||
func TestBackupService_RestoreBackup_SplitPartsMissingPartDoesNotRestore(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
dumpContent := entropyBackupFixture(256)
|
||||
dumper := &mockDumper{}
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, dumper, store)
|
||||
|
||||
compressed := gzipBackupBytes(t, dumpContent)
|
||||
parts := splitBackupBytes(compressed, 11)
|
||||
recordParts := make([]BackupPart, 0, len(parts))
|
||||
for i, data := range parts {
|
||||
key := fmt.Sprintf("backups/split-missing/payload.part-%06d", i+1)
|
||||
store.objects[key] = data
|
||||
recordParts = append(recordParts, BackupPart{
|
||||
Index: i + 1,
|
||||
S3Key: key,
|
||||
SizeBytes: int64(len(data)),
|
||||
SHA256: fmt.Sprintf("%x", sha256.Sum256(data)),
|
||||
})
|
||||
}
|
||||
delete(store.objects, recordParts[1].S3Key)
|
||||
record := &BackupRecord{ID: "split-missing", Status: "completed", Parts: recordParts}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), record))
|
||||
|
||||
require.Error(t, svc.RestoreBackup(context.Background(), record.ID))
|
||||
require.Empty(t, dumper.restored)
|
||||
}
|
||||
|
||||
func TestBackupService_DownloadBackupPartsRejectsMismatchedMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
part BackupPart
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "size",
|
||||
part: BackupPart{Index: 1, S3Key: "backups/mismatch/size", SizeBytes: 4},
|
||||
want: "size mismatch",
|
||||
},
|
||||
{
|
||||
name: "checksum",
|
||||
part: BackupPart{Index: 1, S3Key: "backups/mismatch/checksum", SizeBytes: 3, SHA256: "bad-checksum"},
|
||||
want: "checksum mismatch",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
store.objects[tt.part.S3Key] = []byte("abc")
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
|
||||
_, err := svc.downloadBackupParts(context.Background(), store, []BackupPart{tt.part})
|
||||
require.ErrorContains(t, err, tt.want)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func gzipBackupBytes(t *testing.T, content []byte) []byte {
|
||||
t.Helper()
|
||||
var out bytes.Buffer
|
||||
writer := gzip.NewWriter(&out)
|
||||
_, err := writer.Write(content)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, writer.Close())
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
func splitBackupBytes(data []byte, partSize int) [][]byte {
|
||||
parts := make([][]byte, 0, (len(data)+partSize-1)/partSize)
|
||||
for len(data) > 0 {
|
||||
size := partSize
|
||||
if len(data) < size {
|
||||
size = len(data)
|
||||
}
|
||||
parts = append(parts, append([]byte(nil), data[:size]...))
|
||||
data = data[size:]
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func TestBackupService_RestoreBackup_NotCompleted(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
@@ -512,6 +807,34 @@ func TestBackupService_DeleteBackup(t *testing.T) {
|
||||
require.ErrorIs(t, err, ErrBackupNotFound)
|
||||
}
|
||||
|
||||
func TestBackupService_DeleteBackup_RunningKeepsUploadObjects(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
parts := []BackupPart{
|
||||
{Index: 1, S3Key: "backups/running/payload.part-000001", SizeBytes: 3},
|
||||
{Index: 2, S3Key: "backups/running/payload.part-000002", SizeBytes: 3},
|
||||
}
|
||||
for _, part := range parts {
|
||||
store.objects[part.S3Key] = []byte("abc")
|
||||
}
|
||||
record := &BackupRecord{ID: "running-delete", Status: "running", Parts: parts}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), record))
|
||||
|
||||
err := svc.DeleteBackup(context.Background(), record.ID)
|
||||
require.ErrorIs(t, err, ErrBackupInProgress)
|
||||
store.mu.Lock()
|
||||
require.Empty(t, store.deletedKeys)
|
||||
for _, part := range parts {
|
||||
require.Contains(t, store.objects, part.S3Key)
|
||||
}
|
||||
store.mu.Unlock()
|
||||
got, getErr := svc.GetBackupRecord(context.Background(), record.ID)
|
||||
require.NoError(t, getErr)
|
||||
require.Equal(t, "running", got.Status)
|
||||
}
|
||||
|
||||
func TestBackupService_GetDownloadURL(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
@@ -523,9 +846,103 @@ func TestBackupService_GetDownloadURL(t *testing.T) {
|
||||
record, err := svc.CreateBackup(context.Background(), "manual", 14)
|
||||
require.NoError(t, err)
|
||||
|
||||
url, err := svc.GetBackupDownloadURL(context.Background(), record.ID)
|
||||
download, err := svc.GetBackupDownloadURL(context.Background(), record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, url, "https://presigned.example.com/")
|
||||
require.Contains(t, download.URL, "https://presigned.example.com/")
|
||||
}
|
||||
|
||||
func TestBackupService_DeleteBackup_SplitPartsFailureKeepsRecord(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
|
||||
parts := []BackupPart{
|
||||
{Index: 1, S3Key: "backups/split/payload.part-000001", SizeBytes: 3},
|
||||
{Index: 2, S3Key: "backups/split/payload.part-000002", SizeBytes: 3},
|
||||
{Index: 3, S3Key: "backups/split/payload.part-000003", SizeBytes: 3},
|
||||
}
|
||||
for _, part := range parts {
|
||||
store.objects[part.S3Key] = []byte("abc")
|
||||
}
|
||||
store.failDeleteKeys[parts[1].S3Key] = fmt.Errorf("delete failed")
|
||||
record := &BackupRecord{ID: "split-delete", Status: "completed", Parts: parts}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), record))
|
||||
|
||||
err := svc.DeleteBackup(context.Background(), record.ID)
|
||||
require.Error(t, err)
|
||||
|
||||
store.mu.Lock()
|
||||
deleted := append([]string(nil), store.deletedKeys...)
|
||||
store.mu.Unlock()
|
||||
for _, part := range parts {
|
||||
require.Contains(t, deleted, part.S3Key)
|
||||
}
|
||||
got, getErr := svc.GetBackupRecord(context.Background(), record.ID)
|
||||
require.NoError(t, getErr)
|
||||
require.Equal(t, record.ID, got.ID)
|
||||
store.mu.Lock()
|
||||
require.Contains(t, store.objects, parts[1].S3Key)
|
||||
store.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestBackupService_GetDownloadURL_SplitParts(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
|
||||
parts := []BackupPart{
|
||||
{Index: 2, S3Key: "backups/split/payload.part-000002", SizeBytes: 7},
|
||||
{Index: 1, S3Key: "backups/split/payload.part-000001", SizeBytes: 5},
|
||||
}
|
||||
record := &BackupRecord{ID: "split-download", Status: "completed", Parts: parts}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), record))
|
||||
|
||||
download, err := svc.GetBackupDownloadURL(context.Background(), record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, download.URL)
|
||||
require.Len(t, download.Parts, 2)
|
||||
require.Equal(t, 1, download.Parts[0].Index)
|
||||
require.Equal(t, int64(5), download.Parts[0].SizeBytes)
|
||||
require.Equal(t, "https://presigned.example.com/backups/split/payload.part-000001", download.Parts[0].URL)
|
||||
require.Equal(t, 2, download.Parts[1].Index)
|
||||
}
|
||||
|
||||
func TestBackupService_CleanupOldBackups_SplitParts(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
now := time.Now()
|
||||
parts := []BackupPart{
|
||||
{Index: 1, S3Key: "backups/old/payload.part-000001", SizeBytes: 3},
|
||||
{Index: 2, S3Key: "backups/old/payload.part-000002", SizeBytes: 3},
|
||||
}
|
||||
for _, part := range parts {
|
||||
store.objects[part.S3Key] = []byte("abc")
|
||||
}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), &BackupRecord{
|
||||
ID: "new",
|
||||
Status: "completed",
|
||||
StartedAt: now.Format(time.RFC3339),
|
||||
}))
|
||||
require.NoError(t, svc.saveRecord(context.Background(), &BackupRecord{
|
||||
ID: "old",
|
||||
Status: "completed",
|
||||
StartedAt: now.Add(-2 * time.Hour).Format(time.RFC3339),
|
||||
Parts: parts,
|
||||
}))
|
||||
|
||||
err := svc.cleanupOldBackups(context.Background(), &BackupScheduleConfig{RetainCount: 1})
|
||||
require.NoError(t, err)
|
||||
_, err = svc.GetBackupRecord(context.Background(), "old")
|
||||
require.ErrorIs(t, err, ErrBackupNotFound)
|
||||
store.mu.Lock()
|
||||
for _, part := range parts {
|
||||
require.NotContains(t, store.objects, part.S3Key)
|
||||
}
|
||||
store.mu.Unlock()
|
||||
}
|
||||
|
||||
func TestBackupService_ListBackups_Sorted(t *testing.T) {
|
||||
@@ -690,6 +1107,65 @@ func TestRecoverStaleRecords(t *testing.T) {
|
||||
require.Contains(t, r2.RestoreError, "server restart")
|
||||
}
|
||||
|
||||
func TestBackupService_RecoverStaleRecords_CleansBackupObjects(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
parts := []BackupPart{
|
||||
{Index: 1, S3Key: "backups/stale/payload.part-000001", SizeBytes: 3},
|
||||
{Index: 2, S3Key: "backups/stale/payload.part-000002", SizeBytes: 3},
|
||||
}
|
||||
for _, part := range parts {
|
||||
store.objects[part.S3Key] = []byte("abc")
|
||||
}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), &BackupRecord{
|
||||
ID: "stale-parts",
|
||||
Status: "running",
|
||||
Parts: parts,
|
||||
StartedAt: time.Now().Add(-time.Hour).Format(time.RFC3339),
|
||||
}))
|
||||
|
||||
svc.recoverStaleRecords()
|
||||
|
||||
record, err := svc.GetBackupRecord(context.Background(), "stale-parts")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "failed", record.Status)
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
for _, part := range parts {
|
||||
require.Contains(t, store.deletedKeys, part.S3Key)
|
||||
require.NotContains(t, store.objects, part.S3Key)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupService_RecoverStaleRecords_PreservesKeysWhenCleanupFails(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
part := BackupPart{Index: 1, S3Key: "backups/stale-failed/payload.part-000001", SizeBytes: 3}
|
||||
store.objects[part.S3Key] = []byte("abc")
|
||||
store.failDeleteKeys[part.S3Key] = fmt.Errorf("delete failed")
|
||||
require.NoError(t, svc.saveRecord(context.Background(), &BackupRecord{
|
||||
ID: "stale-cleanup-failed",
|
||||
Status: "running",
|
||||
Parts: []BackupPart{part},
|
||||
StartedAt: time.Now().Add(-time.Hour).Format(time.RFC3339),
|
||||
}))
|
||||
|
||||
svc.recoverStaleRecords()
|
||||
|
||||
record, err := svc.GetBackupRecord(context.Background(), "stale-cleanup-failed")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "failed", record.Status)
|
||||
require.Contains(t, record.ErrorMsg, "cleanup failed")
|
||||
require.Equal(t, part.S3Key, record.Parts[0].S3Key)
|
||||
store.mu.Lock()
|
||||
defer store.mu.Unlock()
|
||||
require.Contains(t, store.objects, part.S3Key)
|
||||
}
|
||||
|
||||
func TestGracefulShutdown(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
@@ -753,3 +1229,38 @@ func TestStartRestore_Async(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "completed", final.RestoreStatus)
|
||||
}
|
||||
|
||||
func TestBackupService_StartRestore_SplitParts(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
dumpContent := entropyBackupFixture(384)
|
||||
dumper := &mockDumper{}
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, dumper, store)
|
||||
|
||||
compressed := gzipBackupBytes(t, dumpContent)
|
||||
parts := splitBackupBytes(compressed, 13)
|
||||
recordParts := make([]BackupPart, 0, len(parts))
|
||||
for i, data := range parts {
|
||||
key := fmt.Sprintf("backups/split-async/payload.part-%06d", i+1)
|
||||
store.objects[key] = data
|
||||
recordParts = append(recordParts, BackupPart{
|
||||
Index: i + 1,
|
||||
S3Key: key,
|
||||
SizeBytes: int64(len(data)),
|
||||
SHA256: fmt.Sprintf("%x", sha256.Sum256(data)),
|
||||
})
|
||||
}
|
||||
record := &BackupRecord{ID: "split-async", Status: "completed", Parts: recordParts}
|
||||
require.NoError(t, svc.saveRecord(context.Background(), record))
|
||||
|
||||
started, err := svc.StartRestore(context.Background(), record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "running", started.RestoreStatus)
|
||||
svc.wg.Wait()
|
||||
|
||||
final, err := svc.GetBackupRecord(context.Background(), record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "completed", final.RestoreStatus)
|
||||
require.Equal(t, dumpContent, dumper.restored)
|
||||
}
|
||||
|
||||
@@ -41,6 +41,14 @@ const (
|
||||
ContentModerationActionError = "error"
|
||||
ContentModerationActionCyberPolicy = "cyber_policy" // cyber_policy 硬阻断的风控日志 action(封号计数排除按此值过滤)
|
||||
|
||||
// ContentModerationErrorCodeUnavailable 标记"审计链路自身故障导致的拒绝"。
|
||||
// 与 content_policy_violation 区分开:前者是系统不可用(503,客户端可重试),
|
||||
// 后者是内容命中策略(403,重试无用)。
|
||||
ContentModerationErrorCodeUnavailable = "content_moderation_unavailable"
|
||||
// ContentModerationUnavailableMessage 是 fail-closed 时返回给客户端的文案。
|
||||
// 不复用 cfg.BlockMessage:那是给违规内容看的,会误导用户以为自己触犯了策略。
|
||||
ContentModerationUnavailableMessage = "风控审计服务暂时不可用,请稍后重试"
|
||||
|
||||
contentModerationKeywordCategory = "keyword"
|
||||
|
||||
ContentModerationKeywordModeKeywordOnly = "keyword_only"
|
||||
@@ -172,6 +180,36 @@ type ContentModerationConfig struct {
|
||||
// 当次不判定封号,且历史 cyber 行在 CountFlaggedByUserSince 中被排除。
|
||||
// 默认 false(计入,与历史行为一致;旧配置 JSON 无此字段时反序列化为 false)。
|
||||
CyberPolicyExcludeFromBanCount bool `json:"cyber_policy_exclude_from_ban_count"`
|
||||
// FailClosedOnError 为 true 时,pre_block 模式下审计链路自身异常(审计 API 调用
|
||||
// 失败、无可用审计 Key)拒绝请求而不是放行,返回 503 +
|
||||
// content_moderation_unavailable。默认 false(放行,与历史行为一致;旧配置 JSON
|
||||
// 无此字段时反序列化为 false)——这是可用性开关,必须由运维显式打开。
|
||||
FailClosedOnError bool `json:"fail_closed_on_error"`
|
||||
}
|
||||
|
||||
// failClosedOnError 表示"审计链路自身异常时拒绝请求"的运维意图。
|
||||
//
|
||||
// 三个条件缺一不可:审计总开关开着、运维显式打开了 fail-closed、且处于同步前置
|
||||
// 拦截模式。observe 模式本就没有阻断语义,异步 worker 也没有可拒绝的客户端,
|
||||
// 对它们 fail-closed 毫无意义。
|
||||
func (c *ContentModerationConfig) failClosedOnError() bool {
|
||||
return c != nil && c.Enabled && c.FailClosedOnError && c.Mode == ContentModerationModePreBlock
|
||||
}
|
||||
|
||||
// contentModerationUnavailableDecision 构造 fail-closed 决策。
|
||||
//
|
||||
// Flagged 恒为 false 且调用方不写 flagged 日志行,因此这类拒绝永远不会进入
|
||||
// CountFlaggedByUserSince 的自动封号计数——系统故障不该把用户封掉。
|
||||
func contentModerationUnavailableDecision() *ContentModerationDecision {
|
||||
return &ContentModerationDecision{
|
||||
Allowed: false,
|
||||
Blocked: true,
|
||||
Flagged: false,
|
||||
Message: ContentModerationUnavailableMessage,
|
||||
StatusCode: http.StatusServiceUnavailable,
|
||||
Action: ContentModerationActionError,
|
||||
ErrorCode: ContentModerationErrorCodeUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
type ContentModerationConfigView struct {
|
||||
@@ -207,6 +245,7 @@ type ContentModerationConfigView struct {
|
||||
KeywordBlockingMode string `json:"keyword_blocking_mode"`
|
||||
ModelFilter ContentModerationModelFilter `json:"model_filter"`
|
||||
CyberPolicyExcludeFromBanCount bool `json:"cyber_policy_exclude_from_ban_count"`
|
||||
FailClosedOnError bool `json:"fail_closed_on_error"`
|
||||
}
|
||||
|
||||
type ContentModerationAPIKeyStatus struct {
|
||||
@@ -299,6 +338,7 @@ type UpdateContentModerationConfigInput struct {
|
||||
KeywordBlockingMode *string `json:"keyword_blocking_mode"`
|
||||
ModelFilter *ContentModerationModelFilter `json:"model_filter"`
|
||||
CyberPolicyExcludeFromBanCount *bool `json:"cyber_policy_exclude_from_ban_count"`
|
||||
FailClosedOnError *bool `json:"fail_closed_on_error"`
|
||||
}
|
||||
|
||||
type ContentModerationModelFilter struct {
|
||||
@@ -383,6 +423,10 @@ type ContentModerationDecision struct {
|
||||
HighestScore float64 `json:"highest_score"`
|
||||
CategoryScores map[string]float64 `json:"category_scores"`
|
||||
Action string `json:"action"`
|
||||
// ErrorCode 为空时调用方回退到 content_policy_violation。审计链路自身故障
|
||||
// (fail-closed)填 ContentModerationErrorCodeUnavailable,避免把系统不可用
|
||||
// 伪装成内容违规。
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
}
|
||||
|
||||
type ContentModerationLog struct {
|
||||
@@ -699,6 +743,9 @@ func (s *ContentModerationService) UpdateConfig(ctx context.Context, input Updat
|
||||
if input.CyberPolicyExcludeFromBanCount != nil {
|
||||
cfg.CyberPolicyExcludeFromBanCount = *input.CyberPolicyExcludeFromBanCount
|
||||
}
|
||||
if input.FailClosedOnError != nil {
|
||||
cfg.FailClosedOnError = *input.FailClosedOnError
|
||||
}
|
||||
if input.Thresholds != nil {
|
||||
cfg.Thresholds = mergeContentModerationThresholds(ContentModerationDefaultThresholds(), *input.Thresholds)
|
||||
}
|
||||
@@ -824,7 +871,13 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
}
|
||||
runtimeSnapshot, err := s.loadRuntimeSnapshot(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("content_moderation.skip_config_load_failed",
|
||||
// 这里必须放行。loadRuntimeSnapshot 只在"进程冷启动、快照尚未建立"时返回错误
|
||||
// (已有快照时它返回陈旧快照并后台刷新,不返回错误),此刻我们既不知道
|
||||
// risk_control_enabled,也不知道 fail_closed_on_error,无法确认运维的阻断意图。
|
||||
// 若在此拦截,一次设置读取抖动、或一行损坏的 content_moderation_config JSON,
|
||||
// 就会让全部网关流量(含从未开启风控的部署)返回 5xx。
|
||||
// 审计链路自身异常的 fail-closed 由 cfg.failClosedOnError() 在配置已知时处理。
|
||||
slog.Warn("content_moderation.allow_config_load_failed",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
"group_id", contentModerationLogGroupID(input.GroupID),
|
||||
@@ -1021,6 +1074,15 @@ func (s *ContentModerationService) Check(ctx context.Context, input ContentModer
|
||||
if cfg.Mode == ContentModerationModePreBlock {
|
||||
s.recordPreBlockSyncMetric(0, ContentModerationActionError)
|
||||
}
|
||||
if cfg.failClosedOnError() {
|
||||
slog.Warn("content_moderation.block_no_audit_api_keys",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
"group_id", contentModerationLogGroupID(input.GroupID),
|
||||
"endpoint", input.Endpoint,
|
||||
"protocol", input.Protocol)
|
||||
return contentModerationUnavailableDecision(), nil
|
||||
}
|
||||
slog.Warn("content_moderation.skip_no_audit_api_keys",
|
||||
"user_id", input.UserID,
|
||||
"api_key_id", input.APIKeyID,
|
||||
@@ -1068,6 +1130,7 @@ func (s *ContentModerationService) checkSync(ctx context.Context, input ContentM
|
||||
"allow_block", allowBlock,
|
||||
"queue_delay_ms", queueDelay,
|
||||
"latency_ms", latency,
|
||||
"fail_closed", trackPreBlock && cfg.failClosedOnError(),
|
||||
"error", err)
|
||||
if queueDelay != nil {
|
||||
s.asyncErrors.Add(1)
|
||||
@@ -1076,6 +1139,12 @@ func (s *ContentModerationService) checkSync(ctx context.Context, input ContentM
|
||||
log := s.buildLog(input, cfg, ContentModerationActionError, false, "", 0, nil, content.ExcerptText(), &latency, queueDelay, err.Error())
|
||||
_ = s.repo.CreateLog(ctx, log)
|
||||
}
|
||||
// trackPreBlock 恰好等价于"同步前置拦截路径"(queueDelay == nil && allowBlock
|
||||
// && Mode == pre_block),异步 worker 与 observe 模式因此永远走放行分支——
|
||||
// 它们没有可拒绝的客户端。
|
||||
if trackPreBlock && cfg.failClosedOnError() {
|
||||
return contentModerationUnavailableDecision()
|
||||
}
|
||||
return allow
|
||||
}
|
||||
|
||||
@@ -2107,6 +2176,7 @@ func defaultContentModerationConfig() *ContentModerationConfig {
|
||||
Models: []string{},
|
||||
},
|
||||
CyberPolicyExcludeFromBanCount: false,
|
||||
FailClosedOnError: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2439,6 +2509,7 @@ func (s *ContentModerationService) configView(cfg *ContentModerationConfig) *Con
|
||||
KeywordBlockingMode: cfg.KeywordBlockingMode,
|
||||
ModelFilter: cloneContentModerationModelFilter(cfg.ModelFilter),
|
||||
CyberPolicyExcludeFromBanCount: cfg.CyberPolicyExcludeFromBanCount,
|
||||
FailClosedOnError: cfg.FailClosedOnError,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newFailClosedTestService 起一个把审计 API 指向"已关闭端口"的服务:callModeration
|
||||
// 必定失败(连接被拒),用来稳定复现"风控后端异常"。
|
||||
func newFailClosedTestService(t *testing.T, mutate func(cfg *ContentModerationConfig)) (*ContentModerationService, *contentModerationTestRepo) {
|
||||
t.Helper()
|
||||
|
||||
dead := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
|
||||
deadURL := dead.URL
|
||||
dead.Close() // 端口关闭 -> 后续调用连接被拒
|
||||
|
||||
cfg := defaultContentModerationConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.Mode = ContentModerationModePreBlock
|
||||
cfg.BaseURL = deadURL
|
||||
cfg.APIKeys = []string{"sk-test"}
|
||||
cfg.RetryCount = 0
|
||||
cfg.TimeoutMS = 500
|
||||
if mutate != nil {
|
||||
mutate(cfg)
|
||||
}
|
||||
rawCfg, err := json.Marshal(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
repo := &contentModerationTestRepo{}
|
||||
svc := NewContentModerationService(
|
||||
&contentModerationTestSettingRepo{values: map[string]string{
|
||||
SettingKeyRiskControlEnabled: "true",
|
||||
SettingKeyContentModerationConfig: string(rawCfg),
|
||||
}},
|
||||
repo, &contentModerationTestHashCache{}, nil, nil, nil, nil, nil,
|
||||
)
|
||||
return svc, repo
|
||||
}
|
||||
|
||||
func failClosedTestInput() ContentModerationCheckInput {
|
||||
return ContentModerationCheckInput{
|
||||
UserID: 1,
|
||||
Endpoint: "/v1/messages",
|
||||
Provider: "anthropic",
|
||||
Protocol: ContentModerationProtocolAnthropicMessages,
|
||||
Body: []byte(`{"messages":[{"role":"user","content":"hello"}]}`),
|
||||
}
|
||||
}
|
||||
|
||||
// 回归护栏(#5394):配置加载失败时必须放行。此时快照尚未建立,我们无从得知
|
||||
// risk_control_enabled 与 fail_closed_on_error,无法确认运维的阻断意图;若拦截,
|
||||
// 一次设置读取抖动就会让全部网关流量(含从未开启风控的部署)返回 5xx。
|
||||
func TestContentModerationCheck_ConfigLoadFailureAllows(t *testing.T) {
|
||||
svc := &ContentModerationService{
|
||||
settingRepo: &contentModerationTestSettingRepo{
|
||||
values: map[string]string{SettingKeyRiskControlEnabled: "false"},
|
||||
err: fmt.Errorf("settings unavailable"),
|
||||
},
|
||||
repo: &contentModerationTestRepo{},
|
||||
}
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed, "配置加载失败不得阻断流量:意图未知时只能放行")
|
||||
require.False(t, decision.Blocked)
|
||||
require.Equal(t, ContentModerationActionAllow, decision.Action)
|
||||
}
|
||||
|
||||
// 损坏的 content_moderation_config JSON 是确定性失败,快照永远建不起来。若在此
|
||||
// fail-closed,会造成永久性全站中断,且管理员被锁在修复界面之外。
|
||||
func TestContentModerationCheck_MalformedConfigJSONAllows(t *testing.T) {
|
||||
svc := &ContentModerationService{
|
||||
settingRepo: &contentModerationTestSettingRepo{values: map[string]string{
|
||||
SettingKeyRiskControlEnabled: "true",
|
||||
SettingKeyContentModerationConfig: `{"enabled": tru`,
|
||||
}},
|
||||
repo: &contentModerationTestRepo{},
|
||||
}
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed, "第 %d 次请求:损坏配置不得演变成永久中断", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
// 默认行为不变:审计 API 挂掉照旧放行,升级不会给存量部署带来可用性变化。
|
||||
func TestContentModerationCheck_AuditAPIFailureFailsOpenByDefault(t *testing.T) {
|
||||
svc, repo := newFailClosedTestService(t, nil) // FailClosedOnError 默认 false
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed, "默认必须 fail-open")
|
||||
require.False(t, decision.Blocked)
|
||||
require.Empty(t, repo.snapshotLogs(), "RecordNonHits 关闭时不写日志")
|
||||
}
|
||||
|
||||
// #5388 的真正修复:开关打开后,审计 API 异常必须拒绝请求。
|
||||
func TestContentModerationCheck_AuditAPIFailureFailsClosedWhenEnabled(t *testing.T) {
|
||||
svc, repo := newFailClosedTestService(t, func(cfg *ContentModerationConfig) {
|
||||
cfg.FailClosedOnError = true
|
||||
})
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.False(t, decision.Allowed)
|
||||
require.True(t, decision.Blocked, "开关打开后审计后端异常必须阻断")
|
||||
require.Equal(t, http.StatusServiceUnavailable, decision.StatusCode, "系统不可用用 503,不用 403/500")
|
||||
require.Equal(t, ContentModerationErrorCodeUnavailable, decision.ErrorCode)
|
||||
require.Equal(t, ContentModerationUnavailableMessage, decision.Message)
|
||||
require.Equal(t, ContentModerationActionError, decision.Action)
|
||||
|
||||
require.False(t, decision.Flagged, "系统故障不是内容违规,Flagged 必须为 false")
|
||||
for _, log := range repo.snapshotLogs() {
|
||||
require.False(t, log.Flagged, "fail-closed 不得写 flagged 日志,否则会喂给自动封号计数")
|
||||
}
|
||||
}
|
||||
|
||||
// RecordNonHits 打开时仍会落一条 error 日志,但必须是非 flagged 的,
|
||||
// 不能进入 CountFlaggedByUserSince 的封号计数。
|
||||
func TestContentModerationCheck_FailClosedLogIsNotFlagged(t *testing.T) {
|
||||
svc, repo := newFailClosedTestService(t, func(cfg *ContentModerationConfig) {
|
||||
cfg.FailClosedOnError = true
|
||||
cfg.RecordNonHits = true
|
||||
})
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Blocked)
|
||||
|
||||
logs := repo.snapshotLogs()
|
||||
require.Len(t, logs, 1)
|
||||
require.False(t, logs[0].Flagged, "error 日志必须非 flagged")
|
||||
require.Equal(t, ContentModerationActionError, logs[0].Action)
|
||||
}
|
||||
|
||||
// observe 模式没有阻断语义,开关打开也不得拦截。
|
||||
func TestContentModerationCheck_FailClosedIgnoredInObserveMode(t *testing.T) {
|
||||
svc, _ := newFailClosedTestService(t, func(cfg *ContentModerationConfig) {
|
||||
cfg.FailClosedOnError = true
|
||||
cfg.Mode = ContentModerationModeObserve
|
||||
})
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed, "observe 模式永不阻断")
|
||||
}
|
||||
|
||||
// 一个审计 Key 都没配:默认放行,开关打开则拒绝。
|
||||
func TestContentModerationCheck_NoAuditKeysRespectsFailClosedSwitch(t *testing.T) {
|
||||
t.Run("默认放行", func(t *testing.T) {
|
||||
svc, _ := newFailClosedTestService(t, func(cfg *ContentModerationConfig) {
|
||||
cfg.APIKeys = []string{}
|
||||
})
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed)
|
||||
})
|
||||
|
||||
t.Run("开关打开则拒绝", func(t *testing.T) {
|
||||
svc, _ := newFailClosedTestService(t, func(cfg *ContentModerationConfig) {
|
||||
cfg.APIKeys = []string{}
|
||||
cfg.FailClosedOnError = true
|
||||
})
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Blocked)
|
||||
require.Equal(t, http.StatusServiceUnavailable, decision.StatusCode)
|
||||
require.Equal(t, ContentModerationErrorCodeUnavailable, decision.ErrorCode)
|
||||
})
|
||||
}
|
||||
|
||||
// 审计总开关关闭时,无论 fail_closed_on_error 如何都不得阻断。
|
||||
func TestContentModerationCheck_FailClosedIgnoredWhenModerationDisabled(t *testing.T) {
|
||||
svc, _ := newFailClosedTestService(t, func(cfg *ContentModerationConfig) {
|
||||
cfg.Enabled = false
|
||||
cfg.FailClosedOnError = true
|
||||
})
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed, "审计未启用的部署永远不受 fail-closed 影响")
|
||||
}
|
||||
|
||||
// 风控总开关(risk_control_enabled)关闭时同样不得阻断。
|
||||
func TestContentModerationCheck_FailClosedIgnoredWhenRiskControlDisabled(t *testing.T) {
|
||||
cfg := defaultContentModerationConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.Mode = ContentModerationModePreBlock
|
||||
cfg.FailClosedOnError = true
|
||||
cfg.APIKeys = []string{}
|
||||
rawCfg, err := json.Marshal(cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
svc := NewContentModerationService(
|
||||
&contentModerationTestSettingRepo{values: map[string]string{
|
||||
SettingKeyRiskControlEnabled: "false",
|
||||
SettingKeyContentModerationConfig: string(rawCfg),
|
||||
}},
|
||||
&contentModerationTestRepo{}, &contentModerationTestHashCache{}, nil, nil, nil, nil, nil,
|
||||
)
|
||||
|
||||
decision, err := svc.Check(context.Background(), failClosedTestInput())
|
||||
|
||||
require.NoError(t, err)
|
||||
require.True(t, decision.Allowed, "风控中心未启用的部署永远不受 fail-closed 影响")
|
||||
}
|
||||
|
||||
func TestContentModerationConfig_FailClosedOnErrorMatrix(t *testing.T) {
|
||||
build := func(enabled, failClosed bool, mode string) *ContentModerationConfig {
|
||||
cfg := defaultContentModerationConfig()
|
||||
cfg.Enabled = enabled
|
||||
cfg.FailClosedOnError = failClosed
|
||||
cfg.Mode = mode
|
||||
return cfg
|
||||
}
|
||||
|
||||
require.True(t, build(true, true, ContentModerationModePreBlock).failClosedOnError())
|
||||
require.False(t, build(true, false, ContentModerationModePreBlock).failClosedOnError(), "开关关闭")
|
||||
require.False(t, build(false, true, ContentModerationModePreBlock).failClosedOnError(), "审计未启用")
|
||||
require.False(t, build(true, true, ContentModerationModeObserve).failClosedOnError(), "observe 模式")
|
||||
require.False(t, build(true, true, ContentModerationModeOff).failClosedOnError(), "off 模式")
|
||||
|
||||
var nilCfg *ContentModerationConfig
|
||||
require.False(t, nilCfg.failClosedOnError(), "nil 配置不得 panic")
|
||||
}
|
||||
|
||||
func TestContentModerationUpdateConfig_FailClosedOnError(t *testing.T) {
|
||||
repo := &contentModerationTestSettingRepo{values: map[string]string{}}
|
||||
svc := NewContentModerationService(repo, &contentModerationTestRepo{}, nil, nil, nil, nil, nil, nil)
|
||||
ctx := context.Background()
|
||||
|
||||
view, err := svc.GetConfig(ctx)
|
||||
require.NoError(t, err)
|
||||
require.False(t, view.FailClosedOnError, "默认必须 fail-open")
|
||||
|
||||
enable := true
|
||||
view, err = svc.UpdateConfig(ctx, UpdateContentModerationConfigInput{FailClosedOnError: &enable})
|
||||
require.NoError(t, err)
|
||||
require.True(t, view.FailClosedOnError)
|
||||
|
||||
saved := &ContentModerationConfig{}
|
||||
require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), saved))
|
||||
require.True(t, saved.FailClosedOnError, "必须持久化")
|
||||
|
||||
// 未携带该字段的后续更新不得把它冲掉。
|
||||
view, err = svc.UpdateConfig(ctx, UpdateContentModerationConfigInput{})
|
||||
require.NoError(t, err)
|
||||
require.True(t, view.FailClosedOnError)
|
||||
|
||||
disable := false
|
||||
view, err = svc.UpdateConfig(ctx, UpdateContentModerationConfigInput{FailClosedOnError: &disable})
|
||||
require.NoError(t, err)
|
||||
require.False(t, view.FailClosedOnError)
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
type contentModerationTestSettingRepo struct {
|
||||
values map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *contentModerationTestSettingRepo) Get(ctx context.Context, key string) (*Setting, error) {
|
||||
@@ -44,6 +45,9 @@ func (r *contentModerationTestSettingRepo) Set(ctx context.Context, key, value s
|
||||
}
|
||||
|
||||
func (r *contentModerationTestSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
out := map[string]string{}
|
||||
for _, key := range keys {
|
||||
if value, ok := r.values[key]; ok {
|
||||
|
||||
@@ -23,6 +23,7 @@ export interface BackupRecord {
|
||||
backup_type: string
|
||||
file_name: string
|
||||
s3_key: string
|
||||
parts?: BackupPart[]
|
||||
size_bytes: number
|
||||
triggered_by: string
|
||||
error_message?: string
|
||||
@@ -35,6 +36,24 @@ export interface BackupRecord {
|
||||
restored_at?: string
|
||||
}
|
||||
|
||||
export interface BackupPart {
|
||||
index: number
|
||||
s3_key: string
|
||||
size_bytes: number
|
||||
sha256?: string
|
||||
}
|
||||
|
||||
export interface BackupDownloadPart {
|
||||
index: number
|
||||
size_bytes: number
|
||||
url: string
|
||||
}
|
||||
|
||||
export interface BackupDownloadResponse {
|
||||
url?: string
|
||||
parts?: BackupDownloadPart[]
|
||||
}
|
||||
|
||||
export interface CreateBackupRequest {
|
||||
expire_days?: number
|
||||
}
|
||||
@@ -137,8 +156,8 @@ export async function deleteBackup(id: string): Promise<void> {
|
||||
await apiClient.delete(`/admin/backups/${id}`)
|
||||
}
|
||||
|
||||
export async function getDownloadURL(id: string): Promise<{ url: string }> {
|
||||
const { data } = await apiClient.get<{ url: string }>(`/admin/backups/${id}/download-url`)
|
||||
export async function getDownloadURL(id: string): Promise<BackupDownloadResponse> {
|
||||
const { data } = await apiClient.get<BackupDownloadResponse>(`/admin/backups/${id}/download-url`)
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ export interface ContentModerationConfig {
|
||||
keyword_blocking_mode: KeywordBlockingMode
|
||||
model_filter: ContentModerationModelFilter
|
||||
cyber_policy_exclude_from_ban_count: boolean
|
||||
fail_closed_on_error: boolean
|
||||
}
|
||||
|
||||
export type ContentModerationAPIKeyStatusValue = 'unknown' | 'ok' | 'error' | 'frozen'
|
||||
@@ -122,6 +123,7 @@ export interface UpdateContentModerationConfig {
|
||||
keyword_blocking_mode?: KeywordBlockingMode
|
||||
model_filter?: ContentModerationModelFilter
|
||||
cyber_policy_exclude_from_ban_count?: boolean
|
||||
fail_closed_on_error?: boolean
|
||||
}
|
||||
|
||||
export interface ContentModerationRuntimeStatus {
|
||||
|
||||
@@ -314,6 +314,8 @@ export default {
|
||||
autoBanHint: 'Disable the user, invalidate auth cache, and send a ban notice after the hit threshold is reached.',
|
||||
cyberPolicyExcludeBan: 'Exclude Cyber Policy Hits from Ban Count',
|
||||
cyberPolicyExcludeBanHint: 'When enabled, cyber_policy hits no longer count toward auto-ban violations: no ban judgment on the hit itself, and history rows are excluded from the rolling count. Logs and notice emails are unaffected.',
|
||||
failClosedOnError: 'Reject Requests When Audit Fails',
|
||||
failClosedOnErrorHint: 'Pre-block mode only. When enabled, a failed moderation API call or a missing audit key rejects the request with 503 instead of letting it through. When disabled (default), audit failures fail open and availability wins. Enable this only if your moderation service is reliable — otherwise an outage there takes down live traffic.',
|
||||
violationNotCounted: 'Not counted',
|
||||
banThreshold: 'Ban Threshold',
|
||||
violationWindowHours: 'Count Window (hours)',
|
||||
|
||||
@@ -144,6 +144,7 @@ export default {
|
||||
status: 'Status',
|
||||
fileName: 'File Name',
|
||||
size: 'Size',
|
||||
parts: 'Parts',
|
||||
expiresAt: 'Expires At',
|
||||
triggeredBy: 'Triggered By',
|
||||
startedAt: 'Started At',
|
||||
@@ -168,6 +169,10 @@ export default {
|
||||
empty: 'No backup records',
|
||||
actions: {
|
||||
download: 'Download',
|
||||
downloadParts: 'Download Parts',
|
||||
downloadPartsHint: 'Download every part in order and concatenate the gzip bytes: on Linux/macOS run cat payload.part-* > backup.sql.gz; on Windows run copy /b payload.part-000001+payload.part-000002 backup.sql.gz.',
|
||||
partLabel: 'Part {index}',
|
||||
downloadFailed: 'Download URL is empty',
|
||||
restore: 'Restore',
|
||||
restoreConfirm: 'Are you sure you want to restore from this backup? This will overwrite the current database!',
|
||||
restorePasswordPrompt: 'Please enter your admin password to confirm the restore operation',
|
||||
|
||||
@@ -314,6 +314,8 @@ export default {
|
||||
autoBanHint: '命中次数达到阈值后将禁用用户账号、刷新认证缓存并发送封禁通知邮件。',
|
||||
cyberPolicyExcludeBan: 'cyber_policy 不计入封号次数',
|
||||
cyberPolicyExcludeBanHint: '开启后,cyber_policy 拦截不再计入自动封号的违规次数:当次不判定封号,历史累计亦排除。风控日志与通知邮件照常。',
|
||||
failClosedOnError: '审计异常时拒绝请求',
|
||||
failClosedOnErrorHint: '仅对「前置拦截」模式生效。开启后,审计接口调用失败或无可用审核 Key 时返回 503 拒绝请求,而不是放行;关闭时(默认)审计异常放行,优先保证可用性。开启前请确认审核服务足够稳定,否则审核侧故障会直接中断线上流量。',
|
||||
violationNotCounted: '未计入封号',
|
||||
banThreshold: '封禁触发次数',
|
||||
violationWindowHours: '累计窗口(小时)',
|
||||
|
||||
@@ -144,6 +144,7 @@ export default {
|
||||
status: '状态',
|
||||
fileName: '文件名',
|
||||
size: '大小',
|
||||
parts: '分卷数',
|
||||
expiresAt: '过期时间',
|
||||
triggeredBy: '触发方式',
|
||||
startedAt: '开始时间',
|
||||
@@ -168,6 +169,10 @@ export default {
|
||||
empty: '暂无备份记录',
|
||||
actions: {
|
||||
download: '下载',
|
||||
downloadParts: '下载分卷',
|
||||
downloadPartsHint: '请按顺序下载全部分卷后拼接 gzip 字节流:Linux/macOS 使用 cat payload.part-* > backup.sql.gz;Windows 使用 copy /b payload.part-000001+payload.part-000002 backup.sql.gz。',
|
||||
partLabel: '第 {index} 卷',
|
||||
downloadFailed: '下载地址为空',
|
||||
restore: '恢复',
|
||||
restoreConfirm: '确定要从此备份恢复吗?这将覆盖当前数据库!',
|
||||
restorePasswordPrompt: '请输入管理员密码以确认恢复操作',
|
||||
|
||||
@@ -200,6 +200,7 @@
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.status') }}</th>
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.fileName') }}</th>
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.size') }}</th>
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.parts') }}</th>
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.expiresAt') }}</th>
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.triggeredBy') }}</th>
|
||||
<th class="py-2 pr-4">{{ t('admin.backup.columns.startedAt') }}</th>
|
||||
@@ -221,6 +222,7 @@
|
||||
</td>
|
||||
<td class="py-3 pr-4 text-xs">{{ record.file_name }}</td>
|
||||
<td class="py-3 pr-4 text-xs">{{ formatSize(record.size_bytes) }}</td>
|
||||
<td class="py-3 pr-4 text-xs">{{ record.parts?.length || (record.status === 'running' ? '-' : 1) }}</td>
|
||||
<td class="py-3 pr-4 text-xs">
|
||||
{{ record.expires_at ? formatDate(record.expires_at) : t('admin.backup.neverExpire') }}
|
||||
</td>
|
||||
@@ -248,6 +250,7 @@
|
||||
{{ restoringId === record.id ? t('common.loading') : t('admin.backup.actions.restore') }}
|
||||
</button>
|
||||
<button
|
||||
v-if="record.status !== 'running'"
|
||||
type="button"
|
||||
class="btn btn-danger btn-xs"
|
||||
@click="removeBackup(record.id)"
|
||||
@@ -258,7 +261,7 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr v-if="backups.length === 0">
|
||||
<td colspan="8" class="py-6 text-center text-sm text-gray-500 dark:text-gray-400">
|
||||
<td colspan="9" class="py-6 text-center text-sm text-gray-500 dark:text-gray-400">
|
||||
{{ t('admin.backup.empty') }}
|
||||
</td>
|
||||
</tr>
|
||||
@@ -351,6 +354,48 @@
|
||||
</div>
|
||||
</transition>
|
||||
</teleport>
|
||||
<!-- 分卷下载链接 -->
|
||||
<teleport to="body">
|
||||
<transition name="modal">
|
||||
<div
|
||||
v-if="downloadPartsModalOpen"
|
||||
class="fixed inset-0 z-50 flex items-center justify-center p-4"
|
||||
@mousedown.self="closeDownloadParts"
|
||||
>
|
||||
<div class="fixed inset-0 bg-black/50" @click="closeDownloadParts"></div>
|
||||
<div class="relative max-h-[85vh] w-full max-w-lg overflow-y-auto rounded-xl bg-white p-6 shadow-2xl dark:bg-dark-800">
|
||||
<button
|
||||
type="button"
|
||||
class="absolute right-4 top-4 text-gray-400 hover:text-gray-600 dark:hover:text-gray-200"
|
||||
:aria-label="t('common.close')"
|
||||
@click="closeDownloadParts"
|
||||
>
|
||||
<svg class="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="2"><path stroke-linecap="round" stroke-linejoin="round" d="M6 18L18 6M6 6l12 12" /></svg>
|
||||
</button>
|
||||
<h2 class="mb-1 text-lg font-bold text-gray-900 dark:text-white">{{ t('admin.backup.actions.downloadParts') }}</h2>
|
||||
<p class="mb-4 text-sm text-gray-500 dark:text-gray-400">{{ t('admin.backup.actions.downloadPartsHint') }}</p>
|
||||
<div class="space-y-2">
|
||||
<div
|
||||
v-for="part in downloadParts"
|
||||
:key="part.index"
|
||||
class="flex items-center justify-between gap-3 rounded-lg border border-gray-200 px-3 py-2 dark:border-dark-600"
|
||||
>
|
||||
<span class="text-sm text-gray-700 dark:text-gray-300">
|
||||
{{ t('admin.backup.actions.partLabel', { index: part.index }) }}
|
||||
<span class="ml-2 text-xs text-gray-500 dark:text-gray-400">{{ formatSize(part.size_bytes) }}</span>
|
||||
</span>
|
||||
<a :href="part.url" class="btn btn-secondary btn-xs" rel="noopener">
|
||||
{{ t('admin.backup.actions.download') }}
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="mt-4 text-right">
|
||||
<button type="button" class="btn btn-primary btn-sm" @click="closeDownloadParts">{{ t('common.close') }}</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</transition>
|
||||
</teleport>
|
||||
<TotpStepUpDialog :controller="backupStepUp" />
|
||||
</template>
|
||||
|
||||
@@ -363,6 +408,7 @@ import type {
|
||||
BackupS3Config,
|
||||
BackupScheduleConfig,
|
||||
BackupRecord,
|
||||
BackupDownloadPart,
|
||||
ImageStorageConfig,
|
||||
} from '@/api/admin/backup'
|
||||
import { useStepUp, isStepUpBlocked, isStepUpCancelled, stepUpBlockReason } from '@/composables/useStepUp'
|
||||
@@ -432,6 +478,8 @@ const loadingBackups = ref(false)
|
||||
const creatingBackup = ref(false)
|
||||
const restoringId = ref('')
|
||||
const manualExpireDays = ref(14)
|
||||
const downloadParts = ref<BackupDownloadPart[]>([])
|
||||
const downloadPartsModalOpen = ref(false)
|
||||
|
||||
// Polling
|
||||
const pollingTimer = ref<ReturnType<typeof setInterval> | null>(null)
|
||||
@@ -715,6 +763,14 @@ async function createBackup() {
|
||||
async function downloadBackup(id: string) {
|
||||
try {
|
||||
const result = await backupStepUp.run(() => adminAPI.backup.getDownloadURL(id))
|
||||
if (result.parts && result.parts.length > 0) {
|
||||
downloadParts.value = result.parts
|
||||
downloadPartsModalOpen.value = true
|
||||
return
|
||||
}
|
||||
if (!result.url) {
|
||||
throw new Error(t('admin.backup.actions.downloadFailed'))
|
||||
}
|
||||
// 预签名 URL 带 attachment disposition,同页 anchor 导航直接触发下载;
|
||||
// 不用 window.open:step-up 弹窗 await 会耗尽瞬态用户激活,新标签页会被浏览器拦截。
|
||||
const link = document.createElement('a')
|
||||
@@ -728,6 +784,11 @@ async function downloadBackup(id: string) {
|
||||
}
|
||||
}
|
||||
|
||||
function closeDownloadParts() {
|
||||
downloadPartsModalOpen.value = false
|
||||
downloadParts.value = []
|
||||
}
|
||||
|
||||
async function restoreBackup(id: string) {
|
||||
if (!window.confirm(t('admin.backup.actions.restoreConfirm'))) return
|
||||
const password = window.prompt(t('admin.backup.actions.restorePasswordPrompt'))
|
||||
|
||||
@@ -875,6 +875,13 @@
|
||||
<label class="input-label">{{ t('admin.riskControl.blockMessage') }}</label>
|
||||
<input v-model.trim="configForm.block_message" type="text" class="input" />
|
||||
</div>
|
||||
<div class="flex items-center justify-between rounded-lg border border-gray-100 p-4 dark:border-dark-700 lg:col-span-2">
|
||||
<div>
|
||||
<p class="text-sm font-medium text-gray-900 dark:text-white">{{ t('admin.riskControl.failClosedOnError') }}</p>
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.riskControl.failClosedOnErrorHint') }}</p>
|
||||
</div>
|
||||
<Toggle v-model="configForm.fail_closed_on_error" />
|
||||
</div>
|
||||
<div class="flex items-center justify-between rounded-lg border border-gray-100 p-4 dark:border-dark-700">
|
||||
<div>
|
||||
<p class="text-sm font-medium text-gray-900 dark:text-white">{{ t('admin.riskControl.emailOnHit') }}</p>
|
||||
@@ -1250,6 +1257,7 @@ const configForm = reactive({
|
||||
email_on_hit: true,
|
||||
auto_ban_enabled: true,
|
||||
cyber_policy_exclude_from_ban_count: false,
|
||||
fail_closed_on_error: false,
|
||||
ban_threshold: 10,
|
||||
violation_window_hours: 720,
|
||||
hit_retention_days: 180,
|
||||
@@ -1728,6 +1736,7 @@ function applyConfig(config: ContentModerationConfig) {
|
||||
configForm.email_on_hit = config.email_on_hit ?? true
|
||||
configForm.auto_ban_enabled = config.auto_ban_enabled ?? true
|
||||
configForm.cyber_policy_exclude_from_ban_count = config.cyber_policy_exclude_from_ban_count ?? false
|
||||
configForm.fail_closed_on_error = config.fail_closed_on_error ?? false
|
||||
configForm.ban_threshold = config.ban_threshold || 10
|
||||
configForm.violation_window_hours = config.violation_window_hours || 720
|
||||
configForm.hit_retention_days = config.hit_retention_days || 180
|
||||
@@ -1814,6 +1823,7 @@ async function saveConfig() {
|
||||
email_on_hit: configForm.email_on_hit,
|
||||
auto_ban_enabled: configForm.auto_ban_enabled,
|
||||
cyber_policy_exclude_from_ban_count: configForm.cyber_policy_exclude_from_ban_count,
|
||||
fail_closed_on_error: configForm.fail_closed_on_error,
|
||||
ban_threshold: Number(configForm.ban_threshold) || 10,
|
||||
violation_window_hours: Number(configForm.violation_window_hours) || 720,
|
||||
hit_retention_days: Number(configForm.hit_retention_days) || 180,
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { flushPromises, mount } from '@vue/test-utils'
|
||||
|
||||
import BackupView from '../BackupView.vue'
|
||||
|
||||
const {
|
||||
getS3Config,
|
||||
getImageStorageConfig,
|
||||
getSchedule,
|
||||
listBackups,
|
||||
getDownloadURL,
|
||||
} = vi.hoisted(() => ({
|
||||
getS3Config: vi.fn(),
|
||||
getImageStorageConfig: vi.fn(),
|
||||
getSchedule: vi.fn(),
|
||||
listBackups: vi.fn(),
|
||||
getDownloadURL: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/api', () => ({
|
||||
adminAPI: {
|
||||
backup: {
|
||||
getS3Config,
|
||||
updateS3Config: vi.fn(),
|
||||
testS3Connection: vi.fn(),
|
||||
getImageStorageConfig,
|
||||
updateImageStorageConfig: vi.fn(),
|
||||
testImageStorageConnection: vi.fn(),
|
||||
getSchedule,
|
||||
updateSchedule: vi.fn(),
|
||||
createBackup: vi.fn(),
|
||||
listBackups,
|
||||
getBackup: vi.fn(),
|
||||
deleteBackup: vi.fn(),
|
||||
getDownloadURL,
|
||||
restoreBackup: vi.fn(),
|
||||
},
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@/stores', () => ({
|
||||
useAppStore: () => ({
|
||||
showError: vi.fn(),
|
||||
showSuccess: vi.fn(),
|
||||
showWarning: vi.fn(),
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('@/composables/useStepUp', () => ({
|
||||
useStepUp: () => ({ run: (fn: () => unknown) => fn() }),
|
||||
isStepUpBlocked: () => false,
|
||||
isStepUpCancelled: () => false,
|
||||
stepUpBlockReason: () => '',
|
||||
}))
|
||||
|
||||
vi.mock('vue-i18n', () => ({
|
||||
useI18n: () => ({
|
||||
t: (key: string, params?: Record<string, unknown>) =>
|
||||
params?.index === undefined ? key : `${key}:${params.index}`,
|
||||
}),
|
||||
}))
|
||||
|
||||
const baseRecord = (id: string, parts?: unknown[]) => ({
|
||||
id,
|
||||
status: 'completed',
|
||||
backup_type: 'postgres',
|
||||
file_name: `${id}.sql.gz`,
|
||||
s3_key: `backups/${id}.sql.gz`,
|
||||
parts,
|
||||
size_bytes: 10,
|
||||
triggered_by: 'manual',
|
||||
started_at: '2026-08-09T00:00:00Z',
|
||||
})
|
||||
|
||||
function mountBackupView() {
|
||||
return mount(BackupView, {
|
||||
global: {
|
||||
stubs: {
|
||||
TotpStepUpDialog: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
describe('admin BackupView 分卷备份', () => {
|
||||
beforeEach(() => {
|
||||
getS3Config.mockResolvedValue({})
|
||||
getImageStorageConfig.mockResolvedValue({ config: {}, secret_configured: false })
|
||||
getSchedule.mockResolvedValue({ enabled: false, cron_expr: '', retain_days: 14, retain_count: 10 })
|
||||
getDownloadURL.mockReset()
|
||||
vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
document.body.innerHTML = ''
|
||||
})
|
||||
|
||||
it('显示分卷数并在下载时列出每个分卷链接', async () => {
|
||||
listBackups.mockResolvedValue({
|
||||
items: [baseRecord('split', [{ index: 1 }, { index: 2 }, { index: 3 }])],
|
||||
})
|
||||
getDownloadURL.mockResolvedValue({
|
||||
parts: [
|
||||
{ index: 1, size_bytes: 5, url: 'https://example.test/part-1' },
|
||||
{ index: 2, size_bytes: 6, url: 'https://example.test/part-2' },
|
||||
{ index: 3, size_bytes: 7, url: 'https://example.test/part-3' },
|
||||
],
|
||||
})
|
||||
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
|
||||
expect(wrapper.text()).toContain('3')
|
||||
const downloadButton = wrapper.findAll('button').find(button =>
|
||||
button.text().includes('admin.backup.actions.download'),
|
||||
)
|
||||
expect(downloadButton).toBeDefined()
|
||||
await downloadButton!.trigger('click')
|
||||
await flushPromises()
|
||||
|
||||
expect(document.body.textContent).toContain('admin.backup.actions.partLabel:1')
|
||||
expect(document.body.textContent).toContain('admin.backup.actions.partLabel:3')
|
||||
expect(document.body.querySelector('a[href="https://example.test/part-2"]')).not.toBeNull()
|
||||
})
|
||||
|
||||
it('旧单文件记录仍使用单个下载地址', async () => {
|
||||
listBackups.mockResolvedValue({ items: [baseRecord('legacy')] })
|
||||
getDownloadURL.mockResolvedValue({ url: 'https://example.test/legacy.sql.gz' })
|
||||
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
const downloadButton = wrapper.findAll('button').find(button =>
|
||||
button.text().includes('admin.backup.actions.download'),
|
||||
)
|
||||
await downloadButton!.trigger('click')
|
||||
await flushPromises()
|
||||
|
||||
expect(getDownloadURL).toHaveBeenCalledWith('legacy')
|
||||
expect(document.body.textContent).not.toContain('admin.backup.actions.downloadParts')
|
||||
})
|
||||
|
||||
it('运行中的备份不显示删除入口', async () => {
|
||||
listBackups.mockResolvedValue({
|
||||
items: [{ ...baseRecord('running'), status: 'running', progress: 'uploading' }],
|
||||
})
|
||||
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
|
||||
expect(wrapper.find('tbody tr td:nth-child(5)').text()).toBe('-')
|
||||
expect(wrapper.findAll('button').some(button => button.text() === 'common.delete')).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -415,4 +415,69 @@ describe('admin RiskControlView', () => {
|
||||
'overflow-y-auto',
|
||||
]))
|
||||
})
|
||||
|
||||
// 防"开关静默失效":后端返回的 fail_closed_on_error 必须回填到表单,并在保存时
|
||||
// 原样回传,否则运维打开的开关会被下一次保存悄悄清掉。
|
||||
it('round-trips fail_closed_on_error between load and save', async () => {
|
||||
getConfig.mockResolvedValue({ ...baseConfig(), fail_closed_on_error: true })
|
||||
|
||||
const wrapper = mount(RiskControlView, {
|
||||
global: {
|
||||
stubs: {
|
||||
AppLayout: AppLayoutStub,
|
||||
BaseDialog: BaseDialogStub,
|
||||
Icon: true,
|
||||
Select: true,
|
||||
Toggle: true,
|
||||
Pagination: true,
|
||||
ModelWhitelistSelector: ModelWhitelistSelectorStub,
|
||||
ProxySelector: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await flushPromises()
|
||||
|
||||
await findButtonByText(wrapper, 'admin.riskControl.openSettings').trigger('click')
|
||||
await findButtonByText(wrapper, 'admin.riskControl.saveConfig').trigger('click')
|
||||
await flushPromises()
|
||||
|
||||
expect(updateConfig).toHaveBeenCalledWith(expect.objectContaining({
|
||||
fail_closed_on_error: true,
|
||||
}))
|
||||
expect(showError).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('defaults fail_closed_on_error to false when the backend omits it', async () => {
|
||||
const { fail_closed_on_error: _omitted, ...withoutSwitch } = {
|
||||
...baseConfig(),
|
||||
fail_closed_on_error: false,
|
||||
}
|
||||
getConfig.mockResolvedValue(withoutSwitch)
|
||||
|
||||
const wrapper = mount(RiskControlView, {
|
||||
global: {
|
||||
stubs: {
|
||||
AppLayout: AppLayoutStub,
|
||||
BaseDialog: BaseDialogStub,
|
||||
Icon: true,
|
||||
Select: true,
|
||||
Toggle: true,
|
||||
Pagination: true,
|
||||
ModelWhitelistSelector: ModelWhitelistSelectorStub,
|
||||
ProxySelector: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
await flushPromises()
|
||||
|
||||
await findButtonByText(wrapper, 'admin.riskControl.openSettings').trigger('click')
|
||||
await findButtonByText(wrapper, 'admin.riskControl.saveConfig').trigger('click')
|
||||
await flushPromises()
|
||||
|
||||
expect(updateConfig).toHaveBeenCalledWith(expect.objectContaining({
|
||||
fail_closed_on_error: false,
|
||||
}))
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user