mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-14 03:52:26 +08:00
feat(backup): 支持月度归档与独立保留策略
This commit is contained in:
@@ -140,7 +140,13 @@ func (h *BackupHandler) DeleteBackup(c *gin.Context) {
|
||||
response.BadRequest(c, "backup ID is required")
|
||||
return
|
||||
}
|
||||
if err := h.backupService.DeleteBackup(c.Request.Context(), backupID); err != nil {
|
||||
var err error
|
||||
if c.Query("delete_archived") == "true" {
|
||||
err = h.backupService.DeleteArchivedBackup(c.Request.Context(), backupID)
|
||||
} else {
|
||||
err = h.backupService.DeleteBackup(c.Request.Context(), backupID)
|
||||
}
|
||||
if err != nil {
|
||||
response.ErrorFrom(c, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type backupDelayedReadRepo struct {
|
||||
*mockSettingRepo
|
||||
fired atomic.Bool
|
||||
loaded chan struct{}
|
||||
resume chan struct{}
|
||||
}
|
||||
|
||||
func (r *backupDelayedReadRepo) GetValue(ctx context.Context, key string) (string, error) {
|
||||
value, err := r.mockSettingRepo.GetValue(ctx, key)
|
||||
if key == settingKeyBackupRecords && r.fired.CompareAndSwap(false, true) {
|
||||
close(r.loaded)
|
||||
<-r.resume
|
||||
}
|
||||
return value, err
|
||||
}
|
||||
func TestBackupRecovery_RestartPreservesPeerArchive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
writer := newTestBackupService(repo, &mockDumper{}, store)
|
||||
started := time.Now().UTC()
|
||||
record := archiveRecord("now-permanent", started.Format(time.RFC3339))
|
||||
record.Status = "running"
|
||||
record.S3Key = "now-permanent.sql.gz"
|
||||
store.objects[record.S3Key] = []byte("important archive")
|
||||
require.NoError(t, writer.saveRecord(ctx, record))
|
||||
|
||||
restarting := newTestBackupService(repo, &mockDumper{}, store)
|
||||
delayedRepo := &backupDelayedReadRepo{mockSettingRepo: repo, loaded: make(chan struct{}), resume: make(chan struct{})}
|
||||
restarting.settingRepo = delayedRepo
|
||||
cache := &fakeLeaderLockCache{}
|
||||
writer.SetLeaderLock(cache, nil)
|
||||
restarting.SetLeaderLock(cache, nil)
|
||||
done := make(chan struct{})
|
||||
go func() { restarting.recoverStaleRecords(); close(done) }()
|
||||
<-delayedRepo.loaded
|
||||
record.Status = "completed"
|
||||
// Recovery must own the writer lock across its read and decision. The peer
|
||||
// completes only after recovery releases it; the young operation is active.
|
||||
require.Equal(t, restarting.instanceID, cache.heldBy("backup:records:writer"))
|
||||
written := make(chan error, 1)
|
||||
go func() { written <- writer.saveRecordWithArchive(ctx, record, archiveSchedule(started.Day())) }()
|
||||
close(delayedRepo.resume)
|
||||
<-done
|
||||
require.NoError(t, <-written)
|
||||
require.NotNil(t, record.MonthlyArchive)
|
||||
restarting.recoverStaleRecords()
|
||||
saved, err := writer.GetBackupRecord(ctx, record.ID)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, saved.MonthlyArchive)
|
||||
require.Equal(t, "completed", saved.Status, "completed permanent archive was overwritten by stale restart snapshot")
|
||||
require.Contains(t, store.objects, record.S3Key, "restart must not delete completed permanent archive")
|
||||
}
|
||||
|
||||
func TestBackupRecovery_RestoreRejectsRemovedArchive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
cleaner := newTestBackupService(repo, &mockDumper{}, store)
|
||||
for i, id := range []string{"old", "new"} {
|
||||
record := archiveRecord(id, []string{"2026-09-01T04:00:00Z", "2026-09-15T04:00:00Z"}[i])
|
||||
record.MonthlyArchive = &BackupMonthlyArchive{RetainCount: 1}
|
||||
record.S3Key = id
|
||||
store.objects[id] = []byte("payload")
|
||||
require.NoError(t, cleaner.saveRecord(ctx, record))
|
||||
}
|
||||
restoring := newTestBackupService(repo, &mockDumper{}, store)
|
||||
delayedRepo := &backupDelayedReadRepo{mockSettingRepo: repo, loaded: make(chan struct{}), resume: make(chan struct{})}
|
||||
restoring.settingRepo = delayedRepo
|
||||
cache := &fakeLeaderLockCache{}
|
||||
cleaner.SetLeaderLock(cache, nil)
|
||||
restoring.SetLeaderLock(cache, nil)
|
||||
result := make(chan error)
|
||||
go func() { _, err := restoring.StartRestore(ctx, "old"); result <- err }()
|
||||
<-delayedRepo.loaded
|
||||
cfg := archiveSchedule(1, 15)
|
||||
cfg.MonthlyArchive.RetainCount = 1
|
||||
require.NoError(t, cleaner.cleanupOldBackups(ctx, cfg))
|
||||
require.NotContains(t, store.objects, "old")
|
||||
close(delayedRepo.resume)
|
||||
err := <-result
|
||||
restoring.wg.Wait()
|
||||
saved, getErr := cleaner.GetBackupRecord(ctx, "old")
|
||||
require.Nil(t, saved)
|
||||
require.ErrorIs(t, err, ErrBackupNotFound, "restore must not start against a deleted archive")
|
||||
require.ErrorIs(t, getErr, ErrBackupNotFound, "cleanup-deleted archive must not be resurrected")
|
||||
}
|
||||
|
||||
type backupFailRecordWriteRepo struct{ *mockSettingRepo }
|
||||
|
||||
func (r *backupFailRecordWriteRepo) Set(ctx context.Context, key, value string) error {
|
||||
if key == settingKeyBackupRecords {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
return r.mockSettingRepo.Set(ctx, key, value)
|
||||
}
|
||||
func TestBackupRecovery_RestoreRequiresDurableProtection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
record := archiveRecord("finite-archive", "2026-09-01T04:00:00Z")
|
||||
record.MonthlyArchive = &BackupMonthlyArchive{RetainCount: 1}
|
||||
record.S3Key = record.ID
|
||||
require.NoError(t, svc.saveRecord(ctx, record))
|
||||
svc.settingRepo = &backupFailRecordWriteRepo{repo}
|
||||
_, err := svc.StartRestore(ctx, record.ID)
|
||||
svc.wg.Wait()
|
||||
require.Error(t, err, "failed restore-running marker must prevent restore from starting")
|
||||
}
|
||||
|
||||
func TestBackupRecovery_ActiveAndLegacyRestoresKeepProtection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
svc := newTestBackupService(newMockSettingRepo(), &mockDumper{}, newMockObjectStore())
|
||||
for _, id := range []string{"active", "legacy"} {
|
||||
record := archiveRecord(id, time.Now().AddDate(-1, 0, 0).Format(time.RFC3339))
|
||||
record.MonthlyArchive = &BackupMonthlyArchive{RetainCount: 0}
|
||||
record.RestoreStatus = "running"
|
||||
if id == "active" {
|
||||
record.RestoreStartedAt = time.Now().Format(time.RFC3339)
|
||||
}
|
||||
require.NoError(t, svc.saveRecord(ctx, record))
|
||||
}
|
||||
svc.recoverStaleRecords()
|
||||
for _, id := range []string{"active", "legacy"} {
|
||||
record, err := svc.GetBackupRecord(ctx, id)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "running", record.RestoreStatus)
|
||||
require.NotEmpty(t, record.RestoreStartedAt)
|
||||
require.ErrorIs(t, svc.DeleteArchivedBackup(ctx, id), ErrRestoreInProgress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRecovery_FailedStateWriteNeverDeletesObjects(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
record := archiveRecord("interrupted", time.Now().Add(-time.Hour).Format(time.RFC3339))
|
||||
record.Status, record.S3Key = "running", "interrupted.sql.gz"
|
||||
store.objects[record.S3Key] = []byte("payload")
|
||||
require.NoError(t, svc.saveRecord(ctx, record))
|
||||
svc.settingRepo = &backupFailRecordWriteRepo{repo}
|
||||
svc.recoverStaleRecords()
|
||||
saved, err := svc.GetBackupRecord(ctx, record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "running", saved.Status)
|
||||
require.Contains(t, store.objects, record.S3Key)
|
||||
require.Empty(t, store.deletedKeys)
|
||||
}
|
||||
|
||||
func TestBackupRecovery_RestoreReservationProtectsFromPeerCleanup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
peer := newTestBackupService(repo, &mockDumper{}, store)
|
||||
cache := &fakeLeaderLockCache{}
|
||||
svc.SetLeaderLock(cache, nil)
|
||||
peer.SetLeaderLock(cache, nil)
|
||||
record := archiveRecord("old", time.Now().AddDate(-1, 0, 0).Format(time.RFC3339))
|
||||
record.S3Key = record.ID
|
||||
store.objects[record.ID] = []byte("payload")
|
||||
require.NoError(t, svc.saveRecord(ctx, record))
|
||||
restoring, err := svc.beginBackupRestore(ctx, record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "running", restoring.RestoreStatus)
|
||||
_, err = peer.beginBackupRestore(ctx, record.ID)
|
||||
require.ErrorIs(t, err, ErrRestoreInProgress)
|
||||
require.NoError(t, peer.cleanupOldBackups(ctx, archiveSchedule(1)))
|
||||
require.Contains(t, store.objects, record.ID)
|
||||
require.ErrorIs(t, peer.DeleteBackup(ctx, record.ID), ErrRestoreInProgress)
|
||||
}
|
||||
|
||||
func TestBackupRecovery_FailedRestoreCannotRecreateMissingRecord(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
svc := newTestBackupService(newMockSettingRepo(), &mockDumper{}, newMockObjectStore())
|
||||
record := archiveRecord("removed", time.Now().Format(time.RFC3339))
|
||||
record.RestoreStatus = "failed"
|
||||
require.ErrorIs(t, svc.saveRestoreRecord(ctx, record), ErrBackupNotFound)
|
||||
_, err := svc.GetBackupRecord(ctx, record.ID)
|
||||
require.ErrorIs(t, err, ErrBackupNotFound)
|
||||
}
|
||||
|
||||
func TestBackupRecovery_LateRestoreResultCannotReleaseNewProtection(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
svc := newTestBackupService(newMockSettingRepo(), &mockDumper{}, newMockObjectStore())
|
||||
record := archiveRecord("restoring", time.Now().AddDate(-1, 0, 0).Format(time.RFC3339))
|
||||
record.RestoreStatus = "running"
|
||||
record.RestoreStartedAt = time.Now().Add(-time.Hour).Format(time.RFC3339)
|
||||
require.NoError(t, svc.saveRecord(ctx, record))
|
||||
svc.recoverStaleRecords()
|
||||
current, err := svc.beginBackupRestore(ctx, record.ID)
|
||||
require.NoError(t, err)
|
||||
for _, status := range []string{"failed", "completed"} {
|
||||
record.RestoreStatus = status
|
||||
require.Error(t, svc.saveRestoreRecord(ctx, record))
|
||||
saved, err := svc.GetBackupRecord(ctx, record.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "running", saved.RestoreStatus)
|
||||
require.Equal(t, current.RestoreStartedAt, saved.RestoreStartedAt)
|
||||
require.ErrorIs(t, svc.DeleteBackup(ctx, record.ID), ErrRestoreInProgress)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
|
||||
)
|
||||
|
||||
func (s *BackupService) beginBackupRestore(ctx context.Context, id string) (*BackupRecord, error) {
|
||||
ctx, release, err := s.lockBackupRecordUpdates(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
records, err := s.loadRecordsLocked(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range records {
|
||||
if records[i].ID != id {
|
||||
continue
|
||||
}
|
||||
if records[i].Status != "completed" {
|
||||
return nil, infraerrors.BadRequest("BACKUP_NOT_COMPLETED", "can only restore from a completed backup")
|
||||
}
|
||||
if records[i].RestoreStatus == "running" {
|
||||
return nil, ErrRestoreInProgress
|
||||
}
|
||||
records[i].RestoreStatus = "running"
|
||||
records[i].RestoreStartedAt = time.Now().Format(time.RFC3339)
|
||||
records[i].RestoreError = ""
|
||||
if err := s.saveRecordsLocked(ctx, records); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &records[i], nil
|
||||
}
|
||||
return nil, ErrBackupNotFound
|
||||
}
|
||||
|
||||
// Failed/panicked restores must not resurrect a removed record. A successful
|
||||
// full database restore may itself roll back settings and remove its own record,
|
||||
// so that completion keeps the existing explicit re-registration behavior.
|
||||
func (s *BackupService) saveRestoreRecord(ctx context.Context, record *BackupRecord) error {
|
||||
ctx, release, err := s.lockBackupRecordUpdates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
records, err := s.loadRecordsLocked(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range records {
|
||||
if records[i].ID != record.ID {
|
||||
continue
|
||||
}
|
||||
if records[i].RestoreStartedAt != record.RestoreStartedAt &&
|
||||
(record.RestoreStatus != "completed" || (records[i].RestoreStatus == "running" && records[i].RestoreStartedAt != "")) {
|
||||
return infraerrors.Conflict("BACKUP_RESTORE_STATE_CHANGED", "a newer restore operation owns this backup")
|
||||
}
|
||||
if record.RestoreStatus == "completed" {
|
||||
// Restoring the full database may roll the backup record itself back
|
||||
// to its pre-upload state. Re-register the completed snapshot while
|
||||
// retaining any archive protection already persisted in the database.
|
||||
snapshot := *record
|
||||
if records[i].MonthlyArchive != nil {
|
||||
snapshot.MonthlyArchive = records[i].MonthlyArchive
|
||||
snapshot.ExpiresAt = ""
|
||||
}
|
||||
records[i] = snapshot
|
||||
return s.saveRecordsLocked(ctx, records)
|
||||
}
|
||||
// Only the restore operation's fields change, preserving current archive
|
||||
// metadata and any unrelated progress recorded by another writer.
|
||||
records[i].RestoreStatus = record.RestoreStatus
|
||||
records[i].RestoreError = record.RestoreError
|
||||
records[i].RestoredAt = record.RestoredAt
|
||||
return s.saveRecordsLocked(ctx, records)
|
||||
}
|
||||
if record.RestoreStatus == "completed" {
|
||||
return s.saveRecordsLocked(ctx, append(records, *record))
|
||||
}
|
||||
return ErrBackupNotFound
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
|
||||
const settingKeyBackupArchiveCheckpoint = "backup_monthly_archive_checkpoint"
|
||||
|
||||
// Records and the archive checkpoint share a read/modify/write boundary across
|
||||
// instances, including manual backups and restore status updates. Use the same
|
||||
// backend consistently; a Redis error must not silently switch to a DB lock
|
||||
// while another instance still owns the Redis lock.
|
||||
func (s *BackupService) lockBackupRecordUpdates(ctx context.Context) (context.Context, func(), error) {
|
||||
s.recordsMu.Lock()
|
||||
writeCtx, cancel := context.WithTimeout(ctx, 30*time.Minute)
|
||||
waitCtx, stopWaiting := context.WithTimeout(writeCtx, 10*time.Second)
|
||||
defer stopWaiting()
|
||||
const key = "backup:records:writer"
|
||||
for {
|
||||
var release func()
|
||||
var ok bool
|
||||
if s.lockCache != nil {
|
||||
var err error
|
||||
ok, err = s.lockCache.TryAcquireLeaderLock(waitCtx, key, s.instanceID, backupScheduledLeaderLockTTL)
|
||||
if err != nil {
|
||||
cancel()
|
||||
s.recordsMu.Unlock()
|
||||
return nil, nil, fmt.Errorf("lock backup records: %w", err)
|
||||
}
|
||||
release = func() {
|
||||
ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer done()
|
||||
_ = s.lockCache.ReleaseLeaderLock(ctx, key, s.instanceID)
|
||||
}
|
||||
} else if s.db != nil {
|
||||
// A session advisory lock holds a connection while the repository
|
||||
// needs another. Redis avoids this requirement in production.
|
||||
if s.db.Stats().MaxOpenConnections == 1 {
|
||||
cancel()
|
||||
s.recordsMu.Unlock()
|
||||
return nil, nil, fmt.Errorf("backup record locking requires Redis or at least two database connections")
|
||||
}
|
||||
var err error
|
||||
release, ok, err = tryAcquireDBAdvisoryLockWithError(waitCtx, s.db, hashAdvisoryLockID(key))
|
||||
if err != nil {
|
||||
cancel()
|
||||
s.recordsMu.Unlock()
|
||||
return nil, nil, fmt.Errorf("lock backup records: %w", err)
|
||||
}
|
||||
} else {
|
||||
release, ok = func() {}, true
|
||||
}
|
||||
if ok {
|
||||
return writeCtx, func() { release(); cancel(); s.recordsMu.Unlock() }, nil
|
||||
}
|
||||
select {
|
||||
case <-waitCtx.Done():
|
||||
cancel()
|
||||
s.recordsMu.Unlock()
|
||||
return nil, nil, fmt.Errorf("lock backup records: %w", waitCtx.Err())
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type BackupMonthlyArchiveConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Days []int `json:"days"`
|
||||
IncludeMonthEnd bool `json:"include_month_end"`
|
||||
RetainCount int `json:"retain_count"` // 0 = permanent; positive = finite archive pool size.
|
||||
}
|
||||
|
||||
type BackupMonthlyArchive struct {
|
||||
Dates []string `json:"dates"` // Deduplicated target dates covered by this backup.
|
||||
RetainCount int `json:"retain_count"` // 0 is permanent and never implicitly downgraded.
|
||||
}
|
||||
|
||||
func validateBackupRetention(cfg *BackupScheduleConfig) error {
|
||||
if cfg.RetainDays < 0 || cfg.RetainCount < 0 {
|
||||
return infraerrors.BadRequest("INVALID_BACKUP_RETENTION", "backup retention must not be negative")
|
||||
}
|
||||
archive := cfg.MonthlyArchive
|
||||
if archive == nil {
|
||||
return nil
|
||||
}
|
||||
if archive.RetainCount < 0 || len(archive.Days) > 31 {
|
||||
return infraerrors.BadRequest("INVALID_MONTHLY_ARCHIVE", "invalid archive retention or dates")
|
||||
}
|
||||
seen := make(map[int]bool)
|
||||
days := make([]int, 0, len(archive.Days))
|
||||
for _, day := range archive.Days {
|
||||
if day < 1 || day > 31 {
|
||||
return infraerrors.BadRequest("INVALID_MONTHLY_ARCHIVE", "archive dates must be between 1 and 31")
|
||||
}
|
||||
if !seen[day] {
|
||||
days = append(days, day)
|
||||
seen[day] = true
|
||||
}
|
||||
}
|
||||
sort.Ints(days)
|
||||
archive.Days = days
|
||||
if archive.Enabled && len(days) == 0 && !archive.IncludeMonthEnd {
|
||||
return infraerrors.BadRequest("INVALID_MONTHLY_ARCHIVE", "select at least one archive date")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// assignMonthlyArchive runs under recordsMu and only for a completed scheduled
|
||||
// backup. Successful processing advances a durable date checkpoint even when
|
||||
// no date is due. Removing a record or reducing retention cannot reset it.
|
||||
func (s *BackupService) assignMonthlyArchive(ctx context.Context, record *BackupRecord, schedule *BackupScheduleConfig) (string, error) {
|
||||
if record.Status != "completed" || record.TriggeredBy != "scheduled" || record.MonthlyArchive != nil || schedule == nil || schedule.MonthlyArchive == nil || !schedule.MonthlyArchive.Enabled {
|
||||
return "", nil
|
||||
}
|
||||
if err := validateBackupRetention(schedule); err != nil {
|
||||
return "", err
|
||||
}
|
||||
startedAt, err := time.Parse(time.RFC3339, record.StartedAt)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse archive backup start: %w", err)
|
||||
}
|
||||
parser := cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow)
|
||||
spec, err := parser.Parse(schedule.CronExpr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("parse archive schedule: %w", err)
|
||||
}
|
||||
location := time.Local
|
||||
if parsed, ok := spec.(*cron.SpecSchedule); ok {
|
||||
location = parsed.Location
|
||||
}
|
||||
startedAt = startedAt.In(location)
|
||||
today := startedAt.Format(time.DateOnly)
|
||||
checkpoint, err := s.settingRepo.GetValue(ctx, settingKeyBackupArchiveCheckpoint)
|
||||
if err != nil && !errors.Is(err, ErrSettingNotFound) {
|
||||
return "", fmt.Errorf("load archive checkpoint: %w", err)
|
||||
}
|
||||
if checkpoint != "" {
|
||||
if _, err := time.Parse(time.DateOnly, checkpoint); err != nil {
|
||||
return "", infraerrors.InternalServer("BACKUP_ARCHIVE_CHECKPOINT_CORRUPT", "archive checkpoint data is corrupted")
|
||||
}
|
||||
if checkpoint >= today {
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
dates := monthlyArchiveDates(startedAt, schedule.MonthlyArchive)
|
||||
var due []string
|
||||
for _, date := range dates {
|
||||
if date > checkpoint && date <= today {
|
||||
due = append(due, date)
|
||||
}
|
||||
}
|
||||
if len(due) > 0 {
|
||||
record.MonthlyArchive = &BackupMonthlyArchive{Dates: due, RetainCount: schedule.MonthlyArchive.RetainCount}
|
||||
record.ExpiresAt = ""
|
||||
}
|
||||
return today, nil
|
||||
}
|
||||
|
||||
func monthlyArchiveDates(at time.Time, cfg *BackupMonthlyArchiveConfig) []string {
|
||||
lastDay := time.Date(at.Year(), at.Month()+1, 0, 0, 0, 0, 0, at.Location()).Day()
|
||||
unique := make(map[int]bool)
|
||||
for _, day := range cfg.Days {
|
||||
unique[min(day, lastDay)] = true
|
||||
}
|
||||
if cfg.IncludeMonthEnd {
|
||||
unique[lastDay] = true
|
||||
}
|
||||
dates := make([]string, 0, len(unique))
|
||||
for day := range unique {
|
||||
dates = append(dates, time.Date(at.Year(), at.Month(), day, 0, 0, 0, 0, at.Location()).Format(time.DateOnly))
|
||||
}
|
||||
sort.Strings(dates)
|
||||
return dates
|
||||
}
|
||||
|
||||
func backupStartedAfter(a, b BackupRecord) bool {
|
||||
left, leftErr := time.Parse(time.RFC3339, a.StartedAt)
|
||||
right, rightErr := time.Parse(time.RFC3339, b.StartedAt)
|
||||
if leftErr == nil && rightErr == nil && !left.Equal(right) {
|
||||
return left.After(right)
|
||||
}
|
||||
if a.StartedAt != b.StartedAt {
|
||||
return a.StartedAt > b.StartedAt
|
||||
}
|
||||
return a.ID > b.ID
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func archiveSchedule(days ...int) *BackupScheduleConfig {
|
||||
return &BackupScheduleConfig{CronExpr: "CRON_TZ=UTC 0 4 * * *", RetainDays: 14, RetainCount: 10,
|
||||
MonthlyArchive: &BackupMonthlyArchiveConfig{Enabled: true, Days: days}}
|
||||
}
|
||||
|
||||
func archiveRecord(id, started string) *BackupRecord {
|
||||
return &BackupRecord{ID: id, Status: "completed", TriggeredBy: "scheduled", StartedAt: started, ExpiresAt: "2099-01-01T00:00:00Z"}
|
||||
}
|
||||
|
||||
func TestBackupRetention_ValidationAndRoundTrip(t *testing.T) {
|
||||
for _, cfg := range []BackupScheduleConfig{
|
||||
{RetainDays: -1}, {RetainCount: -1},
|
||||
{MonthlyArchive: &BackupMonthlyArchiveConfig{Enabled: true}},
|
||||
{MonthlyArchive: &BackupMonthlyArchiveConfig{Days: []int{0}}},
|
||||
{MonthlyArchive: &BackupMonthlyArchiveConfig{Days: []int{32}}},
|
||||
{MonthlyArchive: &BackupMonthlyArchiveConfig{RetainCount: -1}},
|
||||
} {
|
||||
require.Error(t, validateBackupRetention(&cfg))
|
||||
}
|
||||
repo := newMockSettingRepo()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
cfg := archiveSchedule(15, 1, 15)
|
||||
cfg.RetainDays, cfg.RetainCount = 0, 0
|
||||
cfg.MonthlyArchive.IncludeMonthEnd = true
|
||||
cfg.MonthlyArchive.RetainCount = 17
|
||||
saved, err := svc.UpdateSchedule(context.Background(), *cfg)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []int{1, 15}, saved.MonthlyArchive.Days)
|
||||
loaded, err := svc.GetSchedule(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, saved, loaded)
|
||||
require.Zero(t, loaded.RetainDays)
|
||||
require.Zero(t, loaded.RetainCount)
|
||||
}
|
||||
|
||||
func TestBackupRetention_MonthEndAndLeapYears(t *testing.T) {
|
||||
for _, tc := range []struct{ date, end string }{
|
||||
{"2026-02-01", "2026-02-28"}, {"2028-02-01", "2028-02-29"}, {"2026-04-01", "2026-04-30"},
|
||||
} {
|
||||
t.Run(tc.date, func(t *testing.T) {
|
||||
at, err := time.Parse(time.DateOnly, tc.date)
|
||||
require.NoError(t, err)
|
||||
cfg := &BackupMonthlyArchiveConfig{Days: []int{30, 31}, IncludeMonthEnd: true}
|
||||
require.Equal(t, []string{tc.end}, monthlyArchiveDates(at, cfg))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRetention_SuccessOnlyFallbackAndMultipleDates(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
cfg := archiveSchedule(1, 15)
|
||||
ctx := context.Background()
|
||||
failed := archiveRecord("failed", "2026-09-01T04:00:00Z")
|
||||
failed.Status = "failed"
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, failed, cfg))
|
||||
manual := archiveRecord("manual", "2026-09-01T05:00:00Z")
|
||||
manual.TriggeredBy = "manual"
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, manual, cfg))
|
||||
require.Nil(t, failed.MonthlyArchive)
|
||||
require.Nil(t, manual.MonthlyArchive)
|
||||
first := archiveRecord("first", "2026-09-02T04:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, first, cfg))
|
||||
require.Equal(t, []string{"2026-09-01"}, first.MonthlyArchive.Dates)
|
||||
require.Empty(t, first.ExpiresAt)
|
||||
again := archiveRecord("again", "2026-09-02T06:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, again, cfg))
|
||||
require.Nil(t, again.MonthlyArchive)
|
||||
next := archiveRecord("next", "2026-09-16T04:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, next, cfg))
|
||||
require.Equal(t, []string{"2026-09-15"}, next.MonthlyArchive.Dates)
|
||||
// A later month does not manufacture archives for the missed previous month.
|
||||
newMonth := archiveRecord("oct", "2026-10-20T04:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, newMonth, cfg))
|
||||
require.Equal(t, []string{"2026-10-01", "2026-10-15"}, newMonth.MonthlyArchive.Dates)
|
||||
}
|
||||
|
||||
func TestBackupRetention_ScheduleTimezoneAndStartDate(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
cfg := archiveSchedule(1, 31)
|
||||
cfg.CronExpr = "CRON_TZ=Asia/Tokyo 0 4 * * *"
|
||||
first := archiveRecord("tokyo", "2026-08-31T16:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(context.Background(), first, cfg))
|
||||
require.Equal(t, []string{"2026-09-01"}, first.MonthlyArchive.Dates)
|
||||
last := archiveRecord("cross-midnight", "2026-09-30T14:59:00Z")
|
||||
last.FinishedAt = "2026-09-30T15:10:00Z"
|
||||
require.NoError(t, svc.saveRecordWithArchive(context.Background(), last, cfg))
|
||||
require.Equal(t, []string{"2026-09-30"}, last.MonthlyArchive.Dates)
|
||||
}
|
||||
|
||||
func TestBackupRetention_CheckpointSurvivesRecordRemovalAndRestart(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
cfg := archiveSchedule(1, 15)
|
||||
ctx := context.Background()
|
||||
record := archiveRecord("archive", "2026-09-16T04:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, record, cfg))
|
||||
require.ErrorIs(t, svc.DeleteBackup(ctx, record.ID), ErrBackupArchiveProtected)
|
||||
require.NoError(t, svc.DeleteArchivedBackup(ctx, record.ID))
|
||||
svc = newTestBackupService(repo, &mockDumper{}, store)
|
||||
retry := archiveRecord("retry", "2026-09-17T04:00:00Z")
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, retry, cfg))
|
||||
require.Nil(t, retry.MonthlyArchive)
|
||||
}
|
||||
|
||||
func TestBackupRetention_CleanupSeparatePoolsAndPermanentProtection(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
store := newMockObjectStore()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, store)
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
for i, id := range []string{"running", "failed", "ordinary-new", "archive-new", "permanent", "ordinary-old", "archive-old", "restoring"} {
|
||||
r := &BackupRecord{ID: id, Status: "completed", StartedAt: now.Add(-time.Duration(i) * time.Hour).Format(time.RFC3339), S3Key: id}
|
||||
switch id {
|
||||
case "running", "failed":
|
||||
r.Status = id
|
||||
case "archive-new", "archive-old":
|
||||
r.MonthlyArchive = &BackupMonthlyArchive{RetainCount: 12}
|
||||
case "permanent":
|
||||
r.MonthlyArchive = &BackupMonthlyArchive{}
|
||||
case "restoring":
|
||||
r.RestoreStatus = "running"
|
||||
}
|
||||
store.objects[id] = []byte(id)
|
||||
require.NoError(t, svc.saveRecord(ctx, r))
|
||||
}
|
||||
cfg := &BackupScheduleConfig{RetainDays: 1, RetainCount: 1, MonthlyArchive: &BackupMonthlyArchiveConfig{RetainCount: 1}}
|
||||
require.NoError(t, svc.cleanupOldBackups(ctx, cfg))
|
||||
for _, id := range []string{"running", "failed", "ordinary-new", "archive-new", "permanent", "restoring"} {
|
||||
_, err := svc.GetBackupRecord(ctx, id)
|
||||
require.NoError(t, err, id)
|
||||
}
|
||||
for _, id := range []string{"ordinary-old", "archive-old"} {
|
||||
_, err := svc.GetBackupRecord(ctx, id)
|
||||
require.ErrorIs(t, err, ErrBackupNotFound)
|
||||
require.NotContains(t, store.objects, id)
|
||||
}
|
||||
finite, err := svc.GetBackupRecord(ctx, "archive-new")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, finite.MonthlyArchive.RetainCount)
|
||||
// Switching the rule off or to permanent does not downgrade prior permanent archives.
|
||||
cfg.MonthlyArchive = &BackupMonthlyArchiveConfig{}
|
||||
cfg.RetainDays = 0
|
||||
require.NoError(t, svc.cleanupOldBackups(ctx, cfg))
|
||||
_, err = svc.GetBackupRecord(ctx, "permanent")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestBackupRetention_Over100ArchivesRemainDiscoverable(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
ctx := context.Background()
|
||||
for i := range 105 {
|
||||
record := archiveRecord(fmt.Sprint(i), "2020-01-01T00:00:00Z")
|
||||
record.MonthlyArchive = &BackupMonthlyArchive{}
|
||||
require.NoError(t, svc.saveRecord(ctx, record))
|
||||
}
|
||||
require.NoError(t, svc.cleanupOldBackups(ctx, &BackupScheduleConfig{RetainDays: 1, RetainCount: 1}))
|
||||
records, err := svc.ListBackups(ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 105)
|
||||
}
|
||||
|
||||
type failingArchiveRepo struct {
|
||||
*mockSettingRepo
|
||||
fail bool
|
||||
}
|
||||
|
||||
func (r *failingArchiveRepo) SetMultiple(ctx context.Context, values map[string]string) error {
|
||||
if r.fail {
|
||||
return errors.New("injected atomic save failure")
|
||||
}
|
||||
return r.mockSettingRepo.SetMultiple(ctx, values)
|
||||
}
|
||||
|
||||
func TestBackupRetention_PersistenceFailuresFailClosed(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
ctx := context.Background()
|
||||
failing := &failingArchiveRepo{mockSettingRepo: repo, fail: true}
|
||||
svc.settingRepo = failing
|
||||
record := archiveRecord("new", "2026-09-01T04:00:00Z")
|
||||
cfg := archiveSchedule(1)
|
||||
require.Error(t, svc.saveRecordWithArchive(ctx, record, cfg))
|
||||
require.Nil(t, record.MonthlyArchive, "failed commit must not mutate the caller's retry state")
|
||||
require.Empty(t, repo.data[settingKeyBackupArchiveCheckpoint])
|
||||
failing.fail = false
|
||||
require.NoError(t, svc.saveRecordWithArchive(ctx, record, cfg))
|
||||
require.NotNil(t, record.MonthlyArchive)
|
||||
oldRecords := repo.data[settingKeyBackupRecords]
|
||||
repo.getValueErr = errors.New("read failed")
|
||||
require.Error(t, svc.saveRecord(ctx, archiveRecord("other", "2026-09-02T04:00:00Z")))
|
||||
require.Equal(t, oldRecords, repo.data[settingKeyBackupRecords])
|
||||
require.Error(t, svc.cleanupOldBackups(ctx, cfg))
|
||||
require.Error(t, func() error { _, err := svc.GetSchedule(ctx); return err }())
|
||||
repo.getValueErr = nil
|
||||
repo.data[settingKeyBackupRecords] = "corrupted"
|
||||
require.ErrorIs(t, svc.saveRecord(ctx, record), ErrBackupRecordsCorrupt)
|
||||
require.Equal(t, "corrupted", repo.data[settingKeyBackupRecords])
|
||||
repo.data[settingKeyBackupRecords] = oldRecords
|
||||
repo.data[settingKeyBackupArchiveCheckpoint] = "bad date"
|
||||
require.Error(t, svc.saveRecordWithArchive(ctx, archiveRecord("later", "2026-09-02T04:00:00Z"), cfg))
|
||||
require.Equal(t, oldRecords, repo.data[settingKeyBackupRecords])
|
||||
}
|
||||
|
||||
func TestBackupRetention_ScheduledExecutionAndZeroDays(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
seedS3Config(t, repo)
|
||||
svc := newTestBackupService(repo, &mockDumper{dumpData: []byte("backup")}, newMockObjectStore())
|
||||
cfg := archiveSchedule(time.Now().UTC().Day())
|
||||
cfg.RetainDays = 0
|
||||
raw, err := json.Marshal(cfg)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, repo.Set(context.Background(), settingKeyBackupSchedule, string(raw)))
|
||||
svc.runScheduledBackup()
|
||||
records, err := svc.ListBackups(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 1)
|
||||
require.NotNil(t, records[0].MonthlyArchive)
|
||||
require.Empty(t, records[0].ExpiresAt)
|
||||
// The following successful backup is ordinary and must still honor zero days.
|
||||
svc.runScheduledBackup()
|
||||
records, err = svc.ListBackups(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 2)
|
||||
for _, record := range records {
|
||||
require.Empty(t, record.ExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackupRetention_CrossInstanceRecordUpdates(t *testing.T) {
|
||||
repo := newMockSettingRepo()
|
||||
cache := &fakeLeaderLockCache{}
|
||||
services := []*BackupService{
|
||||
newTestBackupService(repo, &mockDumper{}, newMockObjectStore()),
|
||||
newTestBackupService(repo, &mockDumper{}, newMockObjectStore()),
|
||||
}
|
||||
for _, svc := range services {
|
||||
svc.SetLeaderLock(cache, nil)
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 24)
|
||||
for i := range 24 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errors <- services[i%2].saveRecord(context.Background(), archiveRecord(fmt.Sprint(i), "2026-09-01T00:00:00Z"))
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
require.NoError(t, err)
|
||||
}
|
||||
records, err := services[0].ListBackups(context.Background())
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 24)
|
||||
}
|
||||
|
||||
func TestBackupRetention_RecordLockWithSingleConnectionPool(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
repo := newMockSettingRepo()
|
||||
svc := newTestBackupService(repo, &mockDumper{}, newMockObjectStore())
|
||||
svc.SetLeaderLock(&fakeLeaderLockCache{}, db)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
record := archiveRecord("single-connection", "2026-09-01T00:00:00Z")
|
||||
require.NoError(t, svc.saveRecord(ctx, record), "Redis locking must not occupy a database connection")
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
svc.SetLeaderLock(nil, db)
|
||||
require.ErrorContains(t, svc.saveRecord(ctx, record), "at least two database connections")
|
||||
}
|
||||
@@ -30,7 +30,6 @@ const (
|
||||
settingKeyBackupSchedule = "backup_schedule"
|
||||
settingKeyBackupRecords = "backup_records"
|
||||
|
||||
maxBackupRecords = 100
|
||||
backupObjectCleanupTimeout = 2 * time.Minute
|
||||
|
||||
// backupScheduledLeaderLockKey gates the scheduled full-database backup so
|
||||
@@ -50,12 +49,13 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrBackupS3NotConfigured = infraerrors.BadRequest("BACKUP_S3_NOT_CONFIGURED", "backup S3 storage is not configured")
|
||||
ErrBackupNotFound = infraerrors.NotFound("BACKUP_NOT_FOUND", "backup record not found")
|
||||
ErrBackupInProgress = infraerrors.Conflict("BACKUP_IN_PROGRESS", "a backup is already in progress")
|
||||
ErrRestoreInProgress = infraerrors.Conflict("RESTORE_IN_PROGRESS", "a restore is already in progress")
|
||||
ErrBackupRecordsCorrupt = infraerrors.InternalServer("BACKUP_RECORDS_CORRUPT", "backup records data is corrupted")
|
||||
ErrBackupS3ConfigCorrupt = infraerrors.InternalServer("BACKUP_S3_CONFIG_CORRUPT", "backup S3 config data is corrupted")
|
||||
ErrBackupS3NotConfigured = infraerrors.BadRequest("BACKUP_S3_NOT_CONFIGURED", "backup S3 storage is not configured")
|
||||
ErrBackupNotFound = infraerrors.NotFound("BACKUP_NOT_FOUND", "backup record not found")
|
||||
ErrBackupInProgress = infraerrors.Conflict("BACKUP_IN_PROGRESS", "a backup is already in progress")
|
||||
ErrRestoreInProgress = infraerrors.Conflict("RESTORE_IN_PROGRESS", "a restore is already in progress")
|
||||
ErrBackupRecordsCorrupt = infraerrors.InternalServer("BACKUP_RECORDS_CORRUPT", "backup records data is corrupted")
|
||||
ErrBackupS3ConfigCorrupt = infraerrors.InternalServer("BACKUP_S3_CONFIG_CORRUPT", "backup S3 config data is corrupted")
|
||||
ErrBackupArchiveProtected = infraerrors.Conflict("BACKUP_ARCHIVE_PROTECTED", "explicit confirmation is required to delete an archived backup")
|
||||
|
||||
// ErrSecretEncryptionKeyNotConfigured is returned when an S3 SecretAccessKey
|
||||
// would be encrypted with an auto-generated (ephemeral) key. That key is
|
||||
@@ -111,30 +111,33 @@ func (c *BackupS3Config) IsConfigured() bool {
|
||||
|
||||
// BackupScheduleConfig 定时备份配置
|
||||
type BackupScheduleConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
CronExpr string `json:"cron_expr"` // cron 表达式,如 "0 2 * * *" 每天凌晨2点
|
||||
RetainDays int `json:"retain_days"` // 备份文件过期天数,默认14,0=不自动清理
|
||||
RetainCount int `json:"retain_count"` // 最多保留份数,0=不限制
|
||||
Enabled bool `json:"enabled"`
|
||||
CronExpr string `json:"cron_expr"` // cron 表达式,如 "0 2 * * *" 每天凌晨2点
|
||||
RetainDays int `json:"retain_days"` // 备份文件过期天数,默认14,0=不自动清理
|
||||
RetainCount int `json:"retain_count"` // 最多保留份数,0=不限制
|
||||
MonthlyArchive *BackupMonthlyArchiveConfig `json:"monthly_archive,omitempty"`
|
||||
}
|
||||
|
||||
// BackupRecord 备份记录
|
||||
type BackupRecord struct {
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"` // pending, running, completed, failed
|
||||
BackupType string `json:"backup_type"` // postgres
|
||||
FileName string `json:"file_name"`
|
||||
S3Key string `json:"s3_key"`
|
||||
Parts []BackupPart `json:"parts,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
TriggeredBy string `json:"triggered_by"` // manual, scheduled
|
||||
ErrorMsg string `json:"error_message,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
ExpiresAt string `json:"expires_at,omitempty"` // 过期时间
|
||||
Progress string `json:"progress,omitempty"` // "dumping", "uploading", ""
|
||||
RestoreStatus string `json:"restore_status,omitempty"` // "", "running", "completed", "failed"
|
||||
RestoreError string `json:"restore_error,omitempty"`
|
||||
RestoredAt string `json:"restored_at,omitempty"`
|
||||
ID string `json:"id"`
|
||||
Status string `json:"status"` // pending, running, completed, failed
|
||||
BackupType string `json:"backup_type"` // postgres
|
||||
FileName string `json:"file_name"`
|
||||
S3Key string `json:"s3_key"`
|
||||
Parts []BackupPart `json:"parts,omitempty"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
TriggeredBy string `json:"triggered_by"` // manual, scheduled
|
||||
ErrorMsg string `json:"error_message,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
FinishedAt string `json:"finished_at,omitempty"`
|
||||
ExpiresAt string `json:"expires_at,omitempty"` // 过期时间
|
||||
Progress string `json:"progress,omitempty"` // "dumping", "uploading", ""
|
||||
RestoreStatus string `json:"restore_status,omitempty"` // "", "running", "completed", "failed"
|
||||
RestoreStartedAt string `json:"restore_started_at,omitempty"`
|
||||
RestoreError string `json:"restore_error,omitempty"`
|
||||
RestoredAt string `json:"restored_at,omitempty"`
|
||||
MonthlyArchive *BackupMonthlyArchive `json:"monthly_archive,omitempty"`
|
||||
}
|
||||
|
||||
// BackupDownloadPart 描述一个可下载的备份分卷。
|
||||
@@ -228,6 +231,9 @@ func (s *BackupService) SetLeaderLock(lockCache LeaderLockCache, db *sql.DB) {
|
||||
// Start 启动定时备份调度器并清理孤立记录
|
||||
func (s *BackupService) Start() {
|
||||
s.cronSched = cron.New()
|
||||
// A recently interrupted job may still belong to a live peer. Revisit it
|
||||
// after the maximum execution window rather than deleting it at startup.
|
||||
_, _ = s.cronSched.AddFunc("@every 5m", s.recoverStaleRecords)
|
||||
s.cronSched.Start()
|
||||
|
||||
// 清理重启后孤立的 running 记录
|
||||
@@ -248,53 +254,78 @@ func (s *BackupService) Start() {
|
||||
}
|
||||
}
|
||||
|
||||
// recoverStaleRecords 启动时将孤立的 running 记录标记为 failed,并清理已上传对象。
|
||||
// recoverStaleRecords expires interrupted operations only after their maximum
|
||||
// runtime. Re-read, status persistence and object deletion share the writer lock.
|
||||
func (s *BackupService) recoverStaleRecords() {
|
||||
loadCtx, loadCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer loadCancel()
|
||||
|
||||
records, err := s.loadRecords(loadCtx)
|
||||
ctx, cancel := context.WithTimeout(s.bgCtx, 5*time.Minute)
|
||||
defer cancel()
|
||||
ctx, release, err := s.lockBackupRecordUpdates(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
records, err := s.loadRecordsLocked(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
changed := false
|
||||
var stale []int
|
||||
for i := range records {
|
||||
if records[i].Status == "running" {
|
||||
staleRecord := records[i]
|
||||
if records[i].Status == "running" && records[i].MonthlyArchive == nil && backupOperationExpired(records[i].StartedAt, now) {
|
||||
records[i].Status = "failed"
|
||||
records[i].ErrorMsg = "interrupted by server restart"
|
||||
records[i].Progress = ""
|
||||
records[i].FinishedAt = time.Now().Format(time.RFC3339)
|
||||
s.saveRecoveredRecord(&records[i])
|
||||
|
||||
if cleanupErr := s.cleanupStaleBackupObjects(&staleRecord); cleanupErr != nil {
|
||||
records[i].ErrorMsg = fmt.Sprintf("interrupted by server restart; cleanup failed, manual deletion may be required: %v", cleanupErr)
|
||||
s.saveRecoveredRecord(&records[i])
|
||||
logger.LegacyPrintf("service.backup", "[Backup] failed to clean stale backup objects for %s: %v", records[i].ID, cleanupErr)
|
||||
}
|
||||
logger.LegacyPrintf("service.backup", "[Backup] recovered stale running record: %s", records[i].ID)
|
||||
records[i].FinishedAt = now.Format(time.RFC3339)
|
||||
changed = true
|
||||
stale = append(stale, i)
|
||||
}
|
||||
if records[i].RestoreStatus == "running" {
|
||||
if records[i].RestoreStartedAt == "" {
|
||||
// Legacy restores have no operation timestamp. Give them a full
|
||||
// grace period; the backup's creation date says nothing about it.
|
||||
records[i].RestoreStartedAt = now.Format(time.RFC3339)
|
||||
changed = true
|
||||
continue
|
||||
}
|
||||
if !backupOperationExpired(records[i].RestoreStartedAt, now) {
|
||||
continue
|
||||
}
|
||||
records[i].RestoreStatus = "failed"
|
||||
records[i].RestoreError = "interrupted by server restart"
|
||||
s.saveRecoveredRecord(&records[i])
|
||||
logger.LegacyPrintf("service.backup", "[Backup] recovered stale restoring record: %s", records[i].ID)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
if err := s.saveRecordsLocked(ctx, records); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存中断记录失败: %v", err)
|
||||
return // Never delete objects if the state transition did not commit.
|
||||
}
|
||||
for _, i := range stale {
|
||||
if cleanupErr := s.cleanupStaleBackupObjects(ctx, &records[i]); cleanupErr != nil {
|
||||
records[i].ErrorMsg = fmt.Sprintf("interrupted by server restart; cleanup failed, manual deletion may be required: %v", cleanupErr)
|
||||
logger.LegacyPrintf("service.backup", "[Backup] failed to clean stale backup objects for %s: %v", records[i].ID, cleanupErr)
|
||||
}
|
||||
}
|
||||
if len(stale) > 0 {
|
||||
if err := s.saveRecordsLocked(ctx, records); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存中断清理结果失败: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *BackupService) saveRecoveredRecord(record *BackupRecord) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := s.saveRecord(ctx, record); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存恢复后的备份记录失败 %s: %v", record.ID, err)
|
||||
}
|
||||
func backupOperationExpired(started string, now time.Time) bool {
|
||||
at, err := time.Parse(time.RFC3339, started)
|
||||
return err == nil && now.Sub(at) > backupScheduledLeaderLockTTL
|
||||
}
|
||||
|
||||
func (s *BackupService) cleanupStaleBackupObjects(record *BackupRecord) error {
|
||||
func (s *BackupService) cleanupStaleBackupObjects(ctx context.Context, record *BackupRecord) error {
|
||||
if len(backupObjectKeys(record)) == 0 {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), backupObjectCleanupTimeout)
|
||||
ctx, cancel := context.WithTimeout(ctx, backupObjectCleanupTimeout)
|
||||
defer cancel()
|
||||
return s.deleteBackupObjects(ctx, record)
|
||||
}
|
||||
@@ -303,15 +334,19 @@ func (s *BackupService) cleanupStaleBackupObjects(record *BackupRecord) error {
|
||||
func (s *BackupService) Stop() {
|
||||
s.shuttingDown.Store(true)
|
||||
|
||||
var cronStopped context.Context
|
||||
s.cronMu.Lock()
|
||||
if s.cronSched != nil {
|
||||
s.cronSched.Stop()
|
||||
cronStopped = s.cronSched.Stop()
|
||||
}
|
||||
s.cronMu.Unlock()
|
||||
|
||||
// 等待活跃备份/恢复完成(最多 5 分钟)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
if cronStopped != nil {
|
||||
<-cronStopped.Done()
|
||||
}
|
||||
s.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
@@ -419,17 +454,23 @@ func (s *BackupService) TestS3Connection(ctx context.Context, cfg BackupS3Config
|
||||
|
||||
func (s *BackupService) GetSchedule(ctx context.Context) (*BackupScheduleConfig, error) {
|
||||
raw, err := s.settingRepo.GetValue(ctx, settingKeyBackupSchedule)
|
||||
if err != nil || raw == "" {
|
||||
return &BackupScheduleConfig{}, nil
|
||||
if err != nil && !errors.Is(err, ErrSettingNotFound) {
|
||||
return nil, fmt.Errorf("load backup schedule: %w", err)
|
||||
}
|
||||
if raw == "" {
|
||||
return &BackupScheduleConfig{RetainDays: 14, RetainCount: 10}, nil
|
||||
}
|
||||
var cfg BackupScheduleConfig
|
||||
if err := json.Unmarshal([]byte(raw), &cfg); err != nil {
|
||||
return &BackupScheduleConfig{}, nil
|
||||
return nil, infraerrors.InternalServer("BACKUP_SCHEDULE_CORRUPT", "backup schedule data is corrupted")
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func (s *BackupService) UpdateSchedule(ctx context.Context, cfg BackupScheduleConfig) (*BackupScheduleConfig, error) {
|
||||
if err := validateBackupRetention(&cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if cfg.Enabled && cfg.CronExpr == "" {
|
||||
return nil, infraerrors.BadRequest("INVALID_CRON", "cron expression is required when schedule is enabled")
|
||||
}
|
||||
@@ -514,14 +555,18 @@ func (s *BackupService) runScheduledBackup() {
|
||||
defer release()
|
||||
|
||||
// 读取定时备份配置中的过期天数
|
||||
schedule, _ := s.GetSchedule(ctx)
|
||||
schedule, err := s.GetSchedule(ctx)
|
||||
if err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 加载备份配置失败: %v", err)
|
||||
return
|
||||
}
|
||||
expireDays := 14 // 默认14天过期
|
||||
if schedule != nil && schedule.RetainDays > 0 {
|
||||
if schedule != nil {
|
||||
expireDays = schedule.RetainDays
|
||||
}
|
||||
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 开始执行定时备份, 过期天数: %d", expireDays)
|
||||
record, err := s.CreateBackup(ctx, "scheduled", expireDays)
|
||||
record, err := s.createBackup(ctx, "scheduled", expireDays, schedule)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrBackupInProgress) {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 定时备份跳过: 已有备份正在进行中")
|
||||
@@ -546,6 +591,10 @@ func (s *BackupService) runScheduledBackup() {
|
||||
// CreateBackup 创建全量数据库备份并上传到 S3。
|
||||
// expireDays: 备份过期天数,0=永不过期,默认14天
|
||||
func (s *BackupService) CreateBackup(ctx context.Context, triggeredBy string, expireDays int) (*BackupRecord, error) {
|
||||
return s.createBackup(ctx, triggeredBy, expireDays, nil)
|
||||
}
|
||||
|
||||
func (s *BackupService) createBackup(ctx context.Context, triggeredBy string, expireDays int, schedule *BackupScheduleConfig) (*BackupRecord, error) {
|
||||
if s.shuttingDown.Load() {
|
||||
return nil, infraerrors.ServiceUnavailable("SERVER_SHUTTING_DOWN", "server is shutting down")
|
||||
}
|
||||
@@ -578,7 +627,7 @@ func (s *BackupService) CreateBackup(ctx context.Context, triggeredBy string, ex
|
||||
|
||||
now := time.Now()
|
||||
backupID := uuid.New().String()[:8]
|
||||
fileName := fmt.Sprintf("%s_%s.sql.gz", s.dbCfg.DBName, now.Format("20060102_150405"))
|
||||
fileName := fmt.Sprintf("%s_%s_%s.sql.gz", s.dbCfg.DBName, now.Format("20060102_150405"), backupID)
|
||||
s3Key := s.buildS3Key(s3Cfg, fileName)
|
||||
|
||||
var expiresAt string
|
||||
@@ -620,8 +669,8 @@ func (s *BackupService) CreateBackup(ctx context.Context, triggeredBy string, ex
|
||||
|
||||
record.Status = "completed"
|
||||
record.FinishedAt = time.Now().Format(time.RFC3339)
|
||||
if err := s.saveRecord(ctx, record); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存备份记录失败: %v", err)
|
||||
if err := s.saveRecordWithArchive(ctx, record, schedule); err != nil {
|
||||
return record, fmt.Errorf("save completed backup: %w", err)
|
||||
}
|
||||
|
||||
return record, nil
|
||||
@@ -667,7 +716,7 @@ func (s *BackupService) StartBackup(ctx context.Context, triggeredBy string, exp
|
||||
|
||||
now := time.Now()
|
||||
backupID := uuid.New().String()[:8]
|
||||
fileName := fmt.Sprintf("%s_%s.sql.gz", s.dbCfg.DBName, now.Format("20060102_150405"))
|
||||
fileName := fmt.Sprintf("%s_%s_%s.sql.gz", s.dbCfg.DBName, now.Format("20060102_150405"), backupID)
|
||||
s3Key := s.buildS3Key(s3Cfg, fileName)
|
||||
|
||||
var expiresAt string
|
||||
@@ -966,8 +1015,12 @@ func (s *BackupService) StartRestore(ctx context.Context, backupID string) (*Bac
|
||||
return nil, fmt.Errorf("init object store: %w", err)
|
||||
}
|
||||
|
||||
record.RestoreStatus = "running"
|
||||
_ = s.saveRecord(ctx, record)
|
||||
// Re-read and reserve under the same lock used by cleanup/deletion. A prior
|
||||
// read above may have become stale while S3 configuration was being loaded.
|
||||
record, err = s.beginBackupRestore(ctx, backupID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
launched = true
|
||||
result := *record
|
||||
@@ -985,7 +1038,7 @@ func (s *BackupService) StartRestore(ctx context.Context, backupID string) (*Bac
|
||||
logger.LegacyPrintf("service.backup", "[Backup] restore panic recovered: %v", r)
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = fmt.Sprintf("internal panic: %v", r)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
_ = s.saveRestoreRecord(context.Background(), record)
|
||||
}
|
||||
}()
|
||||
s.executeRestore(record, objectStore)
|
||||
@@ -1004,19 +1057,19 @@ func (s *BackupService) executeRestore(record *BackupRecord, objectStore BackupO
|
||||
if err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = err.Error()
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
_ = s.saveRestoreRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
defer func() { _ = cleanupBackupFiles(archivePath) }()
|
||||
if err := s.restoreArchive(ctx, archivePath); err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = fmt.Sprintf("pg restore: %v", err)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
_ = s.saveRestoreRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
record.RestoreStatus = "completed"
|
||||
record.RestoredAt = time.Now().Format(time.RFC3339)
|
||||
if err := s.saveRecord(context.Background(), record); err != nil {
|
||||
if err := s.saveRestoreRecord(context.Background(), record); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存恢复记录失败: %v", err)
|
||||
}
|
||||
return
|
||||
@@ -1026,7 +1079,7 @@ func (s *BackupService) executeRestore(record *BackupRecord, objectStore BackupO
|
||||
if err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = fmt.Sprintf("S3 download failed: %v", err)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
_ = s.saveRestoreRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
defer func() { _ = body.Close() }()
|
||||
@@ -1035,7 +1088,7 @@ func (s *BackupService) executeRestore(record *BackupRecord, objectStore BackupO
|
||||
if err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = fmt.Sprintf("gzip reader: %v", err)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
_ = s.saveRestoreRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
defer func() { _ = gzReader.Close() }()
|
||||
@@ -1043,13 +1096,13 @@ func (s *BackupService) executeRestore(record *BackupRecord, objectStore BackupO
|
||||
if err := s.dumper.Restore(ctx, gzReader); err != nil {
|
||||
record.RestoreStatus = "failed"
|
||||
record.RestoreError = fmt.Sprintf("pg restore: %v", err)
|
||||
_ = s.saveRecord(context.Background(), record)
|
||||
_ = s.saveRestoreRecord(context.Background(), record)
|
||||
return
|
||||
}
|
||||
|
||||
record.RestoreStatus = "completed"
|
||||
record.RestoredAt = time.Now().Format(time.RFC3339)
|
||||
if err := s.saveRecord(context.Background(), record); err != nil {
|
||||
if err := s.saveRestoreRecord(context.Background(), record); err != nil {
|
||||
logger.LegacyPrintf("service.backup", "[Backup] 保存恢复记录失败: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -1136,7 +1189,7 @@ func (s *BackupService) ListBackups(ctx context.Context) ([]BackupRecord, error)
|
||||
}
|
||||
// 倒序返回(最新在前)
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].StartedAt > records[j].StartedAt
|
||||
return backupStartedAfter(records[i], records[j])
|
||||
})
|
||||
return records, nil
|
||||
}
|
||||
@@ -1155,8 +1208,20 @@ func (s *BackupService) GetBackupRecord(ctx context.Context, backupID string) (*
|
||||
}
|
||||
|
||||
func (s *BackupService) DeleteBackup(ctx context.Context, backupID string) error {
|
||||
s.recordsMu.Lock()
|
||||
defer s.recordsMu.Unlock()
|
||||
return s.deleteBackup(ctx, backupID, false)
|
||||
}
|
||||
|
||||
// DeleteArchivedBackup requires the caller to explicitly confirm archive removal.
|
||||
func (s *BackupService) DeleteArchivedBackup(ctx context.Context, backupID string) error {
|
||||
return s.deleteBackup(ctx, backupID, true)
|
||||
}
|
||||
|
||||
func (s *BackupService) deleteBackup(ctx context.Context, backupID string, deleteArchived bool) error {
|
||||
ctx, release, err := s.lockBackupRecordUpdates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
records, err := s.loadRecordsLocked(ctx)
|
||||
if err != nil {
|
||||
@@ -1179,6 +1244,12 @@ func (s *BackupService) DeleteBackup(ctx context.Context, backupID string) error
|
||||
// 后台上传仍可能依赖 Parts 计划;删除对象会让随后完成的记录引用失效卷。
|
||||
return ErrBackupInProgress
|
||||
}
|
||||
if found.MonthlyArchive != nil && !deleteArchived {
|
||||
return ErrBackupArchiveProtected
|
||||
}
|
||||
if found.RestoreStatus == "running" {
|
||||
return ErrRestoreInProgress
|
||||
}
|
||||
|
||||
// 从对象存储删除所有单文件或分卷对象。删除不完整时保留记录,便于重试。
|
||||
if err := s.deleteBackupObjects(ctx, found); err != nil {
|
||||
@@ -1301,7 +1372,10 @@ func (s *BackupService) loadRecords(ctx context.Context) ([]BackupRecord, error)
|
||||
// loadRecordsLocked 在已持有 recordsMu 锁的情况下加载记录
|
||||
func (s *BackupService) loadRecordsLocked(ctx context.Context) ([]BackupRecord, error) {
|
||||
raw, err := s.settingRepo.GetValue(ctx, settingKeyBackupRecords)
|
||||
if err != nil || raw == "" {
|
||||
if err != nil && !errors.Is(err, ErrSettingNotFound) {
|
||||
return nil, fmt.Errorf("load backup records: %w", err)
|
||||
}
|
||||
if raw == "" {
|
||||
return nil, nil //nolint:nilnil // no records is a valid state
|
||||
}
|
||||
var records []BackupRecord
|
||||
@@ -1322,27 +1396,64 @@ func (s *BackupService) saveRecordsLocked(ctx context.Context, records []BackupR
|
||||
|
||||
// saveRecord 保存单条记录(带互斥锁保护)
|
||||
func (s *BackupService) saveRecord(ctx context.Context, record *BackupRecord) error {
|
||||
s.recordsMu.Lock()
|
||||
defer s.recordsMu.Unlock()
|
||||
return s.saveRecordWithArchive(ctx, record, nil)
|
||||
}
|
||||
|
||||
records, _ := s.loadRecordsLocked(ctx)
|
||||
func (s *BackupService) saveRecordWithArchive(ctx context.Context, record *BackupRecord, schedule *BackupScheduleConfig) error {
|
||||
ctx, release, err := s.lockBackupRecordUpdates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
records, err := s.loadRecordsLocked(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
snapshot := *record
|
||||
for _, existing := range records {
|
||||
if existing.ID == record.ID && existing.MonthlyArchive != nil {
|
||||
// Preserve durable protection when a progress/restore writer or retry
|
||||
// holds an older record snapshot.
|
||||
snapshot.MonthlyArchive = existing.MonthlyArchive
|
||||
snapshot.ExpiresAt = ""
|
||||
break
|
||||
}
|
||||
}
|
||||
checkpoint, err := s.assignMonthlyArchive(ctx, &snapshot, schedule)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 更新已有记录或追加
|
||||
found := false
|
||||
for i := range records {
|
||||
if records[i].ID == record.ID {
|
||||
records[i] = *record
|
||||
records[i] = snapshot
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
records = append(records, *record)
|
||||
records = append(records, snapshot)
|
||||
}
|
||||
|
||||
// 限制记录数量
|
||||
if len(records) > maxBackupRecords {
|
||||
records = records[len(records)-maxBackupRecords:]
|
||||
// Never discard metadata for retained objects, including permanent archives.
|
||||
// The checkpoint and completed record must commit together so retries cannot
|
||||
// consume a date without an archive, or archive the same date repeatedly.
|
||||
if checkpoint != "" {
|
||||
data, err := json.Marshal(records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.settingRepo.SetMultiple(ctx, map[string]string{
|
||||
settingKeyBackupRecords: string(data),
|
||||
settingKeyBackupArchiveCheckpoint: checkpoint,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
*record = snapshot
|
||||
return nil
|
||||
}
|
||||
|
||||
return s.saveRecordsLocked(ctx, records)
|
||||
@@ -1353,8 +1464,11 @@ func (s *BackupService) cleanupOldBackups(ctx context.Context, schedule *BackupS
|
||||
return nil
|
||||
}
|
||||
|
||||
s.recordsMu.Lock()
|
||||
defer s.recordsMu.Unlock()
|
||||
ctx, release, err := s.lockBackupRecordUpdates(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer release()
|
||||
|
||||
records, err := s.loadRecordsLocked(ctx)
|
||||
if err != nil {
|
||||
@@ -1363,29 +1477,49 @@ func (s *BackupService) cleanupOldBackups(ctx context.Context, schedule *BackupS
|
||||
|
||||
// 按时间倒序
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].StartedAt > records[j].StartedAt
|
||||
return backupStartedAfter(records[i], records[j])
|
||||
})
|
||||
|
||||
var toDelete []BackupRecord
|
||||
var toKeep []BackupRecord
|
||||
|
||||
for i, r := range records {
|
||||
ordinaryCount, archiveCount := 0, 0
|
||||
metadataChanged := false
|
||||
for _, r := range records {
|
||||
shouldDelete := false
|
||||
|
||||
// 按保留份数清理
|
||||
if schedule.RetainCount > 0 && i >= schedule.RetainCount {
|
||||
shouldDelete = true
|
||||
if r.Status != "completed" {
|
||||
toKeep = append(toKeep, r)
|
||||
continue
|
||||
}
|
||||
if r.MonthlyArchive != nil {
|
||||
if r.MonthlyArchive.RetainCount > 0 {
|
||||
archiveCount++
|
||||
// A new finite limit applies to the finite archive pool. Permanent
|
||||
// archives remain protected even after the configuration changes.
|
||||
if cfg := schedule.MonthlyArchive; cfg != nil && cfg.RetainCount > 0 && cfg.RetainCount != r.MonthlyArchive.RetainCount {
|
||||
r.MonthlyArchive.RetainCount = cfg.RetainCount
|
||||
metadataChanged = true
|
||||
}
|
||||
shouldDelete = archiveCount > r.MonthlyArchive.RetainCount
|
||||
}
|
||||
} else {
|
||||
ordinaryCount++
|
||||
|
||||
// 按保留天数清理
|
||||
if schedule.RetainDays > 0 && r.StartedAt != "" {
|
||||
startedAt, err := time.Parse(time.RFC3339, r.StartedAt)
|
||||
if err == nil && time.Since(startedAt) > time.Duration(schedule.RetainDays)*24*time.Hour {
|
||||
// 按保留份数清理
|
||||
if schedule.RetainCount > 0 && ordinaryCount > schedule.RetainCount {
|
||||
shouldDelete = true
|
||||
}
|
||||
|
||||
// 按保留天数清理
|
||||
if schedule.RetainDays > 0 && r.StartedAt != "" {
|
||||
startedAt, err := time.Parse(time.RFC3339, r.StartedAt)
|
||||
if err == nil && time.Now().After(startedAt.AddDate(0, 0, schedule.RetainDays)) {
|
||||
shouldDelete = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if shouldDelete && r.Status == "completed" {
|
||||
if shouldDelete && r.RestoreStatus != "running" {
|
||||
toDelete = append(toDelete, r)
|
||||
} else {
|
||||
toKeep = append(toKeep, r)
|
||||
@@ -1404,7 +1538,7 @@ func (s *BackupService) cleanupOldBackups(ctx context.Context, schedule *BackupS
|
||||
deletedCount++
|
||||
}
|
||||
|
||||
if len(toDelete) > 0 {
|
||||
if len(toDelete) > 0 || metadataChanged {
|
||||
if err := s.saveRecordsLocked(ctx, toKeep); err != nil {
|
||||
cleanupErrs = append(cleanupErrs, fmt.Errorf("save backup records after cleanup: %w", err))
|
||||
}
|
||||
|
||||
@@ -1146,10 +1146,11 @@ func TestRecoverStaleRecords(t *testing.T) {
|
||||
})
|
||||
// 模拟一条孤立的恢复中记录
|
||||
_ = svc.saveRecord(context.Background(), &BackupRecord{
|
||||
ID: "stale-2",
|
||||
Status: "completed",
|
||||
RestoreStatus: "running",
|
||||
StartedAt: time.Now().Add(-1 * time.Hour).Format(time.RFC3339),
|
||||
ID: "stale-2",
|
||||
Status: "completed",
|
||||
RestoreStatus: "running",
|
||||
RestoreStartedAt: time.Now().Add(-1 * time.Hour).Format(time.RFC3339),
|
||||
StartedAt: time.Now().Add(-1 * time.Hour).Format(time.RFC3339),
|
||||
})
|
||||
|
||||
svc.recoverStaleRecords()
|
||||
|
||||
@@ -15,6 +15,19 @@ export interface BackupScheduleConfig {
|
||||
cron_expr: string
|
||||
retain_days: number
|
||||
retain_count: number
|
||||
monthly_archive?: BackupMonthlyArchiveConfig
|
||||
}
|
||||
|
||||
export interface BackupMonthlyArchiveConfig {
|
||||
enabled: boolean
|
||||
days: number[]
|
||||
include_month_end: boolean
|
||||
retain_count: number // 0 = permanent
|
||||
}
|
||||
|
||||
export interface BackupMonthlyArchive {
|
||||
dates: string[]
|
||||
retain_count: number
|
||||
}
|
||||
|
||||
export interface BackupRecord {
|
||||
@@ -32,8 +45,10 @@ export interface BackupRecord {
|
||||
expires_at?: string
|
||||
progress?: string
|
||||
restore_status?: string
|
||||
restore_started_at?: string
|
||||
restore_error?: string
|
||||
restored_at?: string
|
||||
monthly_archive?: BackupMonthlyArchive
|
||||
}
|
||||
|
||||
export interface BackupPart {
|
||||
@@ -152,8 +167,8 @@ export async function getBackup(id: string): Promise<BackupRecord> {
|
||||
return data
|
||||
}
|
||||
|
||||
export async function deleteBackup(id: string): Promise<void> {
|
||||
await apiClient.delete(`/admin/backups/${id}`)
|
||||
export async function deleteBackup(id: string, deleteArchived = false): Promise<void> {
|
||||
await apiClient.delete(`/admin/backups/${id}`, deleteArchived ? { params: { delete_archived: true } } : undefined)
|
||||
}
|
||||
|
||||
export async function getDownloadURL(id: string): Promise<BackupDownloadResponse> {
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
<template>
|
||||
<section class="mt-5 border-t border-gray-200 pt-5 dark:border-dark-600" aria-labelledby="backup-archive-title">
|
||||
<div class="mb-3 flex items-center justify-between gap-3">
|
||||
<h4 id="backup-archive-title" class="text-sm font-medium text-gray-900 dark:text-white">{{ t('admin.backup.archive.title') }}</h4>
|
||||
<label class="inline-flex items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
|
||||
<input data-testid="archive-enabled" type="checkbox" :checked="modelValue.enabled" @change="update({ enabled: ($event.target as HTMLInputElement).checked })" />
|
||||
{{ t('admin.backup.archive.enabled') }}
|
||||
</label>
|
||||
</div>
|
||||
<div v-if="modelValue.enabled">
|
||||
<div class="grid grid-cols-1 items-start gap-3 md:grid-cols-2">
|
||||
<div>
|
||||
<label for="backup-archive-dates" class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">{{ t('admin.backup.archive.dates') }}</label>
|
||||
<div ref="datePicker" class="relative" @keydown.esc.stop.prevent="closeDates(true)" @focusout="handleFocusOut">
|
||||
<button
|
||||
id="backup-archive-dates" ref="dateTrigger" type="button" class="input flex w-full items-center justify-between gap-2 text-left"
|
||||
:aria-expanded="datesOpen" aria-controls="backup-archive-options" aria-describedby="backup-archive-dates-hint"
|
||||
@click="datesOpen = !datesOpen" @keydown.down.prevent="openDatesWithKeyboard"
|
||||
>
|
||||
<span class="flex min-w-0 flex-wrap items-center gap-1">
|
||||
<span v-if="!selectedLabels.length" class="text-gray-400">{{ t('admin.backup.archive.selectDates') }}</span>
|
||||
<span v-for="label in selectedLabels.slice(0, 3)" :key="label" class="rounded bg-primary-50 px-1.5 py-0.5 text-xs text-primary-700 dark:bg-primary-900/30 dark:text-primary-300">{{ label }}</span>
|
||||
<span v-if="selectedLabels.length > 3" class="text-xs">+{{ selectedLabels.length - 3 }}</span>
|
||||
</span>
|
||||
<Icon name="chevronDown" size="sm" class="shrink-0" />
|
||||
</button>
|
||||
<div v-if="datesOpen" id="backup-archive-options" class="absolute left-0 top-full z-30 mt-1 w-full rounded-xl border border-gray-200 bg-white p-2 shadow-lg dark:border-dark-600 dark:bg-dark-800">
|
||||
<div class="mb-1 flex items-center justify-between gap-2 px-1 text-xs text-gray-500 dark:text-gray-400">
|
||||
<span>{{ t('admin.backup.archive.selectedDates', { count: selectedLabels.length }) }}</span>
|
||||
<button type="button" class="min-h-9 px-2 text-primary-600 dark:text-primary-400" @click="closeDates(true)">{{ t('admin.backup.archive.done') }}</button>
|
||||
</div>
|
||||
<div class="grid grid-cols-4 gap-1" role="group" :aria-label="t('admin.backup.archive.dates')">
|
||||
<label v-for="day in 31" :key="day" class="flex min-h-10 cursor-pointer items-center gap-1.5 rounded px-1 text-xs hover:bg-gray-100 dark:hover:bg-dark-700">
|
||||
<input type="checkbox" :value="day" :checked="modelValue.days.includes(day)" :aria-label="t('admin.backup.archive.day', { day })" @change="toggleDay(day)" />
|
||||
{{ day }}
|
||||
</label>
|
||||
<label class="flex min-h-10 cursor-pointer items-center gap-1.5 rounded px-1 text-xs hover:bg-gray-100 dark:hover:bg-dark-700">
|
||||
<input type="checkbox" value="last" :checked="modelValue.include_month_end" :aria-label="t('admin.backup.archive.monthEnd')" @change="update({ include_month_end: ($event.target as HTMLInputElement).checked })" />
|
||||
{{ t('admin.backup.archive.monthEnd') }}
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<p id="backup-archive-dates-hint" class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.archive.datesHint') }}</p>
|
||||
</div>
|
||||
<div>
|
||||
<span class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">{{ t('admin.backup.archive.retention') }}</span>
|
||||
<div class="flex min-h-10 flex-wrap items-center gap-3">
|
||||
<label v-if="!forever" class="flex min-w-0 flex-1 items-center gap-2">
|
||||
<input data-testid="archive-count" :value="modelValue.retain_count" type="number" min="1" step="1" class="input w-full min-w-0" :aria-label="t('admin.backup.archive.count')" aria-describedby="backup-archive-retention-hint" @input="setCount" />
|
||||
<span class="shrink-0 text-sm">{{ t('admin.backup.archive.copies') }}</span>
|
||||
</label>
|
||||
<label class="inline-flex min-h-10 shrink-0 items-center gap-2 text-sm text-gray-700 dark:text-gray-300">
|
||||
<input data-testid="archive-forever" type="checkbox" :checked="forever" @change="toggleForever" />
|
||||
{{ t('admin.backup.archive.forever') }}
|
||||
</label>
|
||||
</div>
|
||||
<p id="backup-archive-retention-hint" class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t(forever ? 'admin.backup.archive.foreverHint' : 'admin.backup.archive.countHint') }}</p>
|
||||
</div>
|
||||
</div>
|
||||
<p class="mt-3 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.archive.fallbackHint') }}</p>
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.archive.independentHint') }}</p>
|
||||
</div>
|
||||
<p v-else class="text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.archive.disabledHint') }}</p>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, nextTick, onBeforeUnmount, onMounted, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import Icon from '@/components/icons/Icon.vue'
|
||||
import type { BackupMonthlyArchiveConfig } from '@/api/admin/backup'
|
||||
|
||||
const props = defineProps<{ modelValue: BackupMonthlyArchiveConfig }>()
|
||||
const emit = defineEmits<{ 'update:modelValue': [value: BackupMonthlyArchiveConfig] }>()
|
||||
const { t } = useI18n()
|
||||
const datesOpen = ref(false)
|
||||
const datePicker = ref<HTMLElement | null>(null)
|
||||
const dateTrigger = ref<HTMLButtonElement | null>(null)
|
||||
const lastFiniteCount = ref(12)
|
||||
const forever = computed(() => props.modelValue.retain_count === 0)
|
||||
const selectedLabels = computed(() => [
|
||||
...[...props.modelValue.days].sort((a, b) => a - b).map(day => t('admin.backup.archive.day', { day })),
|
||||
...(props.modelValue.include_month_end ? [t('admin.backup.archive.monthEnd')] : []),
|
||||
])
|
||||
|
||||
watch(() => props.modelValue.retain_count, value => {
|
||||
if (Number.isInteger(value) && value > 0) lastFiniteCount.value = value
|
||||
}, { immediate: true })
|
||||
watch(() => props.modelValue.enabled, enabled => { if (!enabled) datesOpen.value = false })
|
||||
|
||||
function update(patch: Partial<BackupMonthlyArchiveConfig>) {
|
||||
emit('update:modelValue', { ...props.modelValue, ...patch })
|
||||
}
|
||||
function toggleDay(day: number) {
|
||||
const days = props.modelValue.days.includes(day) ? props.modelValue.days.filter(value => value !== day) : [...props.modelValue.days, day]
|
||||
update({ days: days.sort((a, b) => a - b) })
|
||||
}
|
||||
function setCount(event: Event) {
|
||||
const value = (event.target as HTMLInputElement).valueAsNumber
|
||||
// Zero is the wire representation of permanent retention, selected only by
|
||||
// the checkbox. Typing zero must remain an invalid finite count.
|
||||
update({ retain_count: value === 0 ? Number.NaN : value })
|
||||
}
|
||||
function toggleForever(event: Event) {
|
||||
update({ retain_count: (event.target as HTMLInputElement).checked ? 0 : lastFiniteCount.value })
|
||||
}
|
||||
function closeDates(focus = false) {
|
||||
datesOpen.value = false
|
||||
if (focus) dateTrigger.value?.focus()
|
||||
}
|
||||
async function openDatesWithKeyboard() {
|
||||
datesOpen.value = true
|
||||
await nextTick()
|
||||
datePicker.value?.querySelector<HTMLInputElement>('input')?.focus()
|
||||
}
|
||||
function handleFocusOut(event: FocusEvent) {
|
||||
if (event.relatedTarget instanceof Node && !datePicker.value?.contains(event.relatedTarget)) closeDates()
|
||||
}
|
||||
function handleOutsideClick(event: MouseEvent) {
|
||||
if (event.target instanceof Node && !datePicker.value?.contains(event.target)) closeDates()
|
||||
}
|
||||
onMounted(() => document.addEventListener('click', handleOutsideClick))
|
||||
onBeforeUnmount(() => document.removeEventListener('click', handleOutsideClick))
|
||||
</script>
|
||||
@@ -122,11 +122,43 @@ export default {
|
||||
cronExpr: 'Cron Expression',
|
||||
cronHint: 'e.g. "0 2 * * *" means every day at 2:00 AM',
|
||||
retainDays: 'Backup Expire Days',
|
||||
retainDaysHint: 'Backup files auto-delete after this many days, 0 = never expire',
|
||||
retainDaysHint: 'Ordinary backups are deleted after this many days; 0 = no age limit',
|
||||
retainCount: 'Max Retain Count',
|
||||
retainCountHint: 'Maximum number of backups to keep, 0 = unlimited',
|
||||
retainCountHint: 'Maximum ordinary backups to keep; 0 = no count limit',
|
||||
ordinaryRetention: 'Ordinary backup retention',
|
||||
ordinaryHint: 'Clean up the oldest ordinary backups when either limit is reached. Monthly archives are retained separately.',
|
||||
preview: 'Retention preview',
|
||||
previewBoth: 'Keep up to {count} ordinary backups from the last {days} days.',
|
||||
previewDays: 'Keep ordinary backups from the last {days} days, with no count limit.',
|
||||
previewCount: 'Keep the latest {count} ordinary backups, with no age limit.',
|
||||
previewUnlimited: 'Ordinary backups are not automatically deleted.',
|
||||
saved: 'Schedule configuration saved'
|
||||
},
|
||||
archive: {
|
||||
title: 'Monthly archives',
|
||||
enabled: 'Enable',
|
||||
dates: 'Archive dates (select multiple)',
|
||||
selectDates: 'Select at least one archive date',
|
||||
selectedDates: '{count} dates selected',
|
||||
day: 'Day {day}',
|
||||
monthEnd: 'Month end',
|
||||
done: 'Done',
|
||||
datesHint: 'Archive the first successful scheduled backup for each selected date, using its start date and the schedule time zone.',
|
||||
retention: 'Archive retention',
|
||||
count: 'Number of archives to keep',
|
||||
copies: 'copies',
|
||||
forever: 'Keep forever',
|
||||
foreverHint: 'Keep all new archives permanently. Existing permanent archives remain protected after settings change.',
|
||||
countHint: 'Count non-permanent archives across all selected dates together; delete the oldest when the limit is exceeded.',
|
||||
fallbackHint: 'If no backup succeeds on a selected date, use the next successful backup in that month. Missing dates use month end. Count each backup only once.',
|
||||
independentHint: 'Archives reuse scheduled backups and do not count toward ordinary retention. No archive is created if the month has no later successful backup.',
|
||||
disabledHint: 'Disabling stops new archives. Existing archives keep their previous retention policy.',
|
||||
invalidRetention: 'Retention days and counts must be non-negative integers. Non-permanent archives require at least 1 copy.',
|
||||
preview: 'Archive one backup on {dates} each month: {retention}.',
|
||||
retainLatest: 'keep the latest {count} archives in total',
|
||||
badge: 'Monthly archive',
|
||||
deleteConfirm: 'This is a monthly archive. Remove archive protection and permanently delete this backup? This cannot be undone.',
|
||||
},
|
||||
operations: {
|
||||
title: 'Backup Records',
|
||||
description: 'Create manual backups and manage existing backup records',
|
||||
|
||||
@@ -122,11 +122,43 @@ export default {
|
||||
cronExpr: 'Cron 表达式',
|
||||
cronHint: '例如 "0 2 * * *" 表示每天凌晨 2 点',
|
||||
retainDays: '备份过期天数',
|
||||
retainDaysHint: '备份文件超过此天数后自动删除,0 = 永不过期',
|
||||
retainDaysHint: '普通备份超过此天数后自动删除,0 = 不按天数清理',
|
||||
retainCount: '最大保留份数',
|
||||
retainCountHint: '最多保留的备份数量,0 = 不限制',
|
||||
retainCountHint: '最多保留的普通备份数量,0 = 不按份数清理',
|
||||
ordinaryRetention: '普通备份保留',
|
||||
ordinaryHint: '达到任一清理条件时,清理最旧的普通备份;月度归档独立保留。',
|
||||
preview: '保留效果',
|
||||
previewBoth: '最近 {days} 天内最多保留 {count} 份普通备份。',
|
||||
previewDays: '保留最近 {days} 天的普通备份,份数不限。',
|
||||
previewCount: '保留最新 {count} 份普通备份,天数不限。',
|
||||
previewUnlimited: '普通备份不自动清理。',
|
||||
saved: '定时备份配置已保存'
|
||||
},
|
||||
archive: {
|
||||
title: '月度归档',
|
||||
enabled: '启用',
|
||||
dates: '归档日期(可多选)',
|
||||
selectDates: '请至少选择一个归档日期',
|
||||
selectedDates: '已选择 {count} 个日期',
|
||||
day: '{day} 日',
|
||||
monthEnd: '月末',
|
||||
done: '完成',
|
||||
datesHint: '按备份计划的时区与备份开始日期匹配,每个日期归档首份成功的定时备份。',
|
||||
retention: '归档保留方式',
|
||||
count: '归档保留数量',
|
||||
copies: '份',
|
||||
forever: '永久保留',
|
||||
foreverHint: '所有新归档均永久保留;已有永久归档不会因配置变更被自动删除。',
|
||||
countHint: '所有选定日期的非永久归档合计计数,超出时清理最旧归档。',
|
||||
fallbackHint: '当日无成功备份时,顺延至本月下次成功备份;当月没有所选日期时按月末处理,同一份备份只计一份。',
|
||||
independentHint: '归档不额外执行备份,不占普通备份名额;本月无后续成功备份则不归档。',
|
||||
disabledHint: '关闭后停止产生新归档,已有归档继续按原保留方式管理。',
|
||||
invalidRetention: '保留天数和份数须为非负整数;非永久归档须填写至少 1 份。',
|
||||
preview: '每月 {dates} 各归档一份,{retention}。',
|
||||
retainLatest: '合计保留最近 {count} 份',
|
||||
badge: '月度归档',
|
||||
deleteConfirm: '这是月度归档备份。确定要解除归档保护并永久删除此备份吗?此操作无法撤销。',
|
||||
},
|
||||
operations: {
|
||||
title: '备份记录',
|
||||
description: '创建手动备份和管理已有备份记录',
|
||||
|
||||
@@ -130,7 +130,7 @@
|
||||
</div>
|
||||
|
||||
<!-- Schedule Config -->
|
||||
<div class="card p-6">
|
||||
<div class="card p-6" data-testid="backup-schedule">
|
||||
<div class="mb-4">
|
||||
<h3 class="text-base font-semibold text-gray-900 dark:text-white">
|
||||
{{ t('admin.backup.schedule.title') }}
|
||||
@@ -149,19 +149,27 @@
|
||||
<input v-model="scheduleForm.cron_expr" class="input w-full" placeholder="0 2 * * *" />
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.schedule.cronHint') }}</p>
|
||||
</div>
|
||||
<h4 class="mt-2 text-sm font-medium text-gray-900 dark:text-white md:col-span-2">{{ t('admin.backup.schedule.ordinaryRetention') }}</h4>
|
||||
<div>
|
||||
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">{{ t('admin.backup.schedule.retainDays') }}</label>
|
||||
<input v-model.number="scheduleForm.retain_days" type="number" min="0" class="input w-full" />
|
||||
<input v-model.number="scheduleForm.retain_days" data-testid="backup-retain-days" type="number" min="0" step="1" class="input w-full" />
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.schedule.retainDaysHint') }}</p>
|
||||
</div>
|
||||
<div>
|
||||
<label class="mb-1 block text-xs font-medium text-gray-600 dark:text-gray-400">{{ t('admin.backup.schedule.retainCount') }}</label>
|
||||
<input v-model.number="scheduleForm.retain_count" type="number" min="0" class="input w-full" />
|
||||
<input v-model.number="scheduleForm.retain_count" data-testid="backup-retain-count" type="number" min="0" step="1" class="input w-full" />
|
||||
<p class="mt-1 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.schedule.retainCountHint') }}</p>
|
||||
</div>
|
||||
</div>
|
||||
<p class="mt-2 text-xs text-gray-500 dark:text-gray-400">{{ t('admin.backup.schedule.ordinaryHint') }}</p>
|
||||
<BackupArchiveSettings v-model="archiveForm" />
|
||||
<p v-if="scheduleValidationError" class="mt-3 text-sm text-red-600 dark:text-red-400" role="alert">{{ scheduleValidationError }}</p>
|
||||
<div class="mt-4 rounded-lg bg-gray-50 p-3 text-sm text-gray-700 dark:bg-dark-800 dark:text-gray-300" aria-live="polite">
|
||||
<p class="font-medium">{{ t('admin.backup.schedule.preview') }}</p>
|
||||
<p>{{ retentionPreview }}</p>
|
||||
</div>
|
||||
<div class="mt-4">
|
||||
<button type="button" class="btn btn-primary btn-sm" :disabled="savingSchedule" @click="saveSchedule">
|
||||
<button type="button" class="btn btn-primary btn-sm" :disabled="savingSchedule || !!scheduleValidationError" @click="saveSchedule">
|
||||
{{ savingSchedule ? t('common.loading') : t('common.save') }}
|
||||
</button>
|
||||
</div>
|
||||
@@ -220,11 +228,17 @@
|
||||
: t(`admin.backup.status.${record.status}`) }}
|
||||
</span>
|
||||
</td>
|
||||
<td class="py-3 pr-4 text-xs">{{ record.file_name }}</td>
|
||||
<td class="py-3 pr-4 text-xs">
|
||||
{{ record.file_name }}
|
||||
<span v-if="record.monthly_archive" class="ml-1 inline-block rounded bg-primary-50 px-1.5 py-0.5 text-primary-700 dark:bg-primary-900/30 dark:text-primary-300">{{ t('admin.backup.archive.badge') }}</span>
|
||||
<div v-if="record.monthly_archive" class="mt-1 text-gray-500 dark:text-gray-400">{{ record.monthly_archive.dates.join(' / ') }}</div>
|
||||
</td>
|
||||
<td class="py-3 pr-4 text-xs">{{ formatSize(record.size_bytes) }}</td>
|
||||
<td class="py-3 pr-4 text-xs">{{ record.parts?.length || (record.status === 'running' ? '-' : 1) }}</td>
|
||||
<td class="py-3 pr-4 text-xs">
|
||||
{{ record.expires_at ? formatDate(record.expires_at) : t('admin.backup.neverExpire') }}
|
||||
{{ record.monthly_archive
|
||||
? record.monthly_archive.retain_count === 0 ? t('admin.backup.archive.forever') : t('admin.backup.archive.retainLatest', { count: record.monthly_archive.retain_count })
|
||||
: record.expires_at ? formatDate(record.expires_at) : t('admin.backup.neverExpire') }}
|
||||
</td>
|
||||
<td class="py-3 pr-4 text-xs">
|
||||
{{ record.triggered_by === 'scheduled' ? t('admin.backup.trigger.scheduled') : t('admin.backup.trigger.manual') }}
|
||||
@@ -407,12 +421,14 @@ import { useAppStore } from '@/stores'
|
||||
import type {
|
||||
BackupS3Config,
|
||||
BackupScheduleConfig,
|
||||
BackupMonthlyArchiveConfig,
|
||||
BackupRecord,
|
||||
BackupDownloadPart,
|
||||
ImageStorageConfig,
|
||||
} from '@/api/admin/backup'
|
||||
import { useStepUp, isStepUpBlocked, isStepUpCancelled, stepUpBlockReason } from '@/composables/useStepUp'
|
||||
import TotpStepUpDialog from '@/components/auth/TotpStepUpDialog.vue'
|
||||
import BackupArchiveSettings from '@/components/admin/BackupArchiveSettings.vue'
|
||||
|
||||
const { t } = useI18n()
|
||||
const appStore = useAppStore()
|
||||
@@ -471,6 +487,31 @@ const scheduleForm = ref<BackupScheduleConfig>({
|
||||
retain_count: 10,
|
||||
})
|
||||
const savingSchedule = ref(false)
|
||||
const archiveForm = ref<BackupMonthlyArchiveConfig>({ enabled: false, days: [1], include_month_end: false, retain_count: 0 })
|
||||
const scheduleValidationError = computed(() => {
|
||||
if (![scheduleForm.value.retain_days, scheduleForm.value.retain_count, archiveForm.value.retain_count].every(value => Number.isSafeInteger(value) && value >= 0)) {
|
||||
return t('admin.backup.archive.invalidRetention')
|
||||
}
|
||||
if (archiveForm.value.enabled && !archiveForm.value.days.length && !archiveForm.value.include_month_end) {
|
||||
return t('admin.backup.archive.selectDates')
|
||||
}
|
||||
return ''
|
||||
})
|
||||
const retentionPreview = computed(() => {
|
||||
if (scheduleValidationError.value) return scheduleValidationError.value
|
||||
const { retain_days: days, retain_count: count } = scheduleForm.value
|
||||
const ordinary = days > 0 && count > 0
|
||||
? t('admin.backup.schedule.previewBoth', { days, count })
|
||||
: days > 0 ? t('admin.backup.schedule.previewDays', { days })
|
||||
: count > 0 ? t('admin.backup.schedule.previewCount', { count }) : t('admin.backup.schedule.previewUnlimited')
|
||||
if (!archiveForm.value.enabled) return ordinary
|
||||
const dates = [
|
||||
...[...archiveForm.value.days].sort((a, b) => a - b).map(day => t('admin.backup.archive.day', { day })),
|
||||
...(archiveForm.value.include_month_end ? [t('admin.backup.archive.monthEnd')] : []),
|
||||
].join(', ')
|
||||
const retention = archiveForm.value.retain_count === 0 ? t('admin.backup.archive.forever') : t('admin.backup.archive.retainLatest', { count: archiveForm.value.retain_count })
|
||||
return `${ordinary} ${t('admin.backup.archive.preview', { dates, retention })}`
|
||||
})
|
||||
|
||||
// Backups
|
||||
const backups = ref<BackupRecord[]>([])
|
||||
@@ -703,8 +744,14 @@ async function loadSchedule() {
|
||||
scheduleForm.value = {
|
||||
enabled: cfg.enabled,
|
||||
cron_expr: cfg.cron_expr || '0 2 * * *',
|
||||
retain_days: cfg.retain_days || 14,
|
||||
retain_count: cfg.retain_count || 10,
|
||||
retain_days: cfg.retain_days ?? 14,
|
||||
retain_count: cfg.retain_count ?? 10,
|
||||
}
|
||||
archiveForm.value = {
|
||||
enabled: cfg.monthly_archive?.enabled ?? false,
|
||||
days: cfg.monthly_archive?.days ?? [1],
|
||||
include_month_end: cfg.monthly_archive?.include_month_end ?? false,
|
||||
retain_count: cfg.monthly_archive?.retain_count ?? 0,
|
||||
}
|
||||
} catch (error) {
|
||||
appStore.showError((error as { message?: string })?.message || t('errors.networkError'))
|
||||
@@ -712,9 +759,10 @@ async function loadSchedule() {
|
||||
}
|
||||
|
||||
async function saveSchedule() {
|
||||
if (scheduleValidationError.value) return
|
||||
savingSchedule.value = true
|
||||
try {
|
||||
await adminAPI.backup.updateSchedule(scheduleForm.value)
|
||||
await adminAPI.backup.updateSchedule({ ...scheduleForm.value, monthly_archive: { ...archiveForm.value, days: [...archiveForm.value.days] } })
|
||||
appStore.showSuccess(t('admin.backup.schedule.saved'))
|
||||
} catch (error) {
|
||||
appStore.showError((error as { message?: string })?.message || t('errors.networkError'))
|
||||
@@ -812,9 +860,10 @@ async function restoreBackup(id: string) {
|
||||
}
|
||||
|
||||
async function removeBackup(id: string) {
|
||||
if (!window.confirm(t('admin.backup.actions.deleteConfirm'))) return
|
||||
const archived = !!backups.value.find(record => record.id === id)?.monthly_archive
|
||||
if (!window.confirm(t(archived ? 'admin.backup.archive.deleteConfirm' : 'admin.backup.actions.deleteConfirm'))) return
|
||||
try {
|
||||
await adminAPI.backup.deleteBackup(id)
|
||||
await adminAPI.backup.deleteBackup(id, archived)
|
||||
appStore.showSuccess(t('admin.backup.actions.deleted'))
|
||||
await loadBackups()
|
||||
} catch (error) {
|
||||
|
||||
@@ -7,12 +7,16 @@ const {
|
||||
getS3Config,
|
||||
getImageStorageConfig,
|
||||
getSchedule,
|
||||
updateSchedule,
|
||||
deleteBackup,
|
||||
listBackups,
|
||||
getDownloadURL,
|
||||
} = vi.hoisted(() => ({
|
||||
getS3Config: vi.fn(),
|
||||
getImageStorageConfig: vi.fn(),
|
||||
getSchedule: vi.fn(),
|
||||
updateSchedule: vi.fn(),
|
||||
deleteBackup: vi.fn(),
|
||||
listBackups: vi.fn(),
|
||||
getDownloadURL: vi.fn(),
|
||||
}))
|
||||
@@ -27,11 +31,11 @@ vi.mock('@/api', () => ({
|
||||
updateImageStorageConfig: vi.fn(),
|
||||
testImageStorageConnection: vi.fn(),
|
||||
getSchedule,
|
||||
updateSchedule: vi.fn(),
|
||||
updateSchedule,
|
||||
createBackup: vi.fn(),
|
||||
listBackups,
|
||||
getBackup: vi.fn(),
|
||||
deleteBackup: vi.fn(),
|
||||
deleteBackup,
|
||||
getDownloadURL,
|
||||
restoreBackup: vi.fn(),
|
||||
},
|
||||
@@ -56,7 +60,7 @@ vi.mock('@/composables/useStepUp', () => ({
|
||||
vi.mock('vue-i18n', () => ({
|
||||
useI18n: () => ({
|
||||
t: (key: string, params?: Record<string, unknown>) =>
|
||||
params?.index === undefined ? key : `${key}:${params.index}`,
|
||||
params?.index !== undefined ? `${key}:${params.index}` : params?.day !== undefined ? `${key}:${params.day}` : key,
|
||||
}),
|
||||
}))
|
||||
|
||||
@@ -72,26 +76,34 @@ const baseRecord = (id: string, parts?: unknown[]) => ({
|
||||
started_at: '2026-08-09T00:00:00Z',
|
||||
})
|
||||
|
||||
const wrappers: ReturnType<typeof mount>[] = []
|
||||
|
||||
function mountBackupView() {
|
||||
return mount(BackupView, {
|
||||
const wrapper = mount(BackupView, {
|
||||
global: {
|
||||
stubs: {
|
||||
TotpStepUpDialog: true,
|
||||
},
|
||||
},
|
||||
})
|
||||
wrappers.push(wrapper)
|
||||
return wrapper
|
||||
}
|
||||
|
||||
describe('admin BackupView 分卷备份', () => {
|
||||
describe('admin BackupView', () => {
|
||||
beforeEach(() => {
|
||||
getS3Config.mockResolvedValue({})
|
||||
getImageStorageConfig.mockResolvedValue({ config: {}, secret_configured: false })
|
||||
getSchedule.mockResolvedValue({ enabled: false, cron_expr: '', retain_days: 14, retain_count: 10 })
|
||||
updateSchedule.mockReset().mockResolvedValue({})
|
||||
deleteBackup.mockReset().mockResolvedValue(undefined)
|
||||
listBackups.mockResolvedValue({ items: [] })
|
||||
getDownloadURL.mockReset()
|
||||
vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
wrappers.splice(0).forEach(wrapper => wrapper.unmount())
|
||||
vi.restoreAllMocks()
|
||||
document.body.innerHTML = ''
|
||||
})
|
||||
@@ -151,4 +163,90 @@ describe('admin BackupView 分卷备份', () => {
|
||||
expect(wrapper.find('tbody tr td:nth-child(5)').text()).toBe('-')
|
||||
expect(wrapper.findAll('button').some(button => button.text() === 'common.delete')).toBe(false)
|
||||
})
|
||||
|
||||
it('兼容旧配置并保留 0 值,不自动启用月度归档', async () => {
|
||||
getSchedule.mockResolvedValue({ enabled: true, cron_expr: '0 4 * * *', retain_days: 0, retain_count: 0 })
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
expect((wrapper.get('[data-testid="backup-retain-days"]').element as HTMLInputElement).value).toBe('0')
|
||||
expect((wrapper.get('[data-testid="backup-retain-count"]').element as HTMLInputElement).value).toBe('0')
|
||||
expect((wrapper.get('[data-testid="archive-enabled"]').element as HTMLInputElement).checked).toBe(false)
|
||||
await wrapper.get('[data-testid="backup-schedule"] .btn-primary').trigger('click')
|
||||
await flushPromises()
|
||||
expect(updateSchedule).toHaveBeenCalledWith(expect.objectContaining({ retain_days: 0, retain_count: 0, monthly_archive: expect.objectContaining({ enabled: false }) }))
|
||||
})
|
||||
|
||||
it('多选日期,手填份数;永久保留隐藏数量并在取消后恢复', async () => {
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
await wrapper.get('[data-testid="archive-enabled"]').setValue(true)
|
||||
await wrapper.get('#backup-archive-dates').trigger('click')
|
||||
await wrapper.get('#backup-archive-options input[value="15"]').setValue(true)
|
||||
await wrapper.get('#backup-archive-options input[value="last"]').setValue(true)
|
||||
expect(wrapper.get('#backup-archive-dates').attributes('aria-expanded')).toBe('true')
|
||||
expect(wrapper.find('[data-testid="archive-count"]').exists()).toBe(false)
|
||||
await wrapper.get('[data-testid="archive-forever"]').setValue(false)
|
||||
await wrapper.get('[data-testid="archive-count"]').setValue(17)
|
||||
await wrapper.get('[data-testid="archive-forever"]').setValue(true)
|
||||
expect(wrapper.find('[data-testid="archive-count"]').exists()).toBe(false)
|
||||
expect(wrapper.findComponent({ name: 'BackupArchiveSettings' }).text()).not.toContain('admin.backup.archive.copies')
|
||||
await wrapper.get('[data-testid="archive-forever"]').setValue(false)
|
||||
expect((wrapper.get('[data-testid="archive-count"]').element as HTMLInputElement).value).toBe('17')
|
||||
await wrapper.get('[data-testid="backup-schedule"] .btn-primary').trigger('click')
|
||||
await flushPromises()
|
||||
expect(updateSchedule).toHaveBeenLastCalledWith(expect.objectContaining({ monthly_archive: { enabled: true, days: [1, 15], include_month_end: true, retain_count: 17 } }))
|
||||
await wrapper.get('[data-testid="archive-forever"]').setValue(true)
|
||||
await wrapper.get('[data-testid="backup-schedule"] .btn-primary').trigger('click')
|
||||
expect(updateSchedule).toHaveBeenLastCalledWith(expect.objectContaining({ monthly_archive: expect.objectContaining({ retain_count: 0 }) }))
|
||||
})
|
||||
|
||||
it('空日期和非法份数不可保存,输入 0 不会误选永久保留', async () => {
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
await wrapper.get('[data-testid="archive-enabled"]').setValue(true)
|
||||
await wrapper.get('#backup-archive-dates').trigger('click')
|
||||
await wrapper.get('#backup-archive-options input[value="1"]').setValue(false)
|
||||
const save = wrapper.get('[data-testid="backup-schedule"] .btn-primary')
|
||||
expect(save.attributes('disabled')).toBeDefined()
|
||||
await wrapper.get('#backup-archive-options input[value="15"]').setValue(true)
|
||||
await wrapper.get('[data-testid="archive-forever"]').setValue(false)
|
||||
for (const value of ['0', '-1', '1.5', '']) {
|
||||
await wrapper.get('[data-testid="archive-count"]').setValue(value)
|
||||
expect(save.attributes('disabled')).toBeDefined()
|
||||
expect((wrapper.get('[data-testid="archive-forever"]').element as HTMLInputElement).checked).toBe(false)
|
||||
}
|
||||
await save.trigger('click')
|
||||
expect(updateSchedule).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('加载归档设置并支持键盘关闭日期选择器', async () => {
|
||||
getSchedule.mockResolvedValue({ enabled: true, cron_expr: '0 4 * * *', retain_days: 14, retain_count: 10,
|
||||
monthly_archive: { enabled: true, days: [1, 15], include_month_end: true, retain_count: 23 } })
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
expect((wrapper.get('[data-testid="archive-count"]').element as HTMLInputElement).value).toBe('23')
|
||||
await wrapper.get('#backup-archive-dates').trigger('keydown', { key: 'ArrowDown' })
|
||||
await flushPromises()
|
||||
expect(wrapper.find('#backup-archive-options').exists()).toBe(true)
|
||||
expect((wrapper.get('#backup-archive-options input[value="15"]').element as HTMLInputElement).checked).toBe(true)
|
||||
await wrapper.get('#backup-archive-options').trigger('keydown', { key: 'Escape' })
|
||||
expect(wrapper.find('#backup-archive-options').exists()).toBe(false)
|
||||
})
|
||||
|
||||
it('归档删除需要单独确认,有限归档不会显示永不过期', async () => {
|
||||
listBackups.mockResolvedValue({ items: [{ ...baseRecord('archived'), monthly_archive: { dates: ['2026-09-01', '2026-09-15'], retain_count: 12 } }] })
|
||||
const confirm = vi.spyOn(window, 'confirm').mockReturnValue(false)
|
||||
const wrapper = mountBackupView()
|
||||
await flushPromises()
|
||||
expect(wrapper.text()).toContain('admin.backup.archive.badge')
|
||||
expect(wrapper.get('tbody tr td:nth-child(6)').text()).toBe('admin.backup.archive.retainLatest')
|
||||
const button = wrapper.findAll('button').find(button => button.text() === 'common.delete')!
|
||||
await button.trigger('click')
|
||||
expect(confirm).toHaveBeenCalledWith('admin.backup.archive.deleteConfirm')
|
||||
expect(deleteBackup).not.toHaveBeenCalled()
|
||||
confirm.mockReturnValue(true)
|
||||
await button.trigger('click')
|
||||
await flushPromises()
|
||||
expect(deleteBackup).toHaveBeenCalledWith('archived', true)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user