From c2ea33846cfdbe1d982e95c6d8c3b24b844bce96 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 6 Jul 2026 21:52:24 +0000 Subject: [PATCH] fix(server): skip zero-byte backups and honor sub-hour freshness thresholds Zero-byte .sql.gz files are failed/partial dumps, so they no longer count as the latest restorable backup. PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS now accepts fractional values instead of being clamped up to a full hour. Co-Authored-By: Paperclip --- server/src/__tests__/health.test.ts | 72 +++++++++++++++++++ server/src/index.ts | 10 +-- server/src/services/database-backup-health.ts | 7 +- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/server/src/__tests__/health.test.ts b/server/src/__tests__/health.test.ts index 5135b3909b8..cf1b3f43c07 100644 --- a/server/src/__tests__/health.test.ts +++ b/server/src/__tests__/health.test.ts @@ -166,6 +166,78 @@ describe("GET /health", () => { expect(res.body.warnings).toEqual(res.body.databaseBackup.warnings); }); + it("ignores zero-byte backup archives when picking the latest backup", async () => { + const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); + const goodBackup = path.join(backupDir, "paperclip-20260705-031702.sql.gz"); + fs.writeFileSync(goodBackup, "backup"); + fs.utimesSync( + goodBackup, + new Date("2026-07-05T03:17:02.000Z"), + new Date("2026-07-05T03:17:02.000Z"), + ); + const emptyBackup = path.join(backupDir, "paperclip-20260706-120000.sql.gz"); + fs.writeFileSync(emptyBackup, ""); + fs.utimesSync( + emptyBackup, + new Date("2026-07-06T12:00:00.000Z"), + new Date("2026-07-06T12:00:00.000Z"), + ); + const app = createApp(createHealthyDb(), testServerInfo, { + enabled: true, + backupDir, + maxAgeHours: 26, + now: new Date("2026-07-06T13:00:00.000Z"), + }); + + const res = await request(app).get("/health"); + + expect(res.status).toBe(200); + expect(res.body.databaseBackup).toMatchObject({ + status: "warning", + latestBackup: { + name: "paperclip-20260705-031702.sql.gz", + }, + warnings: [ + { + code: "database_backup_stale", + }, + ], + }); + }); + + it("honors sub-hour max age thresholds", async () => { + const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); + const backupFile = path.join(backupDir, "paperclip-20260706-121500.sql.gz"); + fs.writeFileSync(backupFile, "backup"); + fs.utimesSync( + backupFile, + new Date("2026-07-06T12:15:00.000Z"), + new Date("2026-07-06T12:15:00.000Z"), + ); + const app = createApp(createHealthyDb(), testServerInfo, { + enabled: true, + backupDir, + maxAgeHours: 0.5, + now: new Date("2026-07-06T13:00:00.000Z"), + }); + + const res = await request(app).get("/health"); + + expect(res.status).toBe(200); + expect(res.body.databaseBackup).toMatchObject({ + status: "warning", + maxAgeHours: 0.5, + latestBackup: { + ageHours: 0.8, + }, + warnings: [ + { + code: "database_backup_stale", + }, + ], + }); + }); + it("warns instead of reporting fresh when the latest backup mtime is in the future", async () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const backupFile = path.join(backupDir, "paperclip-20260707-120000.sql.gz"); diff --git a/server/src/index.ts b/server/src/index.ts index 7a13ae30eb3..7bc7cfa2abb 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -583,11 +583,11 @@ export async function startServer(): Promise { shareClient: createFeedbackTraceShareClientFromConfig(config), }); const backupSettingsSvc = instanceSettingsService(db); - const databaseBackupMaxAgeHours = Math.max( - 1, - Number(process.env.PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS) || - Math.max(26, Math.ceil((config.databaseBackupIntervalMinutes / 60) * 2)), - ); + const configuredBackupMaxAgeHours = Number(process.env.PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS); + const databaseBackupMaxAgeHours = + Number.isFinite(configuredBackupMaxAgeHours) && configuredBackupMaxAgeHours > 0 + ? configuredBackupMaxAgeHours + : Math.max(26, Math.ceil((config.databaseBackupIntervalMinutes / 60) * 2)); const databaseBackupAlertFile = process.env.PAPERCLIP_DB_BACKUP_ALERT_FILE || resolve(config.databaseBackupDir, "..", "health", "db-backup-to-s3.failure"); diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index 625410cca50..bdf6c7ebc3e 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -93,6 +93,8 @@ async function findLatestBackup(backupDir: string, nowMs: number) { if (isMissingFileError(error)) continue; throw error; } + // Zero-byte archives are failed/partial dumps, not restorable backups. + if (fileStat.size === 0) continue; if (!latest || fileStat.mtimeMs > latest.mtimeMs) { latest = { fullPath, mtimeMs: fileStat.mtimeMs, size: fileStat.size }; } @@ -114,7 +116,8 @@ export async function inspectDatabaseBackupHealth( ): Promise { const warnings: DatabaseBackupHealthWarning[] = []; const now = opts.now ?? new Date(); - const maxAgeHours = Math.max(1, opts.maxAgeHours); + const maxAgeHours = + Number.isFinite(opts.maxAgeHours) && opts.maxAgeHours > 0 ? opts.maxAgeHours : 1; let latestBackup: DatabaseBackupHealthStatus["latestBackup"] = null; let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null; @@ -128,7 +131,7 @@ export async function inspectDatabaseBackupHealth( if (!latestBackup) { warnings.push({ code: "database_backup_missing", - message: `No .sql.gz database backups found in ${opts.backupDir}.`, + message: `No non-empty .sql.gz database backups found in ${opts.backupDir}.`, }); } else if (latestBackup.ageHours < 0) { warnings.push({