diff --git a/packages/db/src/backup-lib.test.ts b/packages/db/src/backup-lib.test.ts index 5cabb4d0be2..615d2f2b180 100644 --- a/packages/db/src/backup-lib.test.ts +++ b/packages/db/src/backup-lib.test.ts @@ -122,6 +122,19 @@ describeEmbeddedPostgres("runDatabaseBackup", () => { `; } + const stalePartial = path.join( + backupDir, + "paperclip-test-20200101-000000.sql.gz.partial", + ); + fs.writeFileSync(stalePartial, "orphaned crashed backup"); + const staleMtime = new Date(Date.now() - 48 * 60 * 60 * 1000); + fs.utimesSync(stalePartial, staleMtime, staleMtime); + const freshPartial = path.join( + backupDir, + "paperclip-test-20990101-000000.sql.gz.partial", + ); + fs.writeFileSync(freshPartial, "concurrent in-progress backup"); + const result = await runDatabaseBackup({ connectionString: sourceConnectionString, backupDir, @@ -133,6 +146,11 @@ describeEmbeddedPostgres("runDatabaseBackup", () => { expect(result.backupFile).toMatch(/paperclip-test-.*\.sql\.gz$/); expect(result.sizeBytes).toBeGreaterThan(0); expect(fs.existsSync(result.backupFile)).toBe(true); + // The completed archive is renamed from its .partial staging path, + // stale orphaned partials are pruned, recent ones are left alone. + expect(fs.existsSync(`${result.backupFile}.partial`)).toBe(false); + expect(fs.existsSync(stalePartial)).toBe(false); + expect(fs.existsSync(freshPartial)).toBe(true); await runDatabaseRestore({ connectionString: restoreConnectionString, diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index 02dc168bdb3..90d14400691 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -1,4 +1,4 @@ -import { createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from "node:fs"; +import { createReadStream, createWriteStream, existsSync, mkdirSync, readdirSync, renameSync, statSync, unlinkSync } from "node:fs"; import { basename, resolve } from "node:path"; import { createInterface } from "node:readline"; import { spawn } from "node:child_process"; @@ -125,10 +125,19 @@ function pruneOldBackups(backupDir: string, retention: BackupRetentionPolicy, fi type BackupEntry = { name: string; fullPath: string; mtimeMs: number }; const entries: BackupEntry[] = []; + const stalePartialCutoff = now - 24 * 60 * 60 * 1000; for (const name of readdirSync(backupDir)) { if (!name.startsWith(`${filenamePrefix}-`)) continue; - if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue; const fullPath = resolve(backupDir, name); + if (name.endsWith(".sql.gz.partial")) { + // Orphaned in-progress archives from a crashed backup run. + const stat = statSync(fullPath); + if (stat.mtimeMs < stalePartialCutoff) { + try { unlinkSync(fullPath); } catch { /* ignore */ } + } + continue; + } + if (!name.endsWith(".sql") && !name.endsWith(".sql.gz")) continue; const stat = statSync(fullPath); entries.push({ name, fullPath, mtimeMs: stat.mtimeMs }); } @@ -536,6 +545,9 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise mkdirSync(opts.backupDir, { recursive: true }); const sqlFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`); const backupFile = `${sqlFile}.gz`; + // Stream to a .partial path and rename on success so a .sql.gz file only + // ever exists once the archive is complete (crash/kill leaves .partial). + const partialFile = `${backupFile}.partial`; const writer = createBufferedTextFileWriter(sqlFile); try { @@ -545,9 +557,10 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise await closeSql(); await runPgDumpBackup({ connectionString: opts.connectionString, - backupFile, + backupFile: partialFile, connectTimeout, }); + renameSync(partialFile, backupFile); await writer.abort(); const sizeBytes = statSync(backupFile).size; const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix); @@ -557,8 +570,10 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise prunedCount, }; } catch (error) { - if (existsSync(backupFile)) { - try { unlinkSync(backupFile); } catch { /* ignore */ } + for (const staleFile of [partialFile, backupFile]) { + if (existsSync(staleFile)) { + try { unlinkSync(staleFile); } catch { /* ignore */ } + } } if (backupEngine === "pg_dump") { throw error; @@ -957,8 +972,9 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise // Compress the SQL file with gzip const sqlReadStream = createReadStream(sqlFile); - const gzWriteStream = createWriteStream(backupFile); + const gzWriteStream = createWriteStream(partialFile); await pipeline(sqlReadStream, createGzip(), gzWriteStream); + renameSync(partialFile, backupFile); unlinkSync(sqlFile); const sizeBytes = statSync(backupFile).size; @@ -971,11 +987,10 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise }; } catch (error) { await writer.abort(); - if (existsSync(backupFile)) { - try { unlinkSync(backupFile); } catch { /* ignore */ } - } - if (existsSync(sqlFile)) { - try { unlinkSync(sqlFile); } catch { /* ignore */ } + for (const staleFile of [partialFile, backupFile, sqlFile]) { + if (existsSync(staleFile)) { + try { unlinkSync(staleFile); } catch { /* ignore */ } + } } throw error; } finally { diff --git a/server/src/__tests__/health.test.ts b/server/src/__tests__/health.test.ts index cf1b3f43c07..8fd736fcf1b 100644 --- a/server/src/__tests__/health.test.ts +++ b/server/src/__tests__/health.test.ts @@ -166,7 +166,7 @@ describe("GET /health", () => { expect(res.body.warnings).toEqual(res.body.databaseBackup.warnings); }); - it("ignores zero-byte backup archives when picking the latest backup", async () => { + it("ignores zero-byte and in-progress .partial archives when picking the latest backup", async () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const goodBackup = path.join(backupDir, "paperclip-20260705-031702.sql.gz"); fs.writeFileSync(goodBackup, "backup"); @@ -182,6 +182,8 @@ describe("GET /health", () => { new Date("2026-07-06T12:00:00.000Z"), new Date("2026-07-06T12:00:00.000Z"), ); + const partialBackup = path.join(backupDir, "paperclip-20260706-123000.sql.gz.partial"); + fs.writeFileSync(partialBackup, "in-progress dump"); const app = createApp(createHealthyDb(), testServerInfo, { enabled: true, backupDir, diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index bdf6c7ebc3e..7a604343604 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -93,7 +93,10 @@ async function findLatestBackup(backupDir: string, nowMs: number) { if (isMissingFileError(error)) continue; throw error; } - // Zero-byte archives are failed/partial dumps, not restorable backups. + // The backup writer streams to `.sql.gz.partial` and renames on + // success, so `.sql.gz` files are complete by convention (the extension + // filter above excludes in-progress/crashed partials). Zero-byte archives + // from older writers are still skipped as failed dumps. if (fileStat.size === 0) continue; if (!latest || fileStat.mtimeMs > latest.mtimeMs) { latest = { fullPath, mtimeMs: fileStat.mtimeMs, size: fileStat.size };