diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 740d264951e..b1065f21ca5 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -638,6 +638,11 @@ Environment overrides: - `PAPERCLIP_DB_BACKUP_ALERT_FILE=/path/to/failure-marker` lets external cron wrappers surface the last failed backup in `/api/health` +The `/api/health` backup check only trusts `.sql.gz` archives that decompress as +complete gzip streams, so truncated dumps from crashed producers (including +external cron jobs that do not stage through `.partial` files) are ignored and +reported via a `database_backup_corrupt` warning instead of counting as fresh. + DB backups are not full instance filesystem backups. For full local disaster recovery, also back up local storage files and the local encrypted secrets key if those providers are enabled. diff --git a/server/src/__tests__/health.test.ts b/server/src/__tests__/health.test.ts index 8fd736fcf1b..2c723880c11 100644 --- a/server/src/__tests__/health.test.ts +++ b/server/src/__tests__/health.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { gzipSync } from "node:zlib"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import express from "express"; import request from "supertest"; @@ -28,6 +29,9 @@ const testServerInfo = { }, } as const; +const completeGzipArchive = () => gzipSync(Buffer.from("-- PostgreSQL database dump\n")); +const truncatedGzipArchive = () => completeGzipArchive().subarray(0, 12); + function createHealthyDb(): Db { return { execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]), @@ -133,7 +137,7 @@ describe("GET /health", () => { it("surfaces a stale database backup warning in full health details", async () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz"); - fs.writeFileSync(backupFile, "backup"); + fs.writeFileSync(backupFile, completeGzipArchive()); fs.utimesSync( backupFile, new Date("2026-07-05T03:17:02.000Z"), @@ -169,7 +173,7 @@ describe("GET /health", () => { it("ignores zero-byte and in-progress .partial archives when picking the latest backup", async () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const goodBackup = path.join(backupDir, "paperclip-20260705-031702.sql.gz"); - fs.writeFileSync(goodBackup, "backup"); + fs.writeFileSync(goodBackup, completeGzipArchive()); fs.utimesSync( goodBackup, new Date("2026-07-05T03:17:02.000Z"), @@ -207,10 +211,83 @@ describe("GET /health", () => { }); }); + it("skips a truncated newest archive, warns, and falls back to the previous complete backup", async () => { + const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); + const completeBackup = path.join(backupDir, "paperclip-20260706-120000.sql.gz"); + fs.writeFileSync(completeBackup, completeGzipArchive()); + fs.utimesSync( + completeBackup, + new Date("2026-07-06T12:00:00.000Z"), + new Date("2026-07-06T12:00:00.000Z"), + ); + const truncatedBackup = path.join(backupDir, "paperclip-20260706-124500.sql.gz"); + fs.writeFileSync(truncatedBackup, truncatedGzipArchive()); + fs.utimesSync( + truncatedBackup, + new Date("2026-07-06T12:45:00.000Z"), + new Date("2026-07-06T12:45:00.000Z"), + ); + const app = createApp(createHealthyDb(), testServerInfo, { + enabled: true, + backupDir, + maxAgeHours: 26, + now: new Date("2026-07-06T13:00:00.000Z"), + }); + + const res = await request(app).get("/health"); + + expect(res.status).toBe(200); + expect(res.body.databaseBackup).toMatchObject({ + status: "warning", + latestBackup: { + name: "paperclip-20260706-120000.sql.gz", + }, + warnings: [ + { + code: "database_backup_corrupt", + message: expect.stringContaining("paperclip-20260706-124500.sql.gz"), + }, + ], + }); + }); + + it("reports backups missing when every archive is an incomplete gzip stream", async () => { + const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); + const truncatedBackup = path.join(backupDir, "paperclip-20260706-124500.sql.gz"); + fs.writeFileSync(truncatedBackup, truncatedGzipArchive()); + fs.utimesSync( + truncatedBackup, + new Date("2026-07-06T12:45:00.000Z"), + new Date("2026-07-06T12:45:00.000Z"), + ); + const app = createApp(createHealthyDb(), testServerInfo, { + enabled: true, + backupDir, + maxAgeHours: 26, + now: new Date("2026-07-06T13:00:00.000Z"), + }); + + const res = await request(app).get("/health"); + + expect(res.status).toBe(200); + expect(res.body.databaseBackup).toMatchObject({ + status: "warning", + latestBackup: null, + warnings: [ + { + code: "database_backup_missing", + }, + { + code: "database_backup_corrupt", + }, + ], + }); + }); + it("honors sub-hour max age thresholds", async () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const backupFile = path.join(backupDir, "paperclip-20260706-121500.sql.gz"); - fs.writeFileSync(backupFile, "backup"); + fs.writeFileSync(backupFile, completeGzipArchive()); fs.utimesSync( backupFile, new Date("2026-07-06T12:15:00.000Z"), @@ -243,7 +320,7 @@ describe("GET /health", () => { it("warns instead of reporting fresh when the latest backup mtime is in the future", async () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const backupFile = path.join(backupDir, "paperclip-20260707-120000.sql.gz"); - fs.writeFileSync(backupFile, "backup"); + fs.writeFileSync(backupFile, completeGzipArchive()); fs.utimesSync( backupFile, new Date("2026-07-07T12:00:00.000Z"), @@ -277,7 +354,7 @@ describe("GET /health", () => { const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-")); const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz"); const alertFile = path.join(backupDir, "db-backup-to-s3.failure"); - fs.writeFileSync(backupFile, "backup"); + fs.writeFileSync(backupFile, completeGzipArchive()); fs.utimesSync( backupFile, new Date("2026-07-06T03:17:02.000Z"), diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 16778f1add5..edf0861c983 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -901,6 +901,7 @@ registry.registerPath({ code: z.enum([ "database_backup_check_failed", "database_backup_clock_skew", + "database_backup_corrupt", "database_backup_last_failure", "database_backup_missing", "database_backup_stale", diff --git a/server/src/services/database-backup-health.ts b/server/src/services/database-backup-health.ts index 7a604343604..cf711d4e9e0 100644 --- a/server/src/services/database-backup-health.ts +++ b/server/src/services/database-backup-health.ts @@ -1,9 +1,14 @@ +import { createReadStream } from "node:fs"; import { readdir, readFile, stat } from "node:fs/promises"; import { basename, join } from "node:path"; +import { Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; export type DatabaseBackupHealthWarningCode = | "database_backup_check_failed" | "database_backup_clock_skew" + | "database_backup_corrupt" | "database_backup_last_failure" | "database_backup_missing" | "database_backup_stale"; @@ -73,16 +78,57 @@ async function readLastFailure(alertFile: string | undefined) { } } -async function findLatestBackup(backupDir: string, nowMs: number) { +type ArchiveIntegrityCacheEntry = { size: number; mtimeMs: number; complete: boolean }; + +const archiveIntegrityCache = new Map(); + +// A complete gzip stream ends with a valid CRC/length trailer, so a full +// decompress pass proves the producer finished writing — including external +// backup jobs that stream straight to `.sql.gz` without Paperclip's +// `.partial` staging convention. Verdicts are cached by (size, mtime) so +// repeated /health polls do not re-decompress unchanged archives. +async function isCompleteGzipArchive( + fullPath: string, + size: number, + mtimeMs: number, +): Promise { + const cached = archiveIntegrityCache.get(fullPath); + if (cached && cached.size === size && cached.mtimeMs === mtimeMs) return cached.complete; + + let complete = false; + try { + await pipeline( + createReadStream(fullPath), + createGunzip(), + new Writable({ + write(_chunk, _encoding, callback) { + callback(); + }, + }), + ); + complete = true; + } catch { + complete = false; + } + archiveIntegrityCache.set(fullPath, { size, mtimeMs, complete }); + return complete; +} + +type LatestBackupScan = { + latest: NonNullable | null; + corruptNames: string[]; +}; + +async function findLatestBackup(backupDir: string, nowMs: number): Promise { let names: string[]; try { names = await readdir(backupDir); } catch (error) { - if (isMissingFileError(error)) return null; + if (isMissingFileError(error)) return { latest: null, corruptNames: [] }; throw error; } - let latest: { fullPath: string; mtimeMs: number; size: number } | null = null; + const candidates: { fullPath: string; mtimeMs: number; size: number }[] = []; for (const name of names) { if (!name.endsWith(".sql.gz")) continue; const fullPath = join(backupDir, name); @@ -93,25 +139,39 @@ async function findLatestBackup(backupDir: string, nowMs: number) { if (isMissingFileError(error)) continue; throw error; } - // The backup writer streams to `.sql.gz.partial` and renames on - // success, so `.sql.gz` files are complete by convention (the extension - // filter above excludes in-progress/crashed partials). Zero-byte archives - // from older writers are still skipped as failed dumps. + // Zero-byte archives are failed dumps; in-progress `.sql.gz.partial` + // staging files are excluded by the extension filter above. if (fileStat.size === 0) continue; - if (!latest || fileStat.mtimeMs > latest.mtimeMs) { - latest = { fullPath, mtimeMs: fileStat.mtimeMs, size: fileStat.size }; + candidates.push({ fullPath, mtimeMs: fileStat.mtimeMs, size: fileStat.size }); + } + + const candidatePaths = new Set(candidates.map((candidate) => candidate.fullPath)); + for (const key of archiveIntegrityCache.keys()) { + if (key.startsWith(backupDir) && !candidatePaths.has(key)) { + archiveIntegrityCache.delete(key); } } - if (!latest) return null; + candidates.sort((a, b) => b.mtimeMs - a.mtimeMs); - return { - name: basename(latest.fullPath), - path: latest.fullPath, - mtime: new Date(latest.mtimeMs).toISOString(), - ageHours: roundHours((nowMs - latest.mtimeMs) / 3_600_000), - sizeBytes: latest.size, - }; + const corruptNames: string[] = []; + for (const candidate of candidates) { + if (await isCompleteGzipArchive(candidate.fullPath, candidate.size, candidate.mtimeMs)) { + return { + latest: { + name: basename(candidate.fullPath), + path: candidate.fullPath, + mtime: new Date(candidate.mtimeMs).toISOString(), + ageHours: roundHours((nowMs - candidate.mtimeMs) / 3_600_000), + sizeBytes: candidate.size, + }, + corruptNames, + }; + } + corruptNames.push(basename(candidate.fullPath)); + } + + return { latest: null, corruptNames }; } export async function inspectDatabaseBackupHealth( @@ -126,15 +186,17 @@ export async function inspectDatabaseBackupHealth( let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null; try { - [latestBackup, lastFailure] = await Promise.all([ + const [scan, failure] = await Promise.all([ findLatestBackup(opts.backupDir, now.getTime()), readLastFailure(opts.alertFile), ]); + latestBackup = scan.latest; + lastFailure = failure; if (!latestBackup) { warnings.push({ code: "database_backup_missing", - message: `No non-empty .sql.gz database backups found in ${opts.backupDir}.`, + message: `No complete non-empty .sql.gz database backups found in ${opts.backupDir}.`, }); } else if (latestBackup.ageHours < 0) { warnings.push({ @@ -148,6 +210,13 @@ export async function inspectDatabaseBackupHealth( }); } + if (scan.corruptNames.length > 0) { + warnings.push({ + code: "database_backup_corrupt", + message: `Ignored ${scan.corruptNames.length} truncated/corrupt .sql.gz backup archive(s) with incomplete gzip streams: ${scan.corruptNames.join(", ")}.`, + }); + } + if (lastFailure) { warnings.push({ code: "database_backup_last_failure",