fix(server): verify gzip completeness before trusting backup archives

The backup health check previously accepted any non-empty .sql.gz file as
the latest backup, so an external producer (e.g. a cron pg_dump | gzip
wrapper that does not use the .partial staging convention) crashing
mid-write could leave a truncated archive that /api/health reported as
fresh. Now the newest candidate must decompress as a complete gzip stream
(a valid trailer is producer-agnostic proof of completion); verdicts are
cached by size+mtime so health polls stay cheap, corrupt archives are
skipped with a database_backup_corrupt warning, and the check falls back
to the newest complete backup.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/DEVELOPING.md

@@ -638,6 +638,11 @@ Environment overrides:
 - `PAPERCLIP_DB_BACKUP_ALERT_FILE=/path/to/failure-marker` lets external cron
   wrappers surface the last failed backup in `/api/health`
 
+The `/api/health` backup check only trusts `.sql.gz` archives that decompress as
+complete gzip streams, so truncated dumps from crashed producers (including
+external cron jobs that do not stage through `.partial` files) are ignored and
+reported via a `database_backup_corrupt` warning instead of counting as fresh.
+
 DB backups are not full instance filesystem backups. For full local disaster
 recovery, also back up local storage files and the local encrypted secrets key if
 those providers are enabled.

server/src/__tests__/health.test.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { gzipSync } from "node:zlib";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import express from "express";
 import request from "supertest";
@@ -28,6 +29,9 @@ const testServerInfo = {
   },
 } as const;
 
+const completeGzipArchive = () => gzipSync(Buffer.from("-- PostgreSQL database dump\n"));
+const truncatedGzipArchive = () => completeGzipArchive().subarray(0, 12);
+
 function createHealthyDb(): Db {
   return {
     execute: vi.fn().mockResolvedValue([{ "?column?": 1 }]),
@@ -133,7 +137,7 @@ describe("GET /health", () => {
   it("surfaces a stale database backup warning in full health details", async () => {
     const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
     const backupFile = path.join(backupDir, "paperclip-20260705-031702.sql.gz");
-    fs.writeFileSync(backupFile, "backup");
+    fs.writeFileSync(backupFile, completeGzipArchive());
     fs.utimesSync(
       backupFile,
       new Date("2026-07-05T03:17:02.000Z"),
@@ -169,7 +173,7 @@ describe("GET /health", () => {
   it("ignores zero-byte and in-progress .partial archives when picking the latest backup", async () => {
     const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
     const goodBackup = path.join(backupDir, "paperclip-20260705-031702.sql.gz");
-    fs.writeFileSync(goodBackup, "backup");
+    fs.writeFileSync(goodBackup, completeGzipArchive());
     fs.utimesSync(
       goodBackup,
       new Date("2026-07-05T03:17:02.000Z"),
@@ -207,10 +211,83 @@ describe("GET /health", () => {
     });
   });
 
+  it("skips a truncated newest archive, warns, and falls back to the previous complete backup", async () => {
+    const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
+    const completeBackup = path.join(backupDir, "paperclip-20260706-120000.sql.gz");
+    fs.writeFileSync(completeBackup, completeGzipArchive());
+    fs.utimesSync(
+      completeBackup,
+      new Date("2026-07-06T12:00:00.000Z"),
+      new Date("2026-07-06T12:00:00.000Z"),
+    );
+    const truncatedBackup = path.join(backupDir, "paperclip-20260706-124500.sql.gz");
+    fs.writeFileSync(truncatedBackup, truncatedGzipArchive());
+    fs.utimesSync(
+      truncatedBackup,
+      new Date("2026-07-06T12:45:00.000Z"),
+      new Date("2026-07-06T12:45:00.000Z"),
+    );
+    const app = createApp(createHealthyDb(), testServerInfo, {
+      enabled: true,
+      backupDir,
+      maxAgeHours: 26,
+      now: new Date("2026-07-06T13:00:00.000Z"),
+    });
+
+    const res = await request(app).get("/health");
+
+    expect(res.status).toBe(200);
+    expect(res.body.databaseBackup).toMatchObject({
+      status: "warning",
+      latestBackup: {
+        name: "paperclip-20260706-120000.sql.gz",
+      },
+      warnings: [
+        {
+          code: "database_backup_corrupt",
+          message: expect.stringContaining("paperclip-20260706-124500.sql.gz"),
+        },
+      ],
+    });
+  });
+
+  it("reports backups missing when every archive is an incomplete gzip stream", async () => {
+    const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
+    const truncatedBackup = path.join(backupDir, "paperclip-20260706-124500.sql.gz");
+    fs.writeFileSync(truncatedBackup, truncatedGzipArchive());
+    fs.utimesSync(
+      truncatedBackup,
+      new Date("2026-07-06T12:45:00.000Z"),
+      new Date("2026-07-06T12:45:00.000Z"),
+    );
+    const app = createApp(createHealthyDb(), testServerInfo, {
+      enabled: true,
+      backupDir,
+      maxAgeHours: 26,
+      now: new Date("2026-07-06T13:00:00.000Z"),
+    });
+
+    const res = await request(app).get("/health");
+
+    expect(res.status).toBe(200);
+    expect(res.body.databaseBackup).toMatchObject({
+      status: "warning",
+      latestBackup: null,
+      warnings: [
+        {
+          code: "database_backup_missing",
+        },
+        {
+          code: "database_backup_corrupt",
+        },
+      ],
+    });
+  });
+
   it("honors sub-hour max age thresholds", async () => {
     const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
     const backupFile = path.join(backupDir, "paperclip-20260706-121500.sql.gz");
-    fs.writeFileSync(backupFile, "backup");
+    fs.writeFileSync(backupFile, completeGzipArchive());
     fs.utimesSync(
       backupFile,
       new Date("2026-07-06T12:15:00.000Z"),
@@ -243,7 +320,7 @@ describe("GET /health", () => {
   it("warns instead of reporting fresh when the latest backup mtime is in the future", async () => {
     const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
     const backupFile = path.join(backupDir, "paperclip-20260707-120000.sql.gz");
-    fs.writeFileSync(backupFile, "backup");
+    fs.writeFileSync(backupFile, completeGzipArchive());
     fs.utimesSync(
       backupFile,
       new Date("2026-07-07T12:00:00.000Z"),
@@ -277,7 +354,7 @@ describe("GET /health", () => {
     const backupDir = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-health-backups-"));
     const backupFile = path.join(backupDir, "paperclip-20260706-031702.sql.gz");
     const alertFile = path.join(backupDir, "db-backup-to-s3.failure");
-    fs.writeFileSync(backupFile, "backup");
+    fs.writeFileSync(backupFile, completeGzipArchive());
     fs.utimesSync(
       backupFile,
       new Date("2026-07-06T03:17:02.000Z"),

server/src/routes/openapi.ts

@@ -901,6 +901,7 @@ registry.registerPath({
           code: z.enum([
             "database_backup_check_failed",
             "database_backup_clock_skew",
+            "database_backup_corrupt",
             "database_backup_last_failure",
             "database_backup_missing",
             "database_backup_stale",

server/src/services/database-backup-health.ts

@@ -1,9 +1,14 @@
+import { createReadStream } from "node:fs";
 import { readdir, readFile, stat } from "node:fs/promises";
 import { basename, join } from "node:path";
+import { Writable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { createGunzip } from "node:zlib";
 
 export type DatabaseBackupHealthWarningCode =
   | "database_backup_check_failed"
   | "database_backup_clock_skew"
+  | "database_backup_corrupt"
   | "database_backup_last_failure"
   | "database_backup_missing"
   | "database_backup_stale";
@@ -73,16 +78,57 @@ async function readLastFailure(alertFile: string | undefined) {
   }
 }
 
-async function findLatestBackup(backupDir: string, nowMs: number) {
+type ArchiveIntegrityCacheEntry = { size: number; mtimeMs: number; complete: boolean };
+
+const archiveIntegrityCache = new Map<string, ArchiveIntegrityCacheEntry>();
+
+// A complete gzip stream ends with a valid CRC/length trailer, so a full
+// decompress pass proves the producer finished writing — including external
+// backup jobs that stream straight to `<name>.sql.gz` without Paperclip's
+// `.partial` staging convention. Verdicts are cached by (size, mtime) so
+// repeated /health polls do not re-decompress unchanged archives.
+async function isCompleteGzipArchive(
+  fullPath: string,
+  size: number,
+  mtimeMs: number,
+): Promise<boolean> {
+  const cached = archiveIntegrityCache.get(fullPath);
+  if (cached && cached.size === size && cached.mtimeMs === mtimeMs) return cached.complete;
+
+  let complete = false;
+  try {
+    await pipeline(
+      createReadStream(fullPath),
+      createGunzip(),
+      new Writable({
+        write(_chunk, _encoding, callback) {
+          callback();
+        },
+      }),
+    );
+    complete = true;
+  } catch {
+    complete = false;
+  }
+  archiveIntegrityCache.set(fullPath, { size, mtimeMs, complete });
+  return complete;
+}
+
+type LatestBackupScan = {
+  latest: NonNullable<DatabaseBackupHealthStatus["latestBackup"]> | null;
+  corruptNames: string[];
+};
+
+async function findLatestBackup(backupDir: string, nowMs: number): Promise<LatestBackupScan> {
   let names: string[];
   try {
     names = await readdir(backupDir);
   } catch (error) {
-    if (isMissingFileError(error)) return null;
+    if (isMissingFileError(error)) return { latest: null, corruptNames: [] };
     throw error;
   }
 
-  let latest: { fullPath: string; mtimeMs: number; size: number } | null = null;
+  const candidates: { fullPath: string; mtimeMs: number; size: number }[] = [];
   for (const name of names) {
     if (!name.endsWith(".sql.gz")) continue;
     const fullPath = join(backupDir, name);
@@ -93,25 +139,39 @@ async function findLatestBackup(backupDir: string, nowMs: number) {
       if (isMissingFileError(error)) continue;
       throw error;
     }
-    // The backup writer streams to `<name>.sql.gz.partial` and renames on
-    // success, so `.sql.gz` files are complete by convention (the extension
-    // filter above excludes in-progress/crashed partials). Zero-byte archives
-    // from older writers are still skipped as failed dumps.
+    // Zero-byte archives are failed dumps; in-progress `.sql.gz.partial`
+    // staging files are excluded by the extension filter above.
     if (fileStat.size === 0) continue;
-    if (!latest || fileStat.mtimeMs > latest.mtimeMs) {
-      latest = { fullPath, mtimeMs: fileStat.mtimeMs, size: fileStat.size };
+    candidates.push({ fullPath, mtimeMs: fileStat.mtimeMs, size: fileStat.size });
+  }
+
+  const candidatePaths = new Set(candidates.map((candidate) => candidate.fullPath));
+  for (const key of archiveIntegrityCache.keys()) {
+    if (key.startsWith(backupDir) && !candidatePaths.has(key)) {
+      archiveIntegrityCache.delete(key);
     }
   }
 
-  if (!latest) return null;
+  candidates.sort((a, b) => b.mtimeMs - a.mtimeMs);
 
-  return {
-    name: basename(latest.fullPath),
-    path: latest.fullPath,
-    mtime: new Date(latest.mtimeMs).toISOString(),
-    ageHours: roundHours((nowMs - latest.mtimeMs) / 3_600_000),
-    sizeBytes: latest.size,
-  };
+  const corruptNames: string[] = [];
+  for (const candidate of candidates) {
+    if (await isCompleteGzipArchive(candidate.fullPath, candidate.size, candidate.mtimeMs)) {
+      return {
+        latest: {
+          name: basename(candidate.fullPath),
+          path: candidate.fullPath,
+          mtime: new Date(candidate.mtimeMs).toISOString(),
+          ageHours: roundHours((nowMs - candidate.mtimeMs) / 3_600_000),
+          sizeBytes: candidate.size,
+        },
+        corruptNames,
+      };
+    }
+    corruptNames.push(basename(candidate.fullPath));
+  }
+
+  return { latest: null, corruptNames };
 }
 
 export async function inspectDatabaseBackupHealth(
@@ -126,15 +186,17 @@ export async function inspectDatabaseBackupHealth(
   let lastFailure: DatabaseBackupHealthStatus["lastFailure"] = null;
 
   try {
-    [latestBackup, lastFailure] = await Promise.all([
+    const [scan, failure] = await Promise.all([
       findLatestBackup(opts.backupDir, now.getTime()),
       readLastFailure(opts.alertFile),
     ]);
+    latestBackup = scan.latest;
+    lastFailure = failure;
 
     if (!latestBackup) {
       warnings.push({
         code: "database_backup_missing",
-        message: `No non-empty .sql.gz database backups found in ${opts.backupDir}.`,
+        message: `No complete non-empty .sql.gz database backups found in ${opts.backupDir}.`,
       });
     } else if (latestBackup.ageHours < 0) {
       warnings.push({
@@ -148,6 +210,13 @@ export async function inspectDatabaseBackupHealth(
       });
     }
 
+    if (scan.corruptNames.length > 0) {
+      warnings.push({
+        code: "database_backup_corrupt",
+        message: `Ignored ${scan.corruptNames.length} truncated/corrupt .sql.gz backup archive(s) with incomplete gzip streams: ${scan.corruptNames.join(", ")}.`,
+      });
+    }
+
     if (lastFailure) {
       warnings.push({
         code: "database_backup_last_failure",