fix: checkpoint database backups before idle sleep

Add opt-in backup checkpoints under the owned idle hold. Verify and sync
each archive before pruning history, retain backup work through completion,
and take a fresh backup after restart using a persistent marker. Preserve
automatic backups and the instance retention policy.

Tested idle/route/spool checks, PostgreSQL backup restore and retention failure
boundaries, recursive typecheck, and build. Full suite is in progress.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/idle-database-backup.test.ts

@@ -0,0 +1,92 @@
+import { mkdtemp, readFile, rm, symlink, utimes, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { gzipSync } from "node:zlib";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { sql } from "drizzle-orm";
+import { createDb, ensurePostgresDatabase, getEmbeddedPostgresTestSupport, runDatabaseBackup, runDatabaseRestore, startEmbeddedPostgresTestDatabase } from "@paperclipai/db";
+import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "../services/idle-database-backup.js";
+
+let directory: string;
+beforeEach(async () => { directory = await mkdtemp(path.join(os.tmpdir(), "idle-backup-")); });
+afterEach(async () => { await rm(directory, { recursive: true, force: true }); });
+
+describe("idle database backup persistence", () => {
+  it("keeps a restart marker until a fresh valid backup is complete", async () => {
+    const archive = path.join(directory, "backup.sql.gz");
+    await markIdleBackupWakeRequired(directory);
+    await writeFile(archive, gzipSync("CREATE TABLE fixture (id integer);\n"));
+    await verifyIdleDatabaseBackup(archive);
+    // Verification on the sleep path must not remove the wake obligation.
+    expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts");
+    await clearIdleBackupWakeRequired(directory);
+    await expect(readFile(idleBackupWakeMarker(directory))).rejects.toMatchObject({ code: "ENOENT" });
+    expect(await readFile(archive)).toEqual(gzipSync("CREATE TABLE fixture (id integer);\n"));
+  });
+
+  it.each([
+    ["empty file", Buffer.alloc(0)], ["empty SQL", gzipSync("")],
+    ["truncated gzip", gzipSync("SELECT 1;").subarray(0, -4)],
+    ["plain text", Buffer.from("SELECT 1;")],
+  ])("rejects %s and retains the restart obligation", async (_label, contents) => {
+    await markIdleBackupWakeRequired(directory);
+    const archive = path.join(directory, "invalid.sql.gz");
+    await writeFile(archive, contents);
+    await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow();
+    expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts");
+  });
+
+  it("rejects a missing archive and symlinked archive or marker", async () => {
+    const archive = path.join(directory, "backup.sql.gz");
+    await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow();
+    const target = path.join(directory, "target");
+    await writeFile(target, gzipSync("SELECT 1;"));
+    await symlink(target, archive);
+    await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow();
+    await symlink(target, idleBackupWakeMarker(directory));
+    await expect(markIdleBackupWakeRequired(directory)).rejects.toThrow();
+    expect(await readFile(target)).toEqual(gzipSync("SELECT 1;"));
+  });
+});
+
+const support = await getEmbeddedPostgresTestSupport();
+describe.skipIf(!support.supported)("idle backup restore", () => {
+  it("restores the last pre-sleep state into a separate database", async () => {
+    const source = await startEmbeddedPostgresTestDatabase("idle-backup-restore-");
+    const sourceDb = createDb(source.connectionString);
+    const targetUrl = new URL(source.connectionString);
+    targetUrl.pathname = "/idle_restore";
+    try {
+      await sourceDb.execute(sql`CREATE TABLE backup_fixture (id integer PRIMARY KEY, value text NOT NULL)`);
+      await sourceDb.execute(sql`INSERT INTO backup_fixture VALUES (1, 'last committed state')`);
+      await markIdleBackupWakeRequired(directory);
+      const result = await runDatabaseBackup({
+        connectionString: source.connectionString, backupDir: directory,
+        retention: { dailyDays: 14, weeklyWeeks: 4, monthlyMonths: 6 },
+        verifyBeforePrune: verifyIdleDatabaseBackup,
+      });
+      await verifyIdleDatabaseBackup(result.backupFile);
+      await ensurePostgresDatabase(source.connectionString, "idle_restore");
+      await runDatabaseRestore({ connectionString: targetUrl.toString(), backupFile: result.backupFile });
+      const target = createDb(targetUrl.toString());
+      expect(await target.execute(sql`SELECT id, value FROM backup_fixture`)).toEqual([{ id: 1, value: "last committed state" }]);
+      const checkpointBytes = await readFile(result.backupFile);
+      expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts");
+      // A failed verification must not delete existing recovery points, even
+      // when their age would make them eligible for the configured pruning.
+      const history = path.join(directory, "paperclip-history.sql.gz");
+      const historyBytes = gzipSync("SELECT 'historical recovery point';");
+      await writeFile(history, historyBytes);
+      await utimes(history, new Date(0), new Date(0));
+      await expect(runDatabaseBackup({
+        connectionString: source.connectionString, backupDir: directory,
+        backupEngine: "javascript", retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 },
+        verifyBeforePrune: async () => { throw new Error("verification failed"); },
+      })).rejects.toThrow("verification failed");
+      expect(await readFile(history)).toEqual(historyBytes);
+      expect(await readFile(result.backupFile)).toEqual(checkpointBytes);
+    } finally {
+      await source.cleanup();
+    }
+  }, 60_000);
+});

server/src/__tests__/idle-local-work.test.ts

@@ -40,10 +40,12 @@ describe("idle cleanup inspection", () => {
     expect(await readIdleLocalWork()).toBe("none");
     markIdleStartupComplete({ scheduledBackups: true });
     expect(await readIdleLocalWork()).toBe("present");
+    expect(await readIdleLocalWork({ backupCheckpoint: true })).toBe("none");
     markIdleStartupComplete({ scheduledBackups: false });
     await fs.mkdir(custom);
     await fs.writeFile(path.join(custom, "receipt.tmp"), "incomplete");
     expect(await readIdleLocalWork()).toBe("present");
+    expect(await readIdleLocalWork({ backupCheckpoint: true })).toBe("present");
     await fs.rm(custom, { recursive: true });
     expect(await readIdleLocalWork()).toBe("none");
   });

server/src/__tests__/idle-sleep-safety.test.ts

@@ -1,4 +1,4 @@
-import { beginIdleTrackedWork } from "../services/task-admission.js";
+import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js";
 import { randomUUID } from "node:crypto";
 import { eq, sql } from "drizzle-orm";
 import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -22,6 +22,73 @@ const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895";
 const owned = () => ({ ...held(), ownerId });
 const emptyLocal = async () => "none" as const;
 
+describe("idle backup checkpoint", () => {
+  const database = (blocked = false) => {
+    const transaction = vi.fn(async (run: (tx: unknown) => Promise<unknown>) => run({ execute: async () => [{ blocked }] }));
+    return { db: { transaction } as unknown as Db, transaction };
+  };
+  const backup = (work: () => Promise<boolean> = async () => true) => vi.fn(async () => {
+    const finish = beginIdleTrackedWork();
+    try { return await work(); } finally { finish(); }
+  });
+  it("backs up after durable inspection and rechecks local work before authorizing sleep", async () => {
+    const { db, transaction } = database();
+    const local = vi.fn(emptyLocal);
+    const checkpoint = backup(async () => {
+      expect(transaction).toHaveBeenCalledOnce();
+      expect(idleWorkSnapshot().active).toBe(1);
+      return true;
+    });
+    expect(await readIdleSleepSafety(db, owned, () => now, ownerId, local, undefined, checkpoint)).toEqual(none);
+    expect(checkpoint).toHaveBeenCalledOnce();
+    expect(local).toHaveBeenCalledTimes(2);
+  });
+  it.each(["present", "unknown"] as const)("avoids DB probes and backups for known local %s work", async state => {
+    const { db, transaction } = database();
+    const checkpoint = backup();
+    const plugins = vi.fn();
+    expect(await readIdleSleepSafety(db, owned, () => now, ownerId, async () => state, plugins, checkpoint))
+      .toEqual({ version: 1, backgroundWork: state });
+    expect(transaction).not.toHaveBeenCalled();
+    expect(plugins).not.toHaveBeenCalled();
+    expect(checkpoint).not.toHaveBeenCalled();
+  });
+  it("does not start a backup for durable work or an unowned hold", async () => {
+    const checkpoint = backup();
+    expect(await readIdleSleepSafety(database(true).db, owned, () => now, ownerId, emptyLocal, undefined, checkpoint)).toEqual(present);
+    expect(await readIdleSleepSafety(database().db, owned, () => now, "stale", emptyLocal, undefined, checkpoint)).toEqual(unknown);
+    expect(checkpoint).not.toHaveBeenCalled();
+  });
+  it.each(["failed", "threw", "untracked", "concurrent", "expired", "replaced"])("keeps the instance awake when the backup is %s", async kind => {
+    let clock = now;
+    let status = owned();
+    const work = async () => {
+      if (kind === "threw") throw new Error("private archive path");
+      if (kind === "concurrent") { const finish = beginIdleTrackedWork(); finish(); }
+      if (kind === "expired") clock += 60_000;
+      if (kind === "replaced") status = { ...status, ownerId: "replacement" };
+      return kind !== "failed";
+    };
+    const checkpoint = kind === "untracked" ? work : backup(work);
+    expect(await readIdleSleepSafety(database().db, () => status, () => clock, ownerId, emptyLocal, undefined, checkpoint)).toEqual(unknown);
+    expect(idleWorkSnapshot().active).toBe(0);
+  });
+  it("retains actual backup work after the owning hold expires", async () => {
+    let finishBackup!: () => void;
+    const pending = new Promise<void>(resolve => { finishBackup = resolve; });
+    const checkpoint = backup(async () => { await pending; return true; });
+    let clock = now;
+    const scan = readIdleSleepSafety(database().db, owned, () => clock, ownerId, emptyLocal, undefined, checkpoint);
+    await vi.waitFor(() => expect(checkpoint).toHaveBeenCalledOnce());
+    clock += 60_000;
+    expect(idleWorkSnapshot().active).toBe(1);
+    expect(await readIdleSleepSafety(database().db, owned, () => clock, ownerId, emptyLocal)).toEqual(unknown);
+    finishBackup();
+    expect(await scan).toEqual(unknown);
+    expect(idleWorkSnapshot().active).toBe(0);
+  });
+});
+
 
 describe("idle sleep safety failure boundaries", () => {
   it.each([
@@ -36,7 +103,7 @@ describe("idle sleep safety failure boundaries", () => {
 
   it("fails closed when the database is unavailable", async () => {
     const db = { transaction: vi.fn().mockRejectedValue(new Error("private connection detail")) } as unknown as Db;
-    expect(await readIdleSleepSafety(db, held, () => now)).toEqual(unknown);
+    expect(await readIdleSleepSafety(db, held, () => now, undefined, emptyLocal)).toEqual(unknown);
   });
 
   it.each([
@@ -46,14 +113,14 @@ describe("idle sleep safety failure boundaries", () => {
     const getStatus = vi.fn().mockReturnValueOnce(held()).mockReturnValue({ ...held(), ...change });
     const execute = vi.fn().mockResolvedValue([{ blocked: true }]);
     const transaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn({ execute }));
-    expect(await readIdleSleepSafety({ transaction } as unknown as Db, getStatus, () => now)).toEqual(unknown);
+    expect(await readIdleSleepSafety({ transaction } as unknown as Db, getStatus, () => now, undefined, emptyLocal)).toEqual(unknown);
     expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
   });
 
   it("reports persisted work when the admission hold remains unchanged", async () => {
     const execute = vi.fn().mockResolvedValue([{ blocked: true }]);
     const transaction = vi.fn(async (fn: (tx: unknown) => Promise<unknown>) => fn({ execute }));
-    expect(await readIdleSleepSafety({ transaction } as unknown as Db, held, () => now)).toEqual(present);
+    expect(await readIdleSleepSafety({ transaction } as unknown as Db, held, () => now, undefined, emptyLocal)).toEqual(present);
   });
 });
 

server/src/__tests__/instance-settings-routes.test.ts

@@ -77,7 +77,7 @@ describe("instance settings routes", () => {
     return { errorHandler, instanceSettingsRoutes };
   });
 
-  function createApp(actor: any, pluginWorkers?: PluginWorkerManager) {
+  function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise<boolean>) {
     const { errorHandler, instanceSettingsRoutes } = routeModules.value;
     const app = express();
     app.use(express.json());
@@ -85,7 +85,7 @@ describe("instance settings routes", () => {
       req.actor = actor;
       next();
     });
-    app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers));
+    app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers, prepareBackup));
     app.use(errorHandler);
     return app;
   }
@@ -973,7 +973,7 @@ describe("instance settings routes", () => {
       await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200);
       expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 });
       await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200);
-      expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, undefined, undefined);
+      expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined);
     });
 
     it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => {
@@ -1000,7 +1000,7 @@ describe("instance settings routes", () => {
       const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1");
       expect(res.status).toBe(200);
       expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report });
-      expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, undefined, undefined);
+      expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined);
       expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus);
     });
 
@@ -1028,6 +1028,21 @@ describe("instance settings routes", () => {
       expect(release).toHaveBeenCalledOnce();
     });
 
+    it("passes the operator backup hook only through the protected safety report", async () => {
+      const backup = vi.fn(async () => true);
+      mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus);
+      const app = createApp(adminActor, undefined, backup);
+      await request(app).get("/api/instance/task-drain").expect(200);
+      expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
+      await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200);
+      expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now,
+        "fixture-owner", expect.any(Function), undefined, backup);
+      mockReadIdleSleepSafety.mockClear();
+      await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403);
+      expect(mockReadIdleSleepSafety).not.toHaveBeenCalled();
+      expect(backup).not.toHaveBeenCalled();
+    });
+
     it.each([
       ["company member", nonAdminActor],
       ["agent", { type: "agent", agentId: "agent-1", companyId: "company-1" }],