diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 9736a2e7be1..84dfd41b1e5 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1687,6 +1687,9 @@ Environment overrides: - `PAPERCLIP_DB_BACKUP_ENABLED=true|false` - `PAPERCLIP_DB_BACKUP_INTERVAL_MINUTES=` +- `PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED=1` enables verified final backups + for owned idle sleep, with restart catch-up. Off by default; see + [idle sleep safety](idle-sleep-safety.md) for the hosting and storage contract. - `PAPERCLIP_DB_BACKUP_RETENTION_DAYS=` - `PAPERCLIP_DB_BACKUP_DIR=/absolute/or/~/path` - `PAPERCLIP_DB_BACKUP_MAX_AGE_HOURS=` controls the `/api/health` diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 61f0977f044..71bd5a549e2 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -52,11 +52,50 @@ counted after readiness until its writes settle. Scheduler work already in flight remains counted until its promise settles. Idle holds pause new scheduler admissions. Database backup promises remain -counted through success or failure. Configured periodic backups also block -sleep until a host owns a durable wake schedule for them. A generation counter invalidates a +counted through success or failure. Configured periodic backups block sleep by +default. The opt-in checkpoint mode below preserves a final backup instead of +running the timer while asleep. A generation counter invalidates a scan if tracked work both starts and finishes during inspection. Reports remain unknown until startup recovery and HTTP tracking are installed. +### Backup checkpoints for idle instances + +`PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED=1` opts an instance with automatic +backups enabled into this mode. Keep its database and persistent backup volume +attached across sleep. All writers must use the instance's admission protocol; +this mode cannot cover independent writers to the same database. +On managed instances, checkpoint-enabled safety reads require a verified signed +Cloud control actor. Tenant instance-admin elevation does not grant platform +backup authority. Self-hosted instance administrators retain operator access. + +Each owned safety read first checks local work, plugin drains and durable work. +Known local blockers return before opening a database transaction. If every +other check passes, the server takes a fresh backup with the existing backup +engine and the retention policy from Instance Settings. It reads the full gzip +archive, checks its trailer and nonempty content, and syncs the archive and +directory before returning `none`. The final pre-stop revalidation takes another +fresh backup. Reads can therefore take longer; the host must bound its request +and still leave enough lease time for a confirmed provider stop. + +There is no cached backup exemption. Failed, concurrent or untracked backups, +other work during the scan, changed owners and expired holds cannot authorize +sleep. Client timeouts do not clear the backup's in-flight token. Slow backups +may retain an awake instance until a later sleep attempt. + +A durable `.idle-backup-wake-required` marker is written before the dump. +No recurring duplicate archives are produced while the instance is asleep. +The last pre-sleep archive and existing history stay on persistent storage; +no retention pruning runs during sleep. On restart, the server attempts a fresh +backup before enabling idle eligibility, then resumes the normal interval. +Only a verified new backup clears the marker. Failure keeps it for the next +scheduled retry. Disabling automatic backups explicitly still disables this +startup backup. Disabling only checkpoint mode restores the sleep blocker and +continues to honor an existing restart marker. + +This does not replace filesystem backups, encryption-key recovery or off-provider +disaster recovery. Validate restoration of a staging checkpoint before enabling +the flag on customer instances. + The report inspects accounting and sandbox-cleanup spools directly. Any entry, including malformed JSON, temporary files and failed write probes, blocks sleep. Read errors and invalid spool paths return unknown. Recovery readers that skip diff --git a/packages/db/src/backup-lib.test.ts b/packages/db/src/backup-lib.test.ts index b6851534991..de3166714cc 100644 --- a/packages/db/src/backup-lib.test.ts +++ b/packages/db/src/backup-lib.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { gunzipSync } from "node:zlib"; +import { gunzipSync, gzipSync } from "node:zlib"; import { afterEach, describe, expect, it } from "vitest"; import postgres from "postgres"; import { createBufferedTextFileWriter, runDatabaseBackup, runDatabaseRestore } from "./backup-lib.js"; @@ -75,6 +75,38 @@ describe("createBufferedTextFileWriter", () => { }); describeEmbeddedPostgres("runDatabaseBackup", () => { + it("preserves a post-pg_dump verification error without falling back or pruning", async () => { + const connectionString = await createTempDatabase(); + const backupDir = createTempDir("checkpoint-pg-dump-"); + const executable = path.join(backupDir, "pg-dump-fixture"); + fs.writeFileSync(executable, "#!/bin/sh\nprintf 'SELECT 1;\\n'\n", { mode: 0o700 }); + const history = path.join(backupDir, "paperclip-history.sql.gz"); + const historyBytes = gzipSync("SELECT 'history';"); + fs.writeFileSync(history, historyBytes); + fs.utimesSync(history, new Date(0), new Date(0)); + const previous = process.env.PAPERCLIP_PG_DUMP_PATH; + process.env.PAPERCLIP_PG_DUMP_PATH = executable; + const failure = new Error("fixture archive fsync failed"); + let verified = 0; + try { + await expect(runDatabaseBackup({ + connectionString, backupDir, + retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 }, + verifyBeforePrune: async archive => { + verified++; + expect(gunzipSync(fs.readFileSync(archive)).toString()).toBe("SELECT 1;\n"); + throw failure; + }, + })).rejects.toBe(failure); + expect(verified).toBe(1); + expect(fs.readFileSync(history)).toEqual(historyBytes); + expect(fs.readdirSync(backupDir).filter(name => name.endsWith(".sql.gz"))).toEqual(["paperclip-history.sql.gz"]); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_PG_DUMP_PATH; + else process.env.PAPERCLIP_PG_DUMP_PATH = previous; + } + }); + it("preserves identity generation, sequence options and progress in JavaScript backups", async () => { const source = await createTempDatabase(); const target = await createSiblingDatabase(source, "identity_restore_target"); diff --git a/packages/db/src/backup-lib.ts b/packages/db/src/backup-lib.ts index a95ac3f1be8..fdf849de266 100644 --- a/packages/db/src/backup-lib.ts +++ b/packages/db/src/backup-lib.ts @@ -6,6 +6,7 @@ import { open as openFile } from "node:fs/promises"; import { pipeline } from "node:stream/promises"; import { createGunzip, createGzip } from "node:zlib"; import postgres from "postgres"; +import { randomUUID } from "node:crypto"; export type BackupRetentionPolicy = { dailyDays: number; @@ -28,6 +29,8 @@ export type RunDatabaseBackupOptions = { excludeTables?: string[]; nullifyColumns?: Record; backupEngine?: "auto" | "pg_dump" | "javascript"; + /** Optional durability gate. A rejected archive must not prune history. */ + verifyBeforePrune?: (backupFile: string) => Promise; }; export type RunDatabaseBackupResult = { @@ -542,13 +545,17 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise await sql.end(); }; mkdirSync(opts.backupDir, { recursive: true }); - const sqlFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}.sql`); + // Initial and final sleep checks can run within the same second. A failed + // second checkpoint must never overwrite the first verified recovery point. + const checkpointSuffix = opts.verifyBeforePrune ? `-${randomUUID()}` : ""; + const sqlFile = resolve(opts.backupDir, `${filenamePrefix}-${timestamp()}${checkpointSuffix}.sql`); const backupFile = `${sqlFile}.gz`; const writer = createBufferedTextFileWriter(sqlFile); try { if (backupEngine === "pg_dump" || (backupEngine === "auto" && canUsePgDump)) { await sql`SELECT 1`; + let dumped = false; try { await closeSql(); await runPgDumpBackup({ @@ -556,14 +563,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise backupFile, connectTimeout, }); - await writer.abort(); - const sizeBytes = statSync(backupFile).size; - const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix); - return { - backupFile, - sizeBytes, - prunedCount, - }; + dumped = true; } catch (error) { if (existsSync(backupFile)) { try { unlinkSync(backupFile); } catch { /* ignore */ } @@ -575,6 +575,16 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise sql = postgres(opts.connectionString, { max: 1, connect_timeout: connectTimeout }); sqlClosed = false; } + if (dumped) { + // Only an engine failure can select the JavaScript fallback. Once + // pg_dump succeeds, archive verification or persistence failures + // must retain their original error and must not reuse a closed writer. + await writer.abort(); + const sizeBytes = statSync(backupFile).size; + await opts.verifyBeforePrune?.(backupFile); + const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix); + return { backupFile, sizeBytes, prunedCount }; + } } await sql`SELECT 1`; @@ -1056,6 +1066,7 @@ export async function runDatabaseBackup(opts: RunDatabaseBackupOptions): Promise unlinkSync(sqlFile); const sizeBytes = statSync(backupFile).size; + await opts.verifyBeforePrune?.(backupFile); const prunedCount = pruneOldBackups(opts.backupDir, retention, filenamePrefix); return { diff --git a/server/src/__tests__/idle-database-backup.test.ts b/server/src/__tests__/idle-database-backup.test.ts new file mode 100644 index 00000000000..07e794c9205 --- /dev/null +++ b/server/src/__tests__/idle-database-backup.test.ts @@ -0,0 +1,92 @@ +import { mkdtemp, readFile, rm, symlink, utimes, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { gzipSync } from "node:zlib"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { sql } from "drizzle-orm"; +import { createDb, ensurePostgresDatabase, getEmbeddedPostgresTestSupport, runDatabaseBackup, runDatabaseRestore, startEmbeddedPostgresTestDatabase } from "@paperclipai/db"; +import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "../services/idle-database-backup.js"; + +let directory: string; +beforeEach(async () => { directory = await mkdtemp(path.join(os.tmpdir(), "idle-backup-")); }); +afterEach(async () => { await rm(directory, { recursive: true, force: true }); }); + +describe("idle database backup persistence", () => { + it("keeps a restart marker until a fresh valid backup is complete", async () => { + const archive = path.join(directory, "backup.sql.gz"); + await markIdleBackupWakeRequired(directory); + await writeFile(archive, gzipSync("CREATE TABLE fixture (id integer);\n")); + await verifyIdleDatabaseBackup(archive); + // Verification on the sleep path must not remove the wake obligation. + expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts"); + await clearIdleBackupWakeRequired(directory); + await expect(readFile(idleBackupWakeMarker(directory))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(archive)).toEqual(gzipSync("CREATE TABLE fixture (id integer);\n")); + }); + + it.each([ + ["empty file", Buffer.alloc(0)], ["empty SQL", gzipSync("")], + ["truncated gzip", gzipSync("SELECT 1;").subarray(0, -4)], + ["plain text", Buffer.from("SELECT 1;")], + ])("rejects %s and retains the restart obligation", async (_label, contents) => { + await markIdleBackupWakeRequired(directory); + const archive = path.join(directory, "invalid.sql.gz"); + await writeFile(archive, contents); + await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow(); + expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts"); + }); + + it("rejects a missing archive and symlinked archive or marker", async () => { + const archive = path.join(directory, "backup.sql.gz"); + await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow(); + const target = path.join(directory, "target"); + await writeFile(target, gzipSync("SELECT 1;")); + await symlink(target, archive); + await expect(verifyIdleDatabaseBackup(archive)).rejects.toThrow(); + await symlink(target, idleBackupWakeMarker(directory)); + await expect(markIdleBackupWakeRequired(directory)).rejects.toThrow(); + expect(await readFile(target)).toEqual(gzipSync("SELECT 1;")); + }); +}); + +const support = await getEmbeddedPostgresTestSupport(); +describe.skipIf(!support.supported)("idle backup restore", () => { + it("restores the last pre-sleep state into a separate database", async () => { + const source = await startEmbeddedPostgresTestDatabase("idle-backup-restore-"); + const sourceDb = createDb(source.connectionString); + const targetUrl = new URL(source.connectionString); + targetUrl.pathname = "/idle_restore"; + try { + await sourceDb.execute(sql`CREATE TABLE backup_fixture (id integer PRIMARY KEY, value text NOT NULL)`); + await sourceDb.execute(sql`INSERT INTO backup_fixture VALUES (1, 'last committed state')`); + await markIdleBackupWakeRequired(directory); + const result = await runDatabaseBackup({ + connectionString: source.connectionString, backupDir: directory, + retention: { dailyDays: 14, weeklyWeeks: 4, monthlyMonths: 6 }, + verifyBeforePrune: verifyIdleDatabaseBackup, + }); + await verifyIdleDatabaseBackup(result.backupFile); + await ensurePostgresDatabase(source.connectionString, "idle_restore"); + await runDatabaseRestore({ connectionString: targetUrl.toString(), backupFile: result.backupFile }); + const target = createDb(targetUrl.toString()); + expect(await target.execute(sql`SELECT id, value FROM backup_fixture`)).toEqual([{ id: 1, value: "last committed state" }]); + const checkpointBytes = await readFile(result.backupFile); + expect(await readFile(idleBackupWakeMarker(directory), "utf8")).toContain("restarts"); + // A failed verification must not delete existing recovery points, even + // when their age would make them eligible for the configured pruning. + const history = path.join(directory, "paperclip-history.sql.gz"); + const historyBytes = gzipSync("SELECT 'historical recovery point';"); + await writeFile(history, historyBytes); + await utimes(history, new Date(0), new Date(0)); + await expect(runDatabaseBackup({ + connectionString: source.connectionString, backupDir: directory, + backupEngine: "javascript", retention: { dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 }, + verifyBeforePrune: async () => { throw new Error("verification failed"); }, + })).rejects.toThrow("verification failed"); + expect(await readFile(history)).toEqual(historyBytes); + expect(await readFile(result.backupFile)).toEqual(checkpointBytes); + } finally { + await source.cleanup(); + } + }, 60_000); +}); diff --git a/server/src/__tests__/idle-local-work.test.ts b/server/src/__tests__/idle-local-work.test.ts index 51e9dae011f..666c58d0b20 100644 --- a/server/src/__tests__/idle-local-work.test.ts +++ b/server/src/__tests__/idle-local-work.test.ts @@ -40,10 +40,12 @@ describe("idle cleanup inspection", () => { expect(await readIdleLocalWork()).toBe("none"); markIdleStartupComplete({ scheduledBackups: true }); expect(await readIdleLocalWork()).toBe("present"); + expect(await readIdleLocalWork({ backupCheckpoint: true })).toBe("none"); markIdleStartupComplete({ scheduledBackups: false }); await fs.mkdir(custom); await fs.writeFile(path.join(custom, "receipt.tmp"), "incomplete"); expect(await readIdleLocalWork()).toBe("present"); + expect(await readIdleLocalWork({ backupCheckpoint: true })).toBe("present"); await fs.rm(custom, { recursive: true }); expect(await readIdleLocalWork()).toBe("none"); }); diff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.ts index 832673aca58..66a023dd9b9 100644 --- a/server/src/__tests__/idle-sleep-safety.test.ts +++ b/server/src/__tests__/idle-sleep-safety.test.ts @@ -1,4 +1,4 @@ -import { beginIdleTrackedWork } from "../services/task-admission.js"; +import { beginIdleTrackedWork, idleWorkSnapshot } from "../services/task-admission.js"; import { randomUUID } from "node:crypto"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -22,6 +22,73 @@ const ownerId = "d0b833f4-4098-42de-8420-1907f3aa4895"; const owned = () => ({ ...held(), ownerId }); const emptyLocal = async () => "none" as const; +describe("idle backup checkpoint", () => { + const database = (blocked = false) => { + const transaction = vi.fn(async (run: (tx: unknown) => Promise) => run({ execute: async () => [{ blocked }] })); + return { db: { transaction } as unknown as Db, transaction }; + }; + const backup = (work: () => Promise = async () => true) => vi.fn(async () => { + const finish = beginIdleTrackedWork(); + try { return await work(); } finally { finish(); } + }); + it("backs up after durable inspection and rechecks local work before authorizing sleep", async () => { + const { db, transaction } = database(); + const local = vi.fn(emptyLocal); + const checkpoint = backup(async () => { + expect(transaction).toHaveBeenCalledOnce(); + expect(idleWorkSnapshot().active).toBe(1); + return true; + }); + expect(await readIdleSleepSafety(db, owned, () => now, ownerId, local, undefined, checkpoint)).toEqual(none); + expect(checkpoint).toHaveBeenCalledOnce(); + expect(local).toHaveBeenCalledTimes(2); + }); + it.each(["present", "unknown"] as const)("avoids DB probes and backups for known local %s work", async state => { + const { db, transaction } = database(); + const checkpoint = backup(); + const plugins = vi.fn(); + expect(await readIdleSleepSafety(db, owned, () => now, ownerId, async () => state, plugins, checkpoint)) + .toEqual({ version: 1, backgroundWork: state }); + expect(transaction).not.toHaveBeenCalled(); + expect(plugins).not.toHaveBeenCalled(); + expect(checkpoint).not.toHaveBeenCalled(); + }); + it("does not start a backup for durable work or an unowned hold", async () => { + const checkpoint = backup(); + expect(await readIdleSleepSafety(database(true).db, owned, () => now, ownerId, emptyLocal, undefined, checkpoint)).toEqual(present); + expect(await readIdleSleepSafety(database().db, owned, () => now, "stale", emptyLocal, undefined, checkpoint)).toEqual(unknown); + expect(checkpoint).not.toHaveBeenCalled(); + }); + it.each(["failed", "threw", "untracked", "concurrent", "expired", "replaced"])("keeps the instance awake when the backup is %s", async kind => { + let clock = now; + let status = owned(); + const work = async () => { + if (kind === "threw") throw new Error("private archive path"); + if (kind === "concurrent") { const finish = beginIdleTrackedWork(); finish(); } + if (kind === "expired") clock += 60_000; + if (kind === "replaced") status = { ...status, ownerId: "replacement" }; + return kind !== "failed"; + }; + const checkpoint = kind === "untracked" ? work : backup(work); + expect(await readIdleSleepSafety(database().db, () => status, () => clock, ownerId, emptyLocal, undefined, checkpoint)).toEqual(unknown); + expect(idleWorkSnapshot().active).toBe(0); + }); + it("retains actual backup work after the owning hold expires", async () => { + let finishBackup!: () => void; + const pending = new Promise(resolve => { finishBackup = resolve; }); + const checkpoint = backup(async () => { await pending; return true; }); + let clock = now; + const scan = readIdleSleepSafety(database().db, owned, () => clock, ownerId, emptyLocal, undefined, checkpoint); + await vi.waitFor(() => expect(checkpoint).toHaveBeenCalledOnce()); + clock += 60_000; + expect(idleWorkSnapshot().active).toBe(1); + expect(await readIdleSleepSafety(database().db, owned, () => clock, ownerId, emptyLocal)).toEqual(unknown); + finishBackup(); + expect(await scan).toEqual(unknown); + expect(idleWorkSnapshot().active).toBe(0); + }); +}); + describe("idle sleep safety failure boundaries", () => { it.each([ @@ -36,7 +103,7 @@ describe("idle sleep safety failure boundaries", () => { it("fails closed when the database is unavailable", async () => { const db = { transaction: vi.fn().mockRejectedValue(new Error("private connection detail")) } as unknown as Db; - expect(await readIdleSleepSafety(db, held, () => now)).toEqual(unknown); + expect(await readIdleSleepSafety(db, held, () => now, undefined, emptyLocal)).toEqual(unknown); }); it.each([ @@ -46,14 +113,14 @@ describe("idle sleep safety failure boundaries", () => { const getStatus = vi.fn().mockReturnValueOnce(held()).mockReturnValue({ ...held(), ...change }); const execute = vi.fn().mockResolvedValue([{ blocked: true }]); const transaction = vi.fn(async (fn: (tx: unknown) => Promise) => fn({ execute })); - expect(await readIdleSleepSafety({ transaction } as unknown as Db, getStatus, () => now)).toEqual(unknown); + expect(await readIdleSleepSafety({ transaction } as unknown as Db, getStatus, () => now, undefined, emptyLocal)).toEqual(unknown); expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" }); }); it("reports persisted work when the admission hold remains unchanged", async () => { const execute = vi.fn().mockResolvedValue([{ blocked: true }]); const transaction = vi.fn(async (fn: (tx: unknown) => Promise) => fn({ execute })); - expect(await readIdleSleepSafety({ transaction } as unknown as Db, held, () => now)).toEqual(present); + expect(await readIdleSleepSafety({ transaction } as unknown as Db, held, () => now, undefined, emptyLocal)).toEqual(present); }); }); diff --git a/server/src/__tests__/instance-settings-routes.test.ts b/server/src/__tests__/instance-settings-routes.test.ts index ffee8171aaf..5d6cf9d0345 100644 --- a/server/src/__tests__/instance-settings-routes.test.ts +++ b/server/src/__tests__/instance-settings-routes.test.ts @@ -77,7 +77,7 @@ describe("instance settings routes", () => { return { errorHandler, instanceSettingsRoutes }; }); - function createApp(actor: any, pluginWorkers?: PluginWorkerManager) { + function createApp(actor: any, pluginWorkers?: PluginWorkerManager, prepareBackup?: () => Promise) { const { errorHandler, instanceSettingsRoutes } = routeModules.value; const app = express(); app.use(express.json()); @@ -85,7 +85,7 @@ describe("instance settings routes", () => { req.actor = actor; next(); }); - app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers)); + app.use("/api", instanceSettingsRoutes(mockDb as any, pluginWorkers, prepareBackup)); app.use(errorHandler); return app; } @@ -973,7 +973,7 @@ describe("instance settings routes", () => { await request(createApp(adminActor)).post("/api/instance/task-drain").send({ purpose: "idle", ttlMs: 60_000 }).expect(200); expect(mockHeartbeatService.computeTaskDrain).toHaveBeenCalledWith({ purpose: "idle", ttlMs: 60_000 }); await request(createApp(adminActor)).get(`/api/instance/task-drain?idleSleepSafety=1&ownerId=${ownerId}`).expect(200); - expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, undefined, undefined); + expect(mockReadIdleSleepSafety).toHaveBeenLastCalledWith(mockDb, expect.any(Function), Date.now, ownerId, expect.any(Function), undefined, undefined); }); it.each([{}, { ttlMs: null }, { ttlMs: 1 }, { ttlMs: 300_001 }])("rejects an unbounded or invalid idle TTL: %j", async body => { @@ -1000,7 +1000,7 @@ describe("instance settings routes", () => { const res = await request(createApp(adminActor)).get("/api/instance/task-drain?idleSleepSafety=1"); expect(res.status).toBe(200); expect(res.body).toEqual({ ...idleStatus, idleSleepSafety: report }); - expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, undefined, undefined); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, undefined, expect.any(Function), undefined, undefined); expect(mockReadIdleSleepSafety.mock.calls[0][1]()).toEqual(idleStatus); }); @@ -1028,6 +1028,42 @@ describe("instance settings routes", () => { expect(release).toHaveBeenCalledOnce(); }); + it("passes the operator backup hook only through the protected safety report", async () => { + const backup = vi.fn(async () => true); + mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus); + const app = createApp(adminActor, undefined, backup); + await request(app).get("/api/instance/task-drain").expect(200); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + await request(app).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, + "fixture-owner", expect.any(Function), undefined, backup); + mockReadIdleSleepSafety.mockClear(); + await request(createApp(nonAdminActor, undefined, backup)).get("/api/instance/task-drain?idleSleepSafety=1").expect(403); + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + expect(backup).not.toHaveBeenCalled(); + }); + + it("reserves managed checkpoint creation for verified Cloud control actors", async () => { + const backup = vi.fn(async () => true); + process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN = "fixture-managed-signal"; + mockHeartbeatService.getTaskDrainStatus.mockReturnValue(idleStatus); + try { + for (const source of ["session", "cloud_tenant", "local_implicit"]) { + const tenantAdmin = createApp({ ...adminActor, source }, undefined, backup); + await request(tenantAdmin).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner") + .set("x-paperclip-cloud-control", "unverified-header").expect(403); + } + expect(mockReadIdleSleepSafety).not.toHaveBeenCalled(); + expect(backup).not.toHaveBeenCalled(); + const control = createApp({ ...adminActor, source: "cloud_control" }, undefined, backup); + await request(control).get("/api/instance/task-drain?idleSleepSafety=1&ownerId=fixture-owner").expect(200); + expect(mockReadIdleSleepSafety).toHaveBeenCalledWith(mockDb, expect.any(Function), Date.now, + "fixture-owner", expect.any(Function), undefined, backup); + } finally { + delete process.env.PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN; + } + }); + it.each([ ["company member", nonAdminActor], ["agent", { type: "agent", agentId: "agent-1", companyId: "company-1" }], diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 7b89d44dd9a..449d8d5b5dd 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -1,9 +1,12 @@ import { idleWorkSnapshot } from "../services/task-admission.js"; import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import * as idleLocalWork from "../services/idle-local-work.js"; +import { idleBackupWakeMarker, markIdleBackupWakeRequired } from "../services/idle-database-backup.js"; import { reconcileBuiltInAgentsOnStartup, reconcileCodexLocalManagedHomesOnStartup, reconcilePersistedRuntimeServicesOnStartup } from "../services/index.js"; import { runDatabaseBackup } from "@paperclipai/db"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { gzipSync } from "node:zlib"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -441,6 +444,69 @@ describe("startServer feedback export wiring", () => { process.env.BETTER_AUTH_SECRET = "test-secret"; }); + it.each([false, true].flatMap(checkpointMode => [false, true].map(fails => ({ checkpointMode, fails })))) + ("finishes restart backup before idle eligibility (checkpoint=$checkpointMode, failure=$fails)", async ({ checkpointMode, fails }) => { + const directory = mkdtempSync(path.join(tmpdir(), "startup-idle-backup-")); + const previousFlag = process.env.PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED; + process.env.PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED = checkpointMode ? "1" : "0"; + loadConfigMock.mockReturnValue(buildTestConfig({ databaseBackupEnabled: true, databaseBackupDir: directory })); + const eligible = vi.spyOn(idleLocalWork, "markIdleStartupComplete"); + let retry: (() => void) | undefined; + const timers = vi.spyOn(globalThis, "setInterval").mockImplementation(((callback: () => void, ms: number) => { + if (ms === 60 * 60 * 1000) retry = callback; + return 1 as unknown as ReturnType; + }) as typeof setInterval); + let finish!: () => void; + const pending = new Promise(resolve => { finish = resolve; }); + const writeBackup = async (options: Parameters[0]) => { + const backupFile = path.join(directory, "paperclip-restart.sql.gz"); + const bytes = gzipSync("SELECT 'restart recovery';\n"); + writeFileSync(backupFile, bytes); + expect(options.retention).toEqual({ dailyDays: 7, weeklyWeeks: 4, monthlyMonths: 1 }); + expect(options.verifyBeforePrune).toEqual(expect.any(Function)); + await options.verifyBeforePrune!(backupFile); + return { backupFile, sizeBytes: bytes.length, prunedCount: 0 }; + }; + vi.mocked(runDatabaseBackup).mockImplementationOnce(async options => { + await pending; + if (fails) throw new Error("fixture restart backup failed"); + return writeBackup(options); + }); + let starting: ReturnType | undefined; + try { + await markIdleBackupWakeRequired(directory); + starting = startServer(); + await vi.waitFor(() => expect(runDatabaseBackup).toHaveBeenCalledOnce()); + expect(eligible).not.toHaveBeenCalled(); + expect(idleWorkSnapshot().active).toBeGreaterThan(0); + finish(); + await starting; + expect(eligible).toHaveBeenCalledWith({ scheduledBackups: true }); + const options = createAppMock.mock.calls[0]?.[1] as unknown as { prepareIdleDatabaseBackup?: unknown }; + expect(Boolean(options.prepareIdleDatabaseBackup)).toBe(checkpointMode); + expect(existsSync(idleBackupWakeMarker(directory))).toBe(fails); + if (fails) { + vi.mocked(runDatabaseBackup).mockImplementationOnce(writeBackup); + expect(retry).toEqual(expect.any(Function)); + const activeBeforeRetry = idleWorkSnapshot().active; + retry!(); + await vi.waitFor(() => { + expect(existsSync(idleBackupWakeMarker(directory))).toBe(false); + // Marker removal precedes directory fsync and release of the work receipt. + expect(idleWorkSnapshot().active).toBe(activeBeforeRetry); + }); + expect(runDatabaseBackup).toHaveBeenCalledTimes(2); + } + } finally { + finish(); + await starting; + timers.mockRestore(); eligible.mockRestore(); + if (previousFlag === undefined) delete process.env.PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED; + else process.env.PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED = previousFlag; + rmSync(directory, { recursive: true, force: true }); + } + }); + it.each(([ ["built-in agents", reconcileBuiltInAgentsOnStartup], ["managed homes", reconcileCodexLocalManagedHomesOnStartup], diff --git a/server/src/app.ts b/server/src/app.ts index db931861787..bda8217716a 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -491,6 +491,7 @@ export async function createApp( }): Promise; }; databaseBackupService?: InstanceDatabaseBackupService; + prepareIdleDatabaseBackup?: () => Promise; databaseBackupHealth?: InspectDatabaseBackupHealthOptions; deploymentMode: DeploymentMode; deploymentExposure: DeploymentExposure; @@ -836,7 +837,7 @@ export async function createApp( api.use(resourceMembershipRoutes(db)); api.use(primaryAgentRoutes(db)); api.use(inboxDismissalRoutes(db)); - api.use(instanceSettingsRoutes(db, workerManager)); + api.use(instanceSettingsRoutes(db, workerManager, opts.prepareIdleDatabaseBackup)); if (opts.databaseBackupService) { api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService)); } diff --git a/server/src/index.ts b/server/src/index.ts index 0c79e723e62..ce72014df01 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,5 +1,6 @@ import { isIdleTaskDrainActive, beginIdleTrackedWork, trackIdleWork } from "./services/task-admission.js"; import { markIdleStartupComplete } from "./services/idle-local-work.js"; +import { clearIdleBackupWakeRequired, idleBackupWakeMarker, markIdleBackupWakeRequired, verifyIdleDatabaseBackup } from "./services/idle-database-backup.js"; import { cloudWarmStandbyServerOptions } from "./middleware/cloud-warm-standby.js"; import { createCloudWarmStandby } from "./services/cloud-warm-standby.js"; import { subscribeAllCompanyLiveEvents } from "./services/live-events.js"; @@ -819,6 +820,8 @@ async function startServerWithDatabaseTeardown( resolve(config.databaseBackupDir, "..", "db-backup-to-s3.failure"), ]; let databaseBackupInFlight = false; + const idleDatabaseBackupEnabled = config.databaseBackupEnabled + && process.env.PAPERCLIP_DB_BACKUP_IDLE_CHECKPOINT_ENABLED === "1"; const runServerDatabaseBackup = async ( trigger: InstanceDatabaseBackupTrigger, ): Promise => { @@ -835,19 +838,25 @@ async function startServerWithDatabaseTeardown( const finishIdleBackup = beginIdleTrackedWork(); const startedAt = new Date(); const startedAtMs = Date.now(); - const label = trigger === "scheduled" ? "Automatic" : "Manual"; + const label = trigger === "manual" ? "Manual" : trigger === "idle" ? "Pre-sleep" : "Automatic"; try { + if (trigger === "idle") await markIdleBackupWakeRequired(config.databaseBackupDir); logger.info({ backupDir: config.databaseBackupDir, trigger }, `${label} database backup starting`); // Read retention from Instance Settings (DB) so changes take effect without restart. const generalSettings = await backupSettingsSvc.getGeneral(); const retention = generalSettings.backupRetention; + const verifyCheckpoint = trigger === "idle" || existsSync(idleBackupWakeMarker(config.databaseBackupDir)); const result = await runDatabaseBackup({ connectionString: activeDatabaseConnectionString, backupDir: config.databaseBackupDir, retention, filenamePrefix: "paperclip", + verifyBeforePrune: verifyCheckpoint ? verifyIdleDatabaseBackup : undefined, }); + if (verifyCheckpoint) { + if (trigger !== "idle") await clearIdleBackupWakeRequired(config.databaseBackupDir); + } const finishedAt = new Date(); const response: InstanceDatabaseBackupRunResult = { ...result, @@ -892,6 +901,9 @@ async function startServerWithDatabaseTeardown( // self-hosted: createApp falls back to its built-in kubernetes-only default. const managedPluginAutoInstall = managedConfig?.plugins.autoInstall ?? null; const app = await createApp(db as any, { + prepareIdleDatabaseBackup: idleDatabaseBackupEnabled + ? async () => (await runServerDatabaseBackup("idle")) !== null + : undefined, cloudWarmStandby: isWarmStandby, uiMode, serverPort: listenPort, @@ -1872,6 +1884,12 @@ async function startServerWithDatabaseTeardown( }, "Automatic database backups enabled", ); + // The final pre-sleep dump remains on the volume throughout sleep. Take + // a fresh backup on restart before idle eligibility is enabled, including + // after rollback of the opt-in flag. Failure retains the marker for retry. + if (!isWarmStandby() && existsSync(idleBackupWakeMarker(config.databaseBackupDir))) { + await runServerDatabaseBackup("scheduled").catch(() => {}); + } setInterval(() => { if (isWarmStandby() || isIdleTaskDrainActive()) return; void runServerDatabaseBackup("scheduled").catch(() => { diff --git a/server/src/routes/instance-database-backups.ts b/server/src/routes/instance-database-backups.ts index f05e3361c2c..9635a1cb31b 100644 --- a/server/src/routes/instance-database-backups.ts +++ b/server/src/routes/instance-database-backups.ts @@ -4,7 +4,7 @@ import { forbidden } from "../errors.js"; import { isCloudManagedInstance } from "../services/cloud-instance.js"; import { assertInstanceAdmin } from "./authz.js"; -export type InstanceDatabaseBackupTrigger = "manual" | "scheduled"; +export type InstanceDatabaseBackupTrigger = "manual" | "scheduled" | "idle"; export type InstanceDatabaseBackupRunResult = RunDatabaseBackupResult & { trigger: InstanceDatabaseBackupTrigger; diff --git a/server/src/routes/instance-settings.ts b/server/src/routes/instance-settings.ts index ef578bb7fda..0c126b729e1 100644 --- a/server/src/routes/instance-settings.ts +++ b/server/src/routes/instance-settings.ts @@ -15,6 +15,7 @@ import { } from "../services/cloud-instance.js"; import { getHiddenSettings } from "../services/settings-visibility.js"; import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; +import { readIdleLocalWork } from "../services/idle-local-work.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; import { validate } from "../middleware/validate.js"; import { logger } from "../middleware/logger.js"; @@ -120,7 +121,7 @@ function withTaskDrainTransition(run: () => Promise): Promise { return turn; } -export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManager) { +export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManager, prepareIdleDatabaseBackup?: () => Promise) { const router = Router(); const svc = instanceSettingsService(db); const environments = environmentService(db); @@ -304,9 +305,18 @@ export function instanceSettingsRoutes(db: Db, pluginWorkers?: PluginWorkerManag // The report covers every company in this process. Ordinary company // members may read process counters, but not instance-wide work state. assertCanManageInstanceSettings(req); + // A checkpoint performs platform-owned backup work. Tenant owners can + // have instance-admin elevation, but that must not grant the manual + // backup authority which managed instances deliberately withhold. + if (prepareIdleDatabaseBackup && isCloudManagedInstance() && req.actor.source !== "cloud_control") { + throw forbidden("Backup checkpoints are platform-managed on cloud-managed instances", { + code: "database_backups_platform_managed", + }); + } const idleSleepSafety = await readIdleSleepSafety(db, () => heartbeat.getTaskDrainStatus(), Date.now, - typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, undefined, - pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers)); + typeof req.query.ownerId === "string" ? req.query.ownerId : undefined, + () => readIdleLocalWork({ backupCheckpoint: Boolean(prepareIdleDatabaseBackup) }), + pluginWorkers?.inspectIdleSleep?.bind(pluginWorkers), prepareIdleDatabaseBackup); res.json({ ...heartbeat.getTaskDrainStatus(), idleSleepSafety }); return; } diff --git a/server/src/services/idle-database-backup.ts b/server/src/services/idle-database-backup.ts new file mode 100644 index 00000000000..271750d118c --- /dev/null +++ b/server/src/services/idle-database-backup.ts @@ -0,0 +1,52 @@ +import { constants, createReadStream } from "node:fs"; +import { mkdir, open, unlink } from "node:fs/promises"; +import path from "node:path"; +import { Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { createGunzip } from "node:zlib"; + +// The marker survives process replacement on persistent instance storage. +// It contains no credentials or tenant data, and never substitutes for a dump. +export const idleBackupWakeMarker = (directory: string) => path.join(directory, ".idle-backup-wake-required"); + +async function syncDirectory(directory: string): Promise { + const handle = await open(directory, "r"); + try { await handle.sync(); } finally { await handle.close(); } +} + +export async function markIdleBackupWakeRequired(directory: string): Promise { + await mkdir(directory, { recursive: true }); + const marker = await open(idleBackupWakeMarker(directory), constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); + try { + await marker.writeFile("Take a fresh database backup after this process restarts.\n"); + await marker.sync(); + } finally { await marker.close(); } + await syncDirectory(directory); +} + +/** Read the entire archive to verify the gzip trailer, then persist its bytes + * and directory entry before allowing compute to stop. The backup runner + * retains its in-flight receipt until this function settles. */ +export async function verifyIdleDatabaseBackup(backupFile: string): Promise { + const handle = await open(backupFile, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size === 0) throw new Error("Idle backup archive is empty or invalid"); + let bytes = 0; + await pipeline( + createReadStream(backupFile, { fd: handle.fd, autoClose: false }), + createGunzip(), + new Writable({ write(chunk, _encoding, done) { bytes += chunk.length; done(); } }), + ); + if (bytes === 0) throw new Error("Idle backup archive contains no SQL"); + await handle.sync(); + } finally { await handle.close(); } + await syncDirectory(path.dirname(backupFile)); +} + +export async function clearIdleBackupWakeRequired(directory: string): Promise { + await unlink(idleBackupWakeMarker(directory)).catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }); + await syncDirectory(directory); +} diff --git a/server/src/services/idle-local-work.ts b/server/src/services/idle-local-work.ts index 1095c298f80..2d54ad90309 100644 --- a/server/src/services/idle-local-work.ts +++ b/server/src/services/idle-local-work.ts @@ -33,9 +33,11 @@ export async function inspectIdleSpool(directory: string): Promise { +export async function readIdleLocalWork(options: { backupCheckpoint?: boolean } = {}): Promise { if (!startupComplete || !ingressTracked) return "unknown"; - if (scheduledBackups || idleWorkSnapshot().active !== 0) return "present"; + // This exemption belongs only to a scan which will finish a new backup + // under the same admission hold before it can return an empty report. + if ((scheduledBackups && !options.backupCheckpoint) || idleWorkSnapshot().active !== 0) return "present"; const directories = new Set([...spoolDirectories, idleAccountingSpoolPath(), idleOrphanSpoolPath()]); const results = await Promise.all([...directories].map(inspectIdleSpool)); if (results.includes("unknown")) return "unknown"; diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index ad24e3037d1..a8c7a80e13b 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -97,12 +97,17 @@ export async function readIdleSleepSafety( ownerId?: string, inspectLocalWork: () => Promise = readIdleLocalWork, inspectPlugins?: InspectIdlePlugins, + prepareBackup?: () => Promise, ): Promise { const unknown: IdleSleepSafety = { version: 1, backgroundWork: "unknown" }; const before = getDrainStatus(); const localBefore = idleWorkSnapshot(); if (!sameQuietHold(before, before, now())) return unknown; try { + // Reject known local blockers before querying (and potentially waking) + // the tenant database. Keep the final inspection below as a second fence. + const localFirst = await inspectLocalWork(); + if (localFirst !== "none") return { version: 1, backgroundWork: localFirst }; let quietPlugins: string[] = []; if (inspectPlugins) { if (!ownerId || ownerId !== before.ownerId || before.expiresAt === null || localBefore.active !== 0) return unknown; @@ -124,11 +129,21 @@ export async function readIdleSleepSafety( if (blocked === undefined || !sameQuietHold(before, getDrainStatus(), now())) return unknown; if (blocked) return { version: 1, backgroundWork: "present" }; if (!ownerId || ownerId !== before.ownerId || before.expiresAt === null || localBefore.active !== 0) return unknown; + let expectedGeneration = localBefore.generation; + if (prepareBackup) { + if (!sameQuietHold(before, getDrainStatus(), now()) || + idleWorkSnapshot().generation !== expectedGeneration || idleWorkSnapshot().active !== 0) return unknown; + // The runner owns exactly one tracked-work receipt through dump, + // archive verification and fsync. Its start/finish are the only work + // allowed during this scan. Any concurrent work invalidates the dump. + if (!(await prepareBackup())) return unknown; + expectedGeneration += 2; + } const local = await inspectLocalWork(); const after = getDrainStatus(); const localAfter = idleWorkSnapshot(); if (!sameQuietHold(before, after, now()) || localAfter.active !== 0 || - localBefore.generation !== localAfter.generation) return unknown; + expectedGeneration !== localAfter.generation) return unknown; return { version: 1, backgroundWork: local }; } catch {