From e9aae9cfb938140fc727065986e7878f6f648c9a Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 01:25:45 -0600 Subject: [PATCH] Preserve recovery backups and use database lease time Co-Authored-By: Paperclip --- .../native-controller-recovery.integration.test.ts | 14 +++++++++++++- .../native-runtime/native-controller-recovery.ts | 2 +- .../native-harness-backup-publication.test.ts | 9 +++++++++ .../native-harness-backup-storage.ts | 7 ++++--- 4 files changed, 27 insertions(+), 5 deletions(-) diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index c4315273e81..d1e5a5a475f 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { eq } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agentTaskSessions, agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db"; import { DurablePrpControlPlane, parseNativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js"; @@ -92,6 +92,18 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId })).toEqual({ kind: "waiting_for_owner", deadline }); }); + it("uses the database clock when an import finds an expired owner", async () => { + const fixture = await seed(); + const now = Date.now(); + await db.update(nativeRunFinalizations).set({ leaseOwner: "expired-owner", leaseExpiresAt: new Date(now - 60_000) }) + .where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(now - 3_600_000); + try { + expect(await importNativeControllerRecovery(db, { ...fixture.scope, expectedConfigDigest: compatibility.configDigest, + supervisor: fixture.supervisor })).toEqual({ kind: "missing_evidence", reason: "normal_ownership_recovery_required" }); + } finally { vi.useRealTimers(); } + }); + it("holds mismatched snapshots and runner bindings instead of falling back to fresh execution", async () => { const fixture = await seed(), currentRunId = randomUUID(); const metadata = createNativeControllerRecovery(fixture.snapshot, compatibility, { runId: fixture.scope.runId, diff --git a/server/src/services/native-runtime/native-controller-recovery.ts b/server/src/services/native-runtime/native-controller-recovery.ts index ffb11b9f1d7..64e9f6d1d23 100644 --- a/server/src/services/native-runtime/native-controller-recovery.ts +++ b/server/src/services/native-runtime/native-controller-recovery.ts @@ -245,7 +245,7 @@ export async function importNativeControllerRecovery(db: Db, input: { const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`); const liveOwner = owner.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt > new Date(clock.now)); if (input.offline ? liveOwner : owner.leaseOwner || task.executionRunId) { - return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date() + return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date(clock.now) ? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt } : { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" }; } diff --git a/server/src/services/native-runtime/native-harness-backup-publication.test.ts b/server/src/services/native-runtime/native-harness-backup-publication.test.ts index df1029f55f9..66dcc70cf00 100644 --- a/server/src/services/native-runtime/native-harness-backup-publication.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-publication.test.ts @@ -53,6 +53,15 @@ describe("native harness backup publication", () => { expect(await fs.readdir(backupRoot)).toEqual(["current"]); expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]); }); + it("restores the old current directory when the pending rename fails", async () => { + const input = await fixture(); const changes: unknown[] = []; + unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + await fs.rm(input.pendingRoot, { recursive: true }); + await expect(publishNativeHarnessBackup(input)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("saved"); + expect(changes).toEqual([]); + }); + it("serves an unrelated request while a retired backup is being removed", async () => { const input = await fixture(); for (let index = 0; index < 40; index++) { diff --git a/server/src/services/native-runtime/native-harness-backup-storage.ts b/server/src/services/native-runtime/native-harness-backup-storage.ts index 85d6a8a7de4..5bf06d58037 100644 --- a/server/src/services/native-runtime/native-harness-backup-storage.ts +++ b/server/src/services/native-runtime/native-harness-backup-storage.ts @@ -1,6 +1,7 @@ +import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; -import { mkdtemp, readFile, rename } from "node:fs/promises"; +import { mkdtemp, readFile } from "node:fs/promises"; import { join } from "node:path"; import os from "node:os"; import { pipeline } from "node:stream/promises"; @@ -199,8 +200,8 @@ export async function restoreDurableNativeHarnessBackup(input: { const pending = join(input.backupRoot, `.durable-${randomUUID()}`); try { await cp(restored, pending, { recursive: true, preserveTimestamps: true, verbatimSymlinks: true, errorOnExist: true, force: false }); - await removeNativeHarnessBackup(current); - await rename(pending, current); + await publishNativeHarnessBackup({ db: input.db, companyId: input.companyId, scopeDigest: input.scopeDigest, + backupRoot: input.backupRoot, pendingRoot: pending }); } finally { await removeNativeHarnessBackup(pending); } return true; } finally { key.fill(0); await removeNativeHarnessBackup(scratch); }