Preserve recovery backups and use database lease time

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/native-runtime/native-controller-recovery.integration.test.ts

@@ -3,7 +3,7 @@ import { mkdtemp, rm } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { eq } from "drizzle-orm";
-import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import { agentTaskSessions, agents, companies, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults } from "@paperclipai/db";
 import { DurablePrpControlPlane, parseNativeExecutionInput, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js";
 import { startRecoveryPostgres, recoveryPostgresSupport } from "../../__tests__/helpers/recovery-postgres.js";
@@ -92,6 +92,18 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d
     expect(await resolveNativeControllerRecovery(db, { ...fixture.scope, currentRunId })).toEqual({ kind: "waiting_for_owner", deadline });
   });
 
+  it("uses the database clock when an import finds an expired owner", async () => {
+    const fixture = await seed();
+    const now = Date.now();
+    await db.update(nativeRunFinalizations).set({ leaseOwner: "expired-owner", leaseExpiresAt: new Date(now - 60_000) })
+      .where(eq(nativeRunFinalizations.runId, fixture.scope.runId));
+    vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(now - 3_600_000);
+    try {
+      expect(await importNativeControllerRecovery(db, { ...fixture.scope, expectedConfigDigest: compatibility.configDigest,
+        supervisor: fixture.supervisor })).toEqual({ kind: "missing_evidence", reason: "normal_ownership_recovery_required" });
+    } finally { vi.useRealTimers(); }
+  });
+
   it("holds mismatched snapshots and runner bindings instead of falling back to fresh execution", async () => {
     const fixture = await seed(), currentRunId = randomUUID();
     const metadata = createNativeControllerRecovery(fixture.snapshot, compatibility, { runId: fixture.scope.runId,

server/src/services/native-runtime/native-controller-recovery.ts

@@ -245,7 +245,7 @@ export async function importNativeControllerRecovery(db: Db, input: {
     const [clock] = await tx.execute<{ now: Date }>(sql`select clock_timestamp() as now`);
     const liveOwner = owner.leaseOwner && (!owner.leaseExpiresAt || owner.leaseExpiresAt > new Date(clock.now));
     if (input.offline ? liveOwner : owner.leaseOwner || task.executionRunId) {
-      return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date()
+      return owner.leaseExpiresAt && owner.leaseExpiresAt > new Date(clock.now)
         ? { kind: "waiting_for_owner" as const, deadline: owner.leaseExpiresAt }
         : { kind: "missing_evidence" as const, reason: "normal_ownership_recovery_required" };
     }

server/src/services/native-runtime/native-harness-backup-publication.test.ts

@@ -53,6 +53,15 @@ describe("native harness backup publication", () => {
     expect(await fs.readdir(backupRoot)).toEqual(["current"]);
     expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]);
   });
+  it("restores the old current directory when the pending rename fails", async () => {
+    const input = await fixture(); const changes: unknown[] = [];
+    unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change));
+    await fs.rm(input.pendingRoot, { recursive: true });
+    await expect(publishNativeHarnessBackup(input)).rejects.toMatchObject({ code: "ENOENT" });
+    expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("saved");
+    expect(changes).toEqual([]);
+  });
+
   it("serves an unrelated request while a retired backup is being removed", async () => {
     const input = await fixture();
     for (let index = 0; index < 40; index++) {

server/src/services/native-runtime/native-harness-backup-storage.ts

@@ -1,6 +1,7 @@
+import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js";
 import { createCipheriv, createDecipheriv, createHash, randomBytes, randomUUID } from "node:crypto";
 import { createReadStream, createWriteStream } from "node:fs";
-import { mkdtemp, readFile, rename } from "node:fs/promises";
+import { mkdtemp, readFile } from "node:fs/promises";
 import { join } from "node:path";
 import os from "node:os";
 import { pipeline } from "node:stream/promises";
@@ -199,8 +200,8 @@ export async function restoreDurableNativeHarnessBackup(input: {
     const pending = join(input.backupRoot, `.durable-${randomUUID()}`);
     try {
       await cp(restored, pending, { recursive: true, preserveTimestamps: true, verbatimSymlinks: true, errorOnExist: true, force: false });
-      await removeNativeHarnessBackup(current);
+      await publishNativeHarnessBackup({ db: input.db, companyId: input.companyId, scopeDigest: input.scopeDigest,
-      await rename(pending, current);
+        backupRoot: input.backupRoot, pendingRoot: pending });
     } finally { await removeNativeHarnessBackup(pending); }
     return true;
   } finally { key.fill(0); await removeNativeHarnessBackup(scratch); }