diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 65cf2148a2b..6d50ff984fd 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2074,7 +2074,10 @@ backup. Paperclip restores that backup into the new sandbox only when all of these are true: the backup's source lease is expired or failed with a provider `destroyed` termination receipt, the source lease's backup stamp names this exact backup manifest, and no provider event for the session was recorded after -the backup completed. Any other state is ambiguous. The run then fails closed +the backup completed. Normal warm handoffs can leave multiple lease records +for that allocation. Restore requires one final owner and the existing matching +predecessor/successor receipts for every earlier owner. Unrelated, live, or +incompletely linked owners remain ambiguous. The run then fails closed with `runner_harness_state_mismatch: backup_without_reusable_lease`. A per-turn sandbox turn can finish through recovery without a Runner diff --git a/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts b/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts index f725280f4ba..7a8ed9f1322 100644 --- a/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts +++ b/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts @@ -1,7 +1,8 @@ import { randomUUID } from "node:crypto"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { agents, companies, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; +import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issues } from "@paperclipai/db"; import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; +import { createReusableLeaseHandoff } from "../environment-lease-handoff.js"; import { retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; const describeDb = recoveryPostgresSupport.supported ? describe : describe.skip; @@ -19,7 +20,8 @@ describeDb("retired native backup source authority", () => { const completedAt = new Date("2026-11-05T05:02:50.000Z"); async function fixture(options: { termination?: "destroyed" | "stopped" | null; status?: string; stampSha?: string; - providerEventAt?: Date; secondLease?: boolean } = {}) { + providerEventAt?: Date; secondLease?: boolean; handoffs?: number; + handoffFault?: "missing_receipt" | "changed_receipt" | "active_predecessor" | "missing_predecessor" } = {}) { const companyId = randomUUID(), agentId = randomUUID(), sessionId = randomUUID(), runnerInstanceId = randomUUID(); const sourceRunId = randomUUID(), currentRunId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Retired source", issuePrefix: `R${companyId.slice(0, 5)}` }); @@ -37,7 +39,43 @@ describeDb("retired native backup source authority", () => { runId: sourceRunId, leaseId: id, provider: "daytona", providerLeaseId, state: options.termination ?? "destroyed", confirmedAt: new Date().toISOString() } }), } }); - await db.insert(environmentLeases).values(lease(leaseId)); + const sourceLease = lease(leaseId); + if (options.handoffs) { + const environmentId = randomUUID(), issueId = randomUUID(), workspaceId = randomUUID(); + await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Retained session" }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId, sourceIssueId: issueId, + mode: "shared_workspace", strategyType: "project_primary", name: "Retained workspace" }); + const history: Array = []; + for (let index = 0; index <= options.handoffs; index++) { + const ownerRunId = index === options.handoffs ? sourceRunId : randomUUID(); + if (ownerRunId !== sourceRunId) await db.insert(heartbeatRuns).values({ id: ownerRunId, companyId, agentId, + status: "succeeded", runtimeMode: "native", nativeSessionId: sessionId, + invocationSource: "assignment", triggerDetail: "system" }); + const acquiredAt = new Date(completedAt.getTime() - (options.handoffs - index + 1) * 60_000); + const row = { ...sourceLease, id: index === options.handoffs ? leaseId : randomUUID(), + heartbeatRunId: ownerRunId, environmentId, issueId, executionWorkspaceId: workspaceId, acquiredAt, + leasePolicy: "reuse_by_environment", metadata: { ...sourceLease.metadata, agentId } }; + if (index < options.handoffs) { + const { nativeHarnessBackup: _backup, remoteExecutionTermination: _termination, ...metadata } = row.metadata; + history.push({ ...row, metadata }); + } else history.push(row); + } + for (let index = 1; index < history.length; index++) { + const previous = history[index - 1]!, current = history[index]!; + const handoff = createReusableLeaseHandoff(previous as typeof environmentLeases.$inferSelect, + current as typeof environmentLeases.$inferSelect); + previous.releasedAt = current.acquiredAt; + previous.metadata = { ...previous.metadata, reusableLeaseSuccessor: handoff }; + current.metadata = { ...current.metadata, reusableLeasePredecessor: handoff }; + } + if (options.handoffFault === "missing_receipt") delete history[0]!.metadata!.reusableLeaseSuccessor; + if (options.handoffFault === "changed_receipt") history[0]!.metadata!.reusableLeaseSuccessor = { + ...(history[0]!.metadata!.reusableLeaseSuccessor as Record), successorRunId: randomUUID() }; + if (options.handoffFault === "active_predecessor") history[0]!.status = "active"; + if (options.handoffFault === "missing_predecessor") history.shift(); + await db.insert(environmentLeases).values(history); + } else await db.insert(environmentLeases).values(sourceLease); if (options.secondLease) await db.insert(environmentLeases).values(lease(randomUUID())); // The backup turn's own provider events precede the checkpoint. await db.insert(heartbeatRunEvents).values({ companyId, runId: sourceRunId, agentId, seq: 1, eventType: "turn.started", @@ -54,6 +92,24 @@ describeDb("retired native backup source authority", () => { await expect(fixture()).resolves.toBe(true); }); + it.each([1, 3])("authorizes a destroyed allocation after %i verified warm lease handoffs", async (handoffs) => { + await expect(fixture({ handoffs })).resolves.toBe(true); + }); + + it.each(["missing_receipt", "changed_receipt", "active_predecessor", "missing_predecessor"] as const)( + "fails closed for a warm lease chain with %s", async (handoffFault) => { + await expect(fixture({ handoffs: 2, handoffFault })).resolves.toBe(false); + }); + + it.each([ + ["an unrelated owner", { secondLease: true }], + ["a stopped source", { termination: "stopped" as const }], + ["another checkpoint", { stampSha: `sha256:${"c".repeat(64)}` }], + ["later provider work", { providerEventAt: new Date(completedAt.getTime() + 60_000) }], + ])("keeps the backup restore held after warm handoffs with %s", async (_label, options) => { + await expect(fixture({ handoffs: 2, ...options })).resolves.toBe(false); + }); + it.each([ ["the provider only stopped the source", { termination: "stopped" as const }], ["no termination receipt exists", { termination: null }], diff --git a/server/src/services/native-runtime/native-retired-backup-source.ts b/server/src/services/native-runtime/native-retired-backup-source.ts index 865a2f43322..bd250afc76d 100644 --- a/server/src/services/native-runtime/native-retired-backup-source.ts +++ b/server/src/services/native-runtime/native-retired-backup-source.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { and, eq, gt, inArray, ne } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { retiredReusableLeasePredecessors } from "../environment-lease-handoff.js"; import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; import { readNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; @@ -42,9 +43,10 @@ export async function retiredSourceAuthorizesNativeBackupRestore(db: Db, input: )); const source = leases.filter(lease => lease.companyId === input.companyId && lease.providerLeaseId === input.manifest.sourceProviderLeaseId); - // One provider allocation maps to one host lease. Anything else is ambiguous. - if (source.length !== 1) return false; - const lease = source[0]!; + const tips = source.filter(lease => !Object.hasOwn(lease.metadata ?? {}, "reusableLeaseSuccessor")); + if (tips.length !== 1) return false; + const lease = tips[0]!; + if (!retiredReusableLeasePredecessors(lease, source.filter(row => row.id !== lease.id))) return false; const termination = lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined; // A released lease can still be selected for resume; only a retired one is final. if (!["expired", "failed"].includes(lease.status) || !hasRemoteTerminationReceipt(lease)