diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 537e43c5ad6..8f0c30290c2 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -3099,8 +3099,33 @@ backup source when all of these are true: the lease is `released` with policy `retain_on_failure`, it has a `workspaceRestoreRecovery` marker for its own run, it has a confirmed stop-and-retain receipt with state `stopped`, and no unsaved writer or pending native workspace export is bound to it. The backup -stamp, single-owner, and later-provider-work checks above still apply. A -stopped lease that is still `reuse_by_environment` remains ambiguous and fails +stamp, single-owner, and later-provider-work checks above still apply. + +Reuse also skips a warm sandbox while its previous run's workspace export is +still saving. A run that starts in that window gets a new sandbox, waits for +the export, and realizes the saved files there. By the time it prepares the +harness, the warm source is idle. An idle warm source also counts as a retired +backup source when all of these are true: the lease is `released` or +`retained` with policy `reuse_by_environment`, its own run is terminal and is +not the current run, no cleanup is in flight, and no unsaved writer or pending +native workspace export is bound to the allocation. The backup stamp, +single-owner, and later-provider-work checks above still apply. With no +provider work after the backup, the idle sandbox holds exactly the checkpointed +state, so the restore continues the same session and never runs two owners. +Before restoring, Paperclip claims the idle source with a compare-and-swap on +the checked row. The claim moves the source's reusable-lease scope into a +`reusableSandboxLeaseSuperseded` receipt, so reuse never resumes that older +state again. The next acquisition for the workspace treats the source as +obsolete and destroys it. If a run resumed or changed the source first, the +claim matches no row and the backup is not restored. + +The same proof covers a resumed lease that never held this session's harness +state, for example the new sandbox of a run that failed before it restored the +backup. When the resumed sandbox has no harness state at all and the backup +came from a different allocation, Paperclip restores the backup into it. State +saved by earlier releases recovers this way after deploy, with no migration. A +resumed allocation whose own state is missing, a source whose run is still +active, and a source with an export still pending remain ambiguous and fail closed. A Runner close can also fail to prove suspension after the control plane diff --git a/server/src/services/environments.ts b/server/src/services/environments.ts index dd49c30b90c..29b73fc9eb9 100644 --- a/server/src/services/environments.ts +++ b/server/src/services/environments.ts @@ -1645,6 +1645,10 @@ export function environmentService(db: Db) { "released", "retained", ]), + // A source whose harness backup was restored into another + // lease holds older session state. That claim and this + // handoff compete on this row, so only one of them wins. + sql`not (coalesce(${environmentLeases.metadata}, '{}'::jsonb) ? 'reusableSandboxLeaseSuperseded')`, ), ) .returning(); diff --git a/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts b/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts index a47ce7c7a04..86e473ea441 100644 --- a/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts +++ b/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts @@ -4,6 +4,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issues } from "@paperclipai/db"; import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { createReusableLeaseHandoff } from "../environment-lease-handoff.js"; +import { environmentService } from "../environments.js"; import { retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; const describeDb = recoveryPostgresSupport.supported ? describe : describe.skip; @@ -18,12 +19,14 @@ describeDb("retired native backup source authority", () => { afterAll(async () => { await database?.cleanup(); }); const manifestSha256 = `sha256:${"b".repeat(64)}`; + let lastSourceLeaseId = ""; const completedAt = new Date("2026-11-05T05:02:50.000Z"); async function fixture(options: { termination?: "destroyed" | "stopped" | null; status?: string; stampSha?: string; providerEventAt?: Date; secondLease?: boolean; handoffs?: number; handoffFault?: "missing_receipt" | "changed_receipt" | "active_predecessor" | "missing_predecessor"; - restoreHold?: "settled" | "no_marker" | "foreign_marker" | "reusable_policy" | "no_stop_receipt" | "dirty_writer" } = {}) { + restoreHold?: "settled" | "no_marker" | "foreign_marker" | "reusable_policy" | "no_stop_receipt" | "dirty_writer"; + idleWarm?: "retained" | "released" | "run_active" | "export_pending" | "same_target" | "cleanup_in_flight" } = {}) { const companyId = randomUUID(), agentId = randomUUID(), sessionId = randomUUID(), runnerInstanceId = randomUUID(); const sourceRunId = randomUUID(), currentRunId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Retired source", issuePrefix: `R${companyId.slice(0, 5)}` }); @@ -60,6 +63,22 @@ describeDb("retired native backup source authority", () => { remoteCwd: "/work", resourceDisposition: "keep_running", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), finalHostSha256: null } } }).where(eq(heartbeatRuns.id, sourceRunId)); } + if (options.idleWarm) { + // The staging row: the previous run succeeded and kept its warm sandbox + // running for reuse. No termination receipt, no hold, reuse scope intact. + const idle = options.idleWarm; + Object.assign(sourceLease, { status: idle === "released" ? "released" : "retained", leasePolicy: "reuse_by_environment", + releasedAt: idle === "released" ? new Date() : null }); + const { remoteExecutionTermination: _termination, ...metadata } = sourceLease.metadata; + sourceLease.metadata = { ...metadata, sandboxState: "started", reusableSandboxLease: { version: 1, provider: "daytona" }, + ...(idle === "cleanup_in_flight" ? { pendingCleanupInFlight: true } : {}) } as typeof sourceLease.metadata; + if (idle === "run_active") await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, sourceRunId)); + if (idle === "export_pending") await db.update(heartbeatRuns).set({ runnerProfileJson: { nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v3", state: "prepared", leaseId, providerLeaseId, workspaceId: randomUUID(), + remoteCwd: "/work", resourceDisposition: "keep_running", descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), + finalHostSha256: null } } }).where(eq(heartbeatRuns.id, sourceRunId)); + } + lastSourceLeaseId = leaseId; if (options.handoffs) { const environmentId = randomUUID(), issueId = randomUUID(), workspaceId = randomUUID(); await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); @@ -107,7 +126,8 @@ describeDb("retired native backup source authority", () => { nativeSessionId: sessionId, invocationSource: "automation", triggerDetail: "system" }).returning())[0]!.id, agentId, seq: 1, eventType: "session.resumed", createdAt: options.providerEventAt }); return retiredSourceAuthorizesNativeBackupRestore(db, { companyId, currentRunId, normalizedSessionId: sessionId, - manifest: { sourceProviderLeaseId: providerLeaseId, runnerInstanceId, normalizedSessionId: sessionId }, manifestSha256 }); + manifest: { sourceProviderLeaseId: providerLeaseId, runnerInstanceId, normalizedSessionId: sessionId }, manifestSha256, + targetProviderLeaseId: options.idleWarm === "same_target" ? providerLeaseId : randomUUID() }); } it("authorizes a destroyed source lease whose stamp names the backup", async () => { @@ -163,6 +183,59 @@ describeDb("retired native backup source authority", () => { await expect(fixture({ termination: "stopped", ...options })).resolves.toBe(false); }); + // Regression: reuse skipped the previous run's warm sandbox + // while its workspace export was saving, so the next run got a created lease. + // Once the export settled, that sandbox was idle and held exactly the backup. + it.each(["retained", "released"] as const)("authorizes an idle %s warm source and claims it as superseded", async (idleWarm) => { + await expect(fixture({ idleWarm })).resolves.toBe(true); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lastSourceLeaseId)); + // Reuse never resumes the source with the older state again; the next + // acquisition treats it as obsolete and destroys it. + expect(source!.status).toBe(idleWarm); + expect(source!.metadata).not.toHaveProperty("reusableSandboxLease"); + expect(source!.metadata?.reusableSandboxLeaseSuperseded).toMatchObject({ + schema: "paperclip.reusable-sandbox-lease-superseded.v1", reusableSandboxLease: { version: 1, provider: "daytona" } }); + }); + + it("refuses a reuse handoff of a source superseded while its provider resume was pending", async () => { + // An acquisition selected the warm source and resumed it at the provider. + // Before it records the handoff, this run claims the source and restores + // its backup elsewhere. The handoff then competes on the same row and loses. + await expect(fixture({ idleWarm: "retained" })).resolves.toBe(true); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lastSourceLeaseId)); + const environmentId = randomUUID(), runId = randomUUID(), agentId = randomUUID(); + await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); + await db.insert(agents).values({ id: agentId, companyId: source!.companyId, name: "Other", adapterType: "paperclip_runner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId: source!.companyId, agentId, status: "running", + runtimeMode: "native", invocationSource: "assignment", triggerDetail: "system" }); + const issueId = randomUUID(), workspaceId = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId: source!.companyId, title: "Warm source" }); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: source!.companyId, sourceIssueId: issueId, + mode: "shared_workspace", strategyType: "project_primary", name: "Warm workspace" }); + await db.update(environmentLeases).set({ environmentId, executionWorkspaceId: workspaceId, issueId }) + .where(eq(environmentLeases.id, source!.id)); + await expect(environmentService(db).acquireLease({ companyId: source!.companyId, environmentId, + executionWorkspaceId: workspaceId, issueId, heartbeatRunId: runId, leasePolicy: "reuse_by_environment", + provider: "daytona", providerLeaseId: source!.providerLeaseId, metadata: { agentId }, + replacesReusableLeaseId: source!.id })).rejects.toThrow("ownership changed"); + const [after] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, source!.id)); + expect(after!.status).toBe("retained"); + }); + + it.each([ + ["its run is still running", { idleWarm: "run_active" as const }], + ["its workspace export is still pending", { idleWarm: "export_pending" as const }], + ["its cleanup is in flight", { idleWarm: "cleanup_in_flight" as const }], + ["the backup would restore into its own allocation", { idleWarm: "same_target" as const }], + ["the stamp names another checkpoint", { idleWarm: "retained" as const, stampSha: `sha256:${"c".repeat(64)}` }], + ["provider work followed the checkpoint", { idleWarm: "retained" as const, providerEventAt: new Date(completedAt.getTime() + 60_000) }], + ["one allocation maps to two host leases", { idleWarm: "retained" as const, secondLease: true }], + ])("keeps an idle warm source's backup held when %s", async (_label, options) => { + await expect(fixture(options)).resolves.toBe(false); + const [source] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lastSourceLeaseId)); + expect(source!.metadata).toHaveProperty("reusableSandboxLease"); + }); + it("ignores provider work that finished before the checkpoint", async () => { await expect(fixture({ providerEventAt: new Date(completedAt.getTime() - 1_000) })).resolves.toBe(true); }); diff --git a/server/src/services/native-runtime/native-retired-backup-source.ts b/server/src/services/native-runtime/native-retired-backup-source.ts index 938985fc420..1997a013d0a 100644 --- a/server/src/services/native-runtime/native-retired-backup-source.ts +++ b/server/src/services/native-runtime/native-retired-backup-source.ts @@ -1,12 +1,12 @@ import { createHash } from "node:crypto"; -import { and, eq, gt, inArray, ne } from "drizzle-orm"; +import { and, eq, gt, inArray, ne, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; import { retiredReusableLeasePredecessors } from "../environment-lease-handoff.js"; import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; import { hasConfirmedSandboxStopAndRetain } from "../sandbox-stop-and-retain.js"; -import { LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "../workspace-restore-recovery-state.js"; +import { LEGACY_WORKSPACE_RECOVERY_KEY, LEGACY_WORKSPACE_RECOVERY_SCHEMA } from "../workspace-restore-recovery-state.js"; import { readNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; -import { nativeWorkspaceExportPendingForLease } from "./native-workspace-finalization-state.js"; +import { nativeWorkspaceExportPendingForLease, nativeWorkspaceExportPendingForResource } from "./native-workspace-finalization-state.js"; /** Durable run events that prove a provider session did work in a run. */ export const NATIVE_PROVIDER_DURABLE_EVENT_TYPES = new Set([ @@ -37,6 +37,15 @@ export function nativeHarnessBackupManifestSha256(manifestBytes: Buffer | string * stopped the sandbox under a stop-only receipt that permits no provider work. * Releases before the hold settlement restored warm reuse left such rows * stranded, so this also unsticks state saved by those releases. + * + * An idle warm source is the third. Its own run is terminal, no run holds the + * lease, and no workspace export is pending on the sandbox, so nothing can do + * provider work there. Reuse skips such a lease while the previous run's + * workspace export is still in flight, and the next run then gets a new lease. + * With no provider work after the backup, the idle sandbox holds exactly the + * checkpointed state, so restoring the backup continues the same session. The + * source is claimed as superseded first, so it is never resumed with that + * older state after the restored lease moves on. */ export async function retiredSourceAuthorizesNativeBackupRestore(db: Db, input: { companyId: string; @@ -44,8 +53,11 @@ export async function retiredSourceAuthorizesNativeBackupRestore(db: Db, input: normalizedSessionId: string; manifest: { sourceProviderLeaseId: string; runnerInstanceId: string; normalizedSessionId: string }; manifestSha256: string; + /** The allocation the backup restores into. Its own missing state never proves a retired source. */ + targetProviderLeaseId?: string | null; }): Promise { - if (!input.manifest.sourceProviderLeaseId || input.manifest.normalizedSessionId !== input.normalizedSessionId) return false; + if (!input.manifest.sourceProviderLeaseId || input.manifest.normalizedSessionId !== input.normalizedSessionId + || input.manifest.sourceProviderLeaseId === input.targetProviderLeaseId) return false; const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.providerLeaseId, input.manifest.sourceProviderLeaseId), @@ -64,7 +76,8 @@ export async function retiredSourceAuthorizesNativeBackupRestore(db: Db, input: || stamp.normalizedSessionId !== input.normalizedSessionId || stamp.runnerInstanceId !== input.manifest.runnerInstanceId) return false; // A released lease can still be selected for resume; only a retired one is final. - if (!destroyed && !await settledRestoreHoldRetiresSource(db, lease)) return false; + const idleSource = !destroyed && !await settledRestoreHoldRetiresSource(db, lease); + if (idleSource && !await idleWarmSourceRetiresBackup(db, lease, input.currentRunId)) return false; // Provider work recorded after the checkpoint means the destroyed sandbox // held newer state than this backup. Restoring would silently rewind it. const later = await db.select({ runId: heartbeatRunEvents.runId }).from(heartbeatRunEvents) @@ -77,7 +90,60 @@ export async function retiredSourceAuthorizesNativeBackupRestore(db: Db, input: inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_DURABLE_EVENT_TYPES]), gt(heartbeatRunEvents.createdAt, new Date(stamp.completedAt)), )).limit(1); - return later.length === 0; + if (later.length) return false; + return !idleSource || await supersedeIdleWarmSource(db, lease, input); +} + +export const SUPERSEDED_REUSABLE_SANDBOX_KEY = "reusableSandboxLeaseSuperseded"; + +/** + * Claims an idle warm source before its backup is restored elsewhere. The + * source still holds the checkpointed session, and the restored lease will + * move past it, so the source must never be resumed again. Removing its reuse + * scope makes the next acquisition for this workspace treat it as obsolete and + * clean it up through the existing obsolete-lease path. The write is a + * compare-and-swap on the checked row: a run that resumed or changed the lease + * in the meantime wins, and the restore does not happen. + */ +async function supersedeIdleWarmSource(db: Db, lease: typeof environmentLeases.$inferSelect, input: { + companyId: string; currentRunId: string; manifestSha256: string; targetProviderLeaseId?: string | null; +}) { + const { reusableSandboxLease: scope, ...metadata } = lease.metadata ?? {}; + const [claimed] = await db.update(environmentLeases).set({ + metadata: { ...metadata, [SUPERSEDED_REUSABLE_SANDBOX_KEY]: { + schema: "paperclip.reusable-sandbox-lease-superseded.v1", reason: "harness_backup_restored_elsewhere", + supersededByRunId: input.currentRunId, supersededByProviderLeaseId: input.targetProviderLeaseId ?? null, + manifestSha256: input.manifestSha256, reusableSandboxLease: scope ?? null, supersededAt: new Date().toISOString(), + } }, + updatedAt: new Date(), + }).where(and( + eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, lease.id), + eq(environmentLeases.status, lease.status), eq(environmentLeases.leasePolicy, "reuse_by_environment"), + eq(environmentLeases.heartbeatRunId, lease.heartbeatRunId!), + sql`${environmentLeases.metadata} = ${JSON.stringify(lease.metadata)}::jsonb`, + )).returning({ id: environmentLeases.id }); + return Boolean(claimed); +} + +const TERMINAL_SOURCE_RUN_STATUSES = new Set(["succeeded", "failed", "cancelled", "timed_out", "interrupted"]); + +/** + * A warm reusable lease whose run has ended and that no run holds. Its sandbox + * may still be running, but only a run does provider work there, and the + * caller's later-work check proves none did after the backup. A pending + * workspace export or cleanup on the allocation keeps it ambiguous. + */ +async function idleWarmSourceRetiresBackup(db: Db, lease: typeof environmentLeases.$inferSelect, currentRunId: string) { + if (lease.leasePolicy !== "reuse_by_environment" || !["released", "retained"].includes(lease.status) + || !lease.heartbeatRunId || lease.heartbeatRunId === currentRunId + || lease.metadata?.pendingCleanupInFlight === true + // A restore hold or a pending shutdown settlement still protects the files; reuse skips these too. + || Object.hasOwn(lease.metadata ?? {}, LEGACY_WORKSPACE_RECOVERY_KEY) + || (lease.metadata?.legacyShutdownWorkspaceSettlement as { state?: unknown } | undefined)?.state === "pending") return false; + const [run] = await db.select({ sourceRunStatus: heartbeatRuns.status }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, lease.companyId), eq(heartbeatRuns.id, lease.heartbeatRunId))).limit(1); + if (!run || !TERMINAL_SOURCE_RUN_STATUSES.has(run.sourceRunStatus)) return false; + return !await nativeWorkspaceExportPendingForResource(db, lease, currentRunId); } /** diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b7fc8561436..928353e57be 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -19996,7 +19996,26 @@ describe("runnerd provider runtime wiring", () => { ["a settled restore hold retained the stopped source", "stopped", "matching", false, true, "settled_hold"], ["a settled restore hold names another backup", "stopped", "foreign", false, false, "settled_hold"], ["a restore hold has no stop-and-retain receipt", "stopped", "matching", false, false, "unconfirmed_hold"], - ] as const)("restores a warm backup into a created lease only when %s", async (_label, terminationState, stampKind, laterProviderWork, restores, hold?: "settled_hold" | "unconfirmed_hold") => { + // Regression: the previous run succeeded and left its warm + // sandbox `retained` for reuse, but its workspace export was still saving when + // the monitor wake acquired a lease, so reuse skipped it and created a new one. + // The export finished before the harness prepared; the idle source held exactly + // the backup, yet the run failed backup_without_reusable_lease and blocked. + ["an idle warm source finished its run and its export", "none", "matching", false, true, "idle_warm"], + ["an idle warm source still has a workspace export pending", "none", "matching", false, false, "idle_warm_export_pending"], + ["the warm source's run is still running", "none", "matching", false, false, "idle_warm_run_active"], + ["an idle warm source had provider work after the backup", "none", "matching", true, false, "idle_warm"], + ["an idle warm source stamp names another backup", "none", "foreign", false, false, "idle_warm"], + ["another run claimed the idle warm source first", "none", "matching", false, false, "idle_warm_claim_lost"], + // Old-saved/new-resumed: the next run resumes the created lease that the failed + // run left without any harness state. It takes the backup under the same proof. + ["a resumed lease never held the session and the warm source is idle", "none", "matching", false, true, "resumed_idle_warm"], + ["a resumed lease lost the state the backup came from", "none", "matching", false, false, "resumed_own_source"], + ] as const)("restores a warm backup into a created lease only when %s", async (_label, terminationState, stampKind, laterProviderWork, restores, hold?: "settled_hold" | "unconfirmed_hold" | "idle_warm" | "idle_warm_export_pending" | "idle_warm_run_active" | "idle_warm_claim_lost" | "resumed_idle_warm" | "resumed_own_source") => { + const restoreHold = hold === "settled_hold" || hold === "unconfirmed_hold"; + const idleWarm = !!hold && !restoreHold; + const resumed = hold === "resumed_idle_warm" || hold === "resumed_own_source"; + const targetProviderLeaseId = hold === "resumed_own_source" ? "prior-sandbox" : "sandbox-created"; // Regression: a legacy follow-up retired the warm sandbox as an obsolete lease, so the // next reply got a created lease. The backup was the newest checkpoint, yet // every reply failed backup_without_reusable_lease and asked for another reply. @@ -20011,18 +20030,21 @@ describe("runnerd provider runtime wiring", () => { const confirmedAt = new Date().toISOString(); const sourceLease = (manifestSha256: string) => ({ id: "source-lease", companyId: current.binding.companyId, heartbeatRunId: "source-run", provider: "daytona", - providerLeaseId: "prior-sandbox", status: hold ? "released" : "expired", cleanupStatus: "success", - leasePolicy: hold ? "retain_on_failure" : "reuse_by_environment", - acquiredAt: new Date(Date.now() - 60_000), releasedAt: new Date(Date.now() + 1_000), expiresAt: null, + providerLeaseId: "prior-sandbox", status: idleWarm ? "retained" : restoreHold ? "released" : "expired", cleanupStatus: "success", + leasePolicy: restoreHold ? "retain_on_failure" : "reuse_by_environment", + acquiredAt: new Date(Date.now() - 60_000), releasedAt: idleWarm ? null : new Date(Date.now() + 1_000), expiresAt: null, metadata: { nativeHarnessBackup: { schema: "paperclip.native-harness-backup-stamp.v2", normalizedSessionId: current.session.normalizedSessionId, runnerInstanceId, sessionScopeSha256: "a".repeat(64), sourceProviderLeaseId: "prior-sandbox", authorizedProviderLeaseId: "prior-sandbox", manifestSha256, completedAt }, - remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: current.binding.companyId, - runId: "source-run", leaseId: "source-lease", provider: "daytona", providerLeaseId: "prior-sandbox", - state: terminationState, confirmedAt }, + // The idle warm source has no termination receipt: its sandbox is still running. + ...(terminationState === "none" ? { pluginId: "daytona-plugin", sandboxState: "started", + reusableSandboxLease: { version: 1, executionWorkspaceId: current.binding.executionWorkspaceId } } : { + remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: current.binding.companyId, + runId: "source-run", leaseId: "source-lease", provider: "daytona", providerLeaseId: "prior-sandbox", + state: terminationState, confirmedAt } }), // The exact row the hold left: marker, stop-and-retain receipt, no reuse. - ...(hold ? { pluginId: "daytona-plugin", + ...(restoreHold ? { pluginId: "daytona-plugin", workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", runId: "source-run" } } : {}), ...(hold === "settled_hold" ? { sandboxStopAndRetainReceipt: { schema: "paperclip.sandbox-stop-and-retain-receipt.v1", requestId: "stop-request", companyId: current.binding.companyId, runId: "source-run", leaseId: "source-lease", @@ -20033,19 +20055,32 @@ describe("runnerd provider runtime wiring", () => { const updates: Array<{ table: unknown; values: Record }> = []; const db = leaseDb(current, {}, {}, updates), ordinarySelect = db.select.bind(db); db.select = ((fields?: Record) => ({ from: (table: unknown) => { - if (table !== executionWorkspaces && table !== environmentLeases && table !== heartbeatRunEvents) + const sourceRunQuery = Boolean(table === heartbeatRuns && fields && "sourceRunStatus" in fields); + if (table !== executionWorkspaces && table !== environmentLeases && table !== heartbeatRunEvents && !sourceRunQuery) return ordinarySelect(fields as never).from(table as never); const manifest = backupDirectory.root ? readFileSync(join(backupDirectory.root, "manifest.json")) : ""; const sha = `sha256:${createHash("sha256").update(stampKind === "matching" ? manifest : "other").digest("hex")}`; - // No unsaved remote writer is bound to the held allocation. - const rows = table === executionWorkspaces || fields?.run ? [] + // No unsaved remote writer is bound to the held allocation, unless the + // previous run's workspace export is still observed on the warm sandbox. + const rows = sourceRunQuery ? [{ sourceRunStatus: hold === "idle_warm_run_active" ? "running" : "succeeded" }] + : fields?.run ? (hold === "idle_warm_export_pending" ? [{ lease: sourceLease(sha), + run: { id: "source-run", runtimeMode: "native", observedRemoteWorkspace: true, reference: null }, workspace: null }] : []) + : table === executionWorkspaces ? [] : table === heartbeatRunEvents ? (laterProviderWork ? [{ runId: "later-run" }] : []) - : [{ metadata: {}, providerLeaseId: "sandbox-created", companyId: current.binding.companyId, + : [{ metadata: {}, providerLeaseId: targetProviderLeaseId, companyId: current.binding.companyId, heartbeatRunId: current.binding.runId }, sourceLease(sha)]; const query = { then: Promise.resolve(rows).then.bind(Promise.resolve(rows)), leftJoin: () => query, innerJoin: () => query, where: () => query, limit: () => query }; return query; } })) as unknown as Db["select"]; + if (hold === "idle_warm_claim_lost") { + // A run resumed the warm source between the proof and the claim; the + // compare-and-swap matches no row and the backup stays unrestored. + db.update = ((table: unknown) => ({ set: () => ({ where: () => ({ returning: async () => { + expect(table).toBe(environmentLeases); + return []; + } }) }) })) as unknown as Db["update"]; + } const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { if (command.args?.[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", @@ -20069,7 +20104,7 @@ describe("runnerd provider runtime wiring", () => { runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment", leaseId: "lease-created", providerKey: "daytona", reusableLeaseConfigured: true, effectiveCapabilities: { runnerWebSocketIngress: true }, - sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "sandbox-created" }, + sandboxLeaseAcquisition: { outcome: resumed ? "resumed" : "created", providerLeaseId: targetProviderLeaseId }, runner: { execute: remoteExecute, syncIn }, } as never, }); @@ -20097,13 +20132,22 @@ describe("runnerd provider runtime wiring", () => { createHash("sha256").update(current.session.normalizedSessionId!).digest("hex")); expect(JSON.parse(await readFile(join(remoteSession, "runner/runner-state.json"), "utf8"))).toEqual(runnerState); expect(await readFile(join(remoteSession, "filesystem/codex-home/sentinel.txt"), "utf8")).toBe("retained provider home"); - expect(updates.filter(update => update.table === environmentLeases)).toEqual([expect.objectContaining({ - values: expect.objectContaining({ metadata: { nativeHarnessBackup: expect.objectContaining({ - normalizedSessionId: current.session.normalizedSessionId, sourceProviderLeaseId: "prior-sandbox", - authorizedProviderLeaseId: "sandbox-created" }) } }), - })]); + expect(updates.filter(update => update.table === environmentLeases)).toEqual([ + // The idle source is claimed first: its reuse scope moves into a superseded + // receipt, so it is never resumed with the older state again. + ...(idleWarm ? [expect.objectContaining({ values: expect.objectContaining({ + metadata: expect.not.objectContaining({ reusableSandboxLease: expect.anything() }) }) })] : []), + expect.objectContaining({ + values: expect.objectContaining({ metadata: { nativeHarnessBackup: expect.objectContaining({ + normalizedSessionId: current.session.normalizedSessionId, sourceProviderLeaseId: "prior-sandbox", + authorizedProviderLeaseId: "sandbox-created" }) } }), + })]); + if (idleWarm) expect(updates.filter(update => update.table === environmentLeases)[0]!.values.metadata).toEqual( + expect.objectContaining({ reusableSandboxLeaseSuperseded: expect.objectContaining({ + supersededByRunId: current.binding.runId, supersededByProviderLeaseId: "sandbox-created", + reusableSandboxLease: { version: 1, executionWorkspaceId: current.binding.executionWorkspaceId } }) })); } else { - await expect(options.prepareExternalRunnerState()).rejects.toThrow("backup_without_reusable_lease"); + await expect(options.prepareExternalRunnerState()).rejects.toThrow(resumed ? "resumed_unavailable" : "backup_without_reusable_lease"); expect(syncIn).not.toHaveBeenCalled(); expect(updates.filter(update => update.table === environmentLeases)).toEqual([]); } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 18a02d097b5..21d39a8995f 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -15685,9 +15685,10 @@ async function createRunnerdBackendWithinSessionClaim( if (!updated) throw new Error("runner_harness_backup_lease_missing"); }; - // A created lease may restore the backup when Paperclip destroyed its - // source sandbox and no provider work followed the checkpoint. - // The verified backup is reused by the restore, so files are hashed once. + // A lease without this session's state may restore the backup when its + // source sandbox is retired or idle and no provider work followed the + // checkpoint. The verified backup is reused by the restore, so files are + // hashed once. let retiredSourceBackup: VerifiedHarnessBackup | null = null; const retiredSourceBackupAuthorized = async (): Promise => { const backup = verifyNativeHarnessBackup({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId }); @@ -15698,6 +15699,7 @@ async function createRunnerdBackendWithinSessionClaim( companyId: input.execution.binding.companyId, currentRunId: input.execution.binding.runId, normalizedSessionId: sessionId, manifest: backup.manifest, manifestSha256: nativeHarnessBackupManifestSha256(readFileSync(resolve(backup.root, "manifest.json"))), + targetProviderLeaseId: sandboxLeaseAcquisition?.providerLeaseId ?? null, }); }; @@ -15884,7 +15886,18 @@ async function createRunnerdBackendWithinSessionClaim( && hasRemoteBootstrapRestartProof(input.execution.binding.runId) && await moveInterruptedBootstrapSessionAside() && takeRemoteBootstrapRestartProof(input.execution.binding.runId); - if ((state.incompleteReason !== "unavailable" && !provenRestart) || backupAvailable || + // A resumed allocation that never held this session (for example + // the created lease of a run that failed before restoring) takes + // the backup under the same retired-or-idle source proof. + const restoresRetiredSource = !provenRestart && state.incompleteReason === "unavailable" + && backupAvailable && remoteTarget.reusableLeaseConfigured === true + && await retiredSourceBackupAuthorized(); + if (restoresRetiredSource) { + await measureNativeRunnerSpan(input.trace, "sandbox.lease.replacement", restoreVerifiedHarnessBackup, { + attributes: { provider: remoteTarget.providerKey ?? "sandbox", harness: input.execution.session.driverKind, + lifecycleMode: input.execution.session.lifecyclePolicy.mode, outcome: "resumed", reason: "state_absent" }, + }); + } else if ((state.incompleteReason !== "unavailable" && !provenRestart) || backupAvailable || !(await claimUntouchedRemoteSession())) { throw new Error(`runner_harness_state_mismatch: resumed_${state.incompleteReason}`); }