diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index a7d98f31ef2..c663f7b55bc 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2020,6 +2020,16 @@ and plugin tasks. The running session then owns authority; the old receipt must not conflict with a later warm run's journal identity. Failed admission retains the receipt. Normal suspension writes a new receipt. +A warm session can lose its reusable lease when Paperclip retires the sandbox, +for example as an obsolete lease during another run's acquisition. The next run +then gets a new sandbox while the controller still holds the session's verified +backup. Paperclip restores that backup into the new sandbox only when all of +these are true: the backup's source lease is expired or failed with a provider +`destroyed` termination receipt, the source lease's backup stamp names this +exact backup manifest, and no provider event for the session was recorded after +the backup completed. Any other state is ambiguous. The run then fails closed +with `runner_harness_state_mismatch: backup_without_reusable_lease`. + A per-turn sandbox turn can finish through recovery without a Runner suspension receipt or failover backup on the controller. For example, its post-turn checkpoint copy can fail on a full sandbox disk. The controller @@ -2437,6 +2447,14 @@ resolved actions whose retained work may still need attention. This observation does not prove provider termination, local descriptor integrity, or successful file preservation, and it does not explain an earlier owner state retroactively. Recorded system recovery failures retain their original attempt and intent. +When system workspace recovery exhausts its bounded attempts or deadline, the +recovery action stays active and its owner changes from system to board. Its +next action names the attempt count and last failure code and tells the board to +use **Retry recovery**. The task blocker shows that same text. Retry starts one +new bounded system generation and returns ownership to the system. A sweep that +finds an action already exhausted and still system-owned applies the same handoff +as bookkeeping only; it grants no attempt. Exhaustion never submits a provider +turn. New source-binding failures record a closed reason and, for a rejected lease policy, an allowlisted policy value. Neither detail includes paths, provider allocation identifiers, credentials, or raw provider errors. diff --git a/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts b/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts new file mode 100644 index 00000000000..f725280f4ba --- /dev/null +++ b/server/src/services/native-runtime/native-retired-backup-source.integration.test.ts @@ -0,0 +1,71 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { agents, companies, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns } from "@paperclipai/db"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; +import { retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; + +const describeDb = recoveryPostgresSupport.supported ? describe : describe.skip; + +describeDb("retired native backup source authority", () => { + let database: Awaited>; + let db: ReturnType; + beforeAll(async () => { + database = await startRecoveryPostgres("retired-backup-source"); + db = createDb(database.connectionString); + }, 120_000); + afterAll(async () => { await database?.cleanup(); }); + + const manifestSha256 = `sha256:${"b".repeat(64)}`; + const completedAt = new Date("2026-11-05T05:02:50.000Z"); + + async function fixture(options: { termination?: "destroyed" | "stopped" | null; status?: string; stampSha?: string; + providerEventAt?: Date; secondLease?: boolean } = {}) { + const companyId = randomUUID(), agentId = randomUUID(), sessionId = randomUUID(), runnerInstanceId = randomUUID(); + const sourceRunId = randomUUID(), currentRunId = randomUUID(), leaseId = randomUUID(), providerLeaseId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Retired source", issuePrefix: `R${companyId.slice(0, 5)}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Codie", adapterType: "paperclip_runner" }); + for (const id of [sourceRunId, currentRunId]) await db.insert(heartbeatRuns).values({ id, companyId, agentId, + status: id === sourceRunId ? "succeeded" : "running", runtimeMode: "native", nativeSessionId: sessionId, + invocationSource: "assignment", triggerDetail: "system" }); + const lease = (id: string) => ({ id, companyId, heartbeatRunId: sourceRunId, provider: "daytona", providerLeaseId, + status: options.status ?? "expired", cleanupStatus: "success", releasedAt: new Date(), + metadata: { + nativeHarnessBackup: { schema: "paperclip.native-harness-backup-stamp.v2", normalizedSessionId: sessionId, runnerInstanceId, + sessionScopeSha256: "a".repeat(64), sourceProviderLeaseId: providerLeaseId, authorizedProviderLeaseId: providerLeaseId, + manifestSha256: options.stampSha ?? manifestSha256, completedAt: completedAt.toISOString() }, + ...(options.termination === null ? {} : { remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId, + runId: sourceRunId, leaseId: id, provider: "daytona", providerLeaseId, state: options.termination ?? "destroyed", + confirmedAt: new Date().toISOString() } }), + } }); + await db.insert(environmentLeases).values(lease(leaseId)); + if (options.secondLease) await db.insert(environmentLeases).values(lease(randomUUID())); + // The backup turn's own provider events precede the checkpoint. + await db.insert(heartbeatRunEvents).values({ companyId, runId: sourceRunId, agentId, seq: 1, eventType: "turn.started", + createdAt: new Date(completedAt.getTime() - 60_000) }); + if (options.providerEventAt) await db.insert(heartbeatRunEvents).values({ companyId, runId: currentRunId === sourceRunId + ? sourceRunId : (await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", runtimeMode: "native", + nativeSessionId: sessionId, invocationSource: "automation", triggerDetail: "system" }).returning())[0]!.id, + agentId, seq: 1, eventType: "session.resumed", createdAt: options.providerEventAt }); + return retiredSourceAuthorizesNativeBackupRestore(db, { companyId, currentRunId, normalizedSessionId: sessionId, + manifest: { sourceProviderLeaseId: providerLeaseId, runnerInstanceId, normalizedSessionId: sessionId }, manifestSha256 }); + } + + it("authorizes a destroyed source lease whose stamp names the backup", async () => { + await expect(fixture()).resolves.toBe(true); + }); + + it.each([ + ["the provider only stopped the source", { termination: "stopped" as const }], + ["no termination receipt exists", { termination: null }], + ["the source lease is still reusable", { status: "released" }], + ["the stamp names another checkpoint", { stampSha: `sha256:${"c".repeat(64)}` }], + ["provider work followed the checkpoint", { providerEventAt: new Date(completedAt.getTime() + 60_000) }], + ["one allocation maps to two host leases", { secondLease: true }], + ])("fails closed when %s", async (_label, options) => { + await expect(fixture(options)).resolves.toBe(false); + }); + + it("ignores provider work that finished before the checkpoint", async () => { + await expect(fixture({ providerEventAt: new Date(completedAt.getTime() - 1_000) })).resolves.toBe(true); + }); +}); diff --git a/server/src/services/native-runtime/native-retired-backup-source.ts b/server/src/services/native-runtime/native-retired-backup-source.ts new file mode 100644 index 00000000000..865a2f43322 --- /dev/null +++ b/server/src/services/native-runtime/native-retired-backup-source.ts @@ -0,0 +1,69 @@ +import { createHash } from "node:crypto"; +import { and, eq, gt, inArray, ne } from "drizzle-orm"; +import { environmentLeases, heartbeatRunEvents, heartbeatRuns, type Db } from "@paperclipai/db"; +import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +import { readNativeHarnessBackupStamp } from "./native-harness-backup-stamp.js"; + +/** Durable run events that prove a provider session did work in a run. */ +export const NATIVE_PROVIDER_DURABLE_EVENT_TYPES = new Set([ + "harness.ready", + "session.started", + "session.resumed", + "session.updated", + "turn.started", + "provider.event", + "provider.rpc_result", +]); + +export function nativeHarnessBackupManifestSha256(manifestBytes: Buffer | string): string { + return `sha256:${createHash("sha256").update(manifestBytes).digest("hex")}`; +} + +/** + * A created sandbox normally refuses a per-turn backup that has no recorded + * reusable lease, because the backup's source sandbox may still hold newer + * provider state. That ambiguity is absent when Paperclip itself destroyed the + * source sandbox: the provider confirmed the termination, the source lease's + * stamp names exactly this backup, and no provider work for the session was + * recorded after the backup completed. This is the same proof a confirmed + * replacement relies on, reached through a lease that was retired first. + */ +export async function retiredSourceAuthorizesNativeBackupRestore(db: Db, input: { + companyId: string; + currentRunId: string; + normalizedSessionId: string; + manifest: { sourceProviderLeaseId: string; runnerInstanceId: string; normalizedSessionId: string }; + manifestSha256: string; +}): Promise { + if (!input.manifest.sourceProviderLeaseId || input.manifest.normalizedSessionId !== input.normalizedSessionId) return false; + const leases = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.providerLeaseId, input.manifest.sourceProviderLeaseId), + )); + const source = leases.filter(lease => lease.companyId === input.companyId + && lease.providerLeaseId === input.manifest.sourceProviderLeaseId); + // One provider allocation maps to one host lease. Anything else is ambiguous. + if (source.length !== 1) return false; + const lease = source[0]!; + const termination = lease.metadata?.remoteExecutionTermination as { state?: unknown } | undefined; + // A released lease can still be selected for resume; only a retired one is final. + if (!["expired", "failed"].includes(lease.status) || !hasRemoteTerminationReceipt(lease) + || termination?.state !== "destroyed") return false; + const stamp = readNativeHarnessBackupStamp(lease.metadata?.nativeHarnessBackup, lease.providerLeaseId!); + if (!stamp || stamp.manifestSha256 !== input.manifestSha256 + || stamp.normalizedSessionId !== input.normalizedSessionId + || stamp.runnerInstanceId !== input.manifest.runnerInstanceId) return false; + // Provider work recorded after the checkpoint means the destroyed sandbox + // held newer state than this backup. Restoring would silently rewind it. + const later = await db.select({ runId: heartbeatRunEvents.runId }).from(heartbeatRunEvents) + .innerJoin(heartbeatRuns, eq(heartbeatRuns.id, heartbeatRunEvents.runId)) + .where(and( + eq(heartbeatRunEvents.companyId, input.companyId), + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.nativeSessionId, input.normalizedSessionId), + ne(heartbeatRuns.id, input.currentRunId), + inArray(heartbeatRunEvents.eventType, [...NATIVE_PROVIDER_DURABLE_EVENT_TYPES]), + gt(heartbeatRunEvents.createdAt, new Date(stamp.completedAt)), + )).limit(1); + return later.length === 0; +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 9f1898dea66..4f9df8a7dea 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -17299,6 +17299,113 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each([ + ["the source sandbox was destroyed by Paperclip", "destroyed", "matching", false, true], + ["the source sandbox was only stopped", "stopped", "matching", false, false], + ["the source stamp names another backup", "destroyed", "foreign", false, false], + ["provider work followed the backup", "destroyed", "matching", true, false], + ] as const)("restores a warm backup into a created lease only when %s", async (_label, terminationState, stampKind, laterProviderWork, restores) => { + // Regression: a legacy follow-up retired the warm sandbox as an obsolete lease, so the + // next reply got a created lease. The backup was the newest checkpoint, yet + // every reply failed backup_without_reusable_lease and asked for another reply. + const remoteCwd = join(isolatedStateDirectory, "remote"); + const current: NativeExecutionInputV1 = { ...execution, + binding: { ...execution.binding, companyId: randomUUID(), agentId: randomUUID(), issueId: randomUUID(), + runId: randomUUID(), executionWorkspaceId: randomUUID() }, + session: { ...execution.session, normalizedSessionId: randomUUID(), lifecyclePolicy: { mode: "warm", idleTimeoutMs: 300_000 } }, + }; + const runnerInstanceId = "retired-warm-runner", completedAt = "2026-11-05T05:02:50.000Z"; + const backupDirectory = { root: "" }; + const sourceLease = (manifestSha256: string) => ({ + id: "source-lease", companyId: current.binding.companyId, heartbeatRunId: "source-run", provider: "daytona", + providerLeaseId: "prior-sandbox", status: "expired", cleanupStatus: "success", releasedAt: new Date(), + metadata: { + nativeHarnessBackup: { schema: "paperclip.native-harness-backup-stamp.v2", normalizedSessionId: current.session.normalizedSessionId, + runnerInstanceId, sessionScopeSha256: "a".repeat(64), sourceProviderLeaseId: "prior-sandbox", + authorizedProviderLeaseId: "prior-sandbox", manifestSha256, completedAt }, + remoteExecutionTermination: { schema: "paperclip.remote-termination.v1", companyId: current.binding.companyId, + runId: "source-run", leaseId: "source-lease", provider: "daytona", providerLeaseId: "prior-sandbox", + state: terminationState, confirmedAt: new Date().toISOString() }, + }, + }); + const updates: Array<{ table: unknown; values: Record }> = []; + const db = leaseDb(current, {}, {}, updates), ordinarySelect = db.select.bind(db); + db.select = ((fields?: Record) => ({ from: (table: unknown) => { + if (table !== executionWorkspaces && table !== environmentLeases && table !== heartbeatRunEvents) + return ordinarySelect(fields as never).from(table as never); + const manifest = backupDirectory.root ? readFileSync(join(backupDirectory.root, "manifest.json")) : ""; + const sha = `sha256:${createHash("sha256").update(stampKind === "matching" ? manifest : "other").digest("hex")}`; + const rows = table === executionWorkspaces ? [] + : table === heartbeatRunEvents ? (laterProviderWork ? [{ runId: "later-run" }] : []) + : [{ metadata: {}, providerLeaseId: "sandbox-created", companyId: current.binding.companyId, + heartbeatRunId: current.binding.runId }, sourceLease(sha)]; + const query = { then: Promise.resolve(rows).then.bind(Promise.resolve(rows)), + leftJoin: () => query, innerJoin: () => query, where: () => query, limit: () => query }; + return query; + } })) as unknown as Db["select"]; + const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { + if (command.args?.[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + }) }; + if (command.args?.[0] === "--version") return { exitCode: 0, timedOut: false, stdout: "codex-cli 0.160.0", stderr: "" }; + if (command.args?.[1]?.includes("base64")) { + try { return { exitCode: 0, timedOut: false, stdout: execFileSync("sh", command.args, { encoding: "utf8" }), stderr: "" }; } + catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } + } + return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + }); + const syncIn = vi.fn(async (operations: Array<{ files: Array<{ sourcePath: string; targetPath: string }> }>) => { + for (const operation of operations) for (const file of operation.files) { + await mkdir(dirname(file.targetPath), { recursive: true }); + await cp(file.sourcePath, file.targetPath, { recursive: true }); + } + }); + await createRunnerdBackend({ db, execution: current, runnerInstanceId, runnerIngressAuthorized: true, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment", + leaseId: "lease-created", providerKey: "daytona", reusableLeaseConfigured: true, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "created", providerLeaseId: "sandbox-created" }, + runner: { execute: remoteExecute, syncIn }, + } as never, + }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise }; + const backupRoot = join(options.stateDirectory!, "failover-backups/current"); + backupDirectory.root = backupRoot; + for (const directory of resolveNativeHarnessPersistenceProfile(current).directories) + await mkdir(join(backupRoot, directory.name), { recursive: true }); + const runnerState = durableRunnerState({ runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId, + runnerInstanceId, environmentLeaseId: "lease-created" }, "suspended"); + await writeFile(join(backupRoot, "runner/runner-state.json"), JSON.stringify(runnerState)); + await writeFile(join(backupRoot, "runner/codex-provider-state.json"), JSON.stringify({ + schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "prepared", threadId: "retained-provider-thread", + providerSessionId: null, activeProviderTurnId: null, config: { provider: "codex", driver: "codex_app_server" }, + })); + await writeFile(join(backupRoot, "codex-home/sentinel.txt"), "retained provider home"); + await writeFile(join(backupRoot, "manifest.json"), JSON.stringify(buildNativeHarnessBackupManifest({ + backupRoot, execution: current, runnerInstanceId, sourceProviderLeaseId: "prior-sandbox", + providerSessionIdentity: { providerSessionId: "retained-provider-thread", providerBackendSessionId: null, providerSessionIdentity: null }, + }))); + if (restores) { + await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + const remoteSession = join(remoteCwd, ".paperclip-runtime/paperclip-runner/sessions", + createHash("sha256").update(current.session.normalizedSessionId!).digest("hex")); + expect(JSON.parse(await readFile(join(remoteSession, "runner/runner-state.json"), "utf8"))).toEqual(runnerState); + expect(await readFile(join(remoteSession, "filesystem/codex-home/sentinel.txt"), "utf8")).toBe("retained provider home"); + expect(updates.filter(update => update.table === environmentLeases)).toEqual([expect.objectContaining({ + values: expect.objectContaining({ metadata: { nativeHarnessBackup: expect.objectContaining({ + normalizedSessionId: current.session.normalizedSessionId, sourceProviderLeaseId: "prior-sandbox", + authorizedProviderLeaseId: "sandbox-created" }) } }), + })]); + } else { + await expect(options.prepareExternalRunnerState()).rejects.toThrow("backup_without_reusable_lease"); + expect(syncIn).not.toHaveBeenCalled(); + expect(updates.filter(update => update.table === environmentLeases)).toEqual([]); + } + }); + it("archives failover evidence with an explicitly replaced provider session", async () => { const remoteCwd = join(isolatedStateDirectory, "remote"); const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a70335eb7b0..f4d1fe94bb0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -2,6 +2,7 @@ import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnost import { nativePostResultFailureCategory, type NativePostResultFailureCategory } from "./native-post-result-failure.js"; import { logger } from "../../middleware/logger.js"; import { canRestoreMigratedNativeSession } from "./native-workspace-migration.js"; +import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES, nativeHarnessBackupManifestSha256, retiredSourceAuthorizesNativeBackupRestore } from "./native-retired-backup-source.js"; import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js"; import { resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; @@ -6752,10 +6753,12 @@ export function shouldRestoreNativeHarnessBackupIntoSandbox(input: { reusableLeaseConfigured: boolean | null | undefined; backupAvailable: boolean; migratedSessionContinuation?: boolean; + retiredSourceContinuation?: boolean; }): boolean { return ( input.acquisitionOutcome === "created" && - (input.reusableLeaseConfigured === false || input.migratedSessionContinuation === true) && + (input.reusableLeaseConfigured === false || input.migratedSessionContinuation === true + || input.retiredSourceContinuation === true) && input.backupAvailable ); } @@ -7325,15 +7328,7 @@ const NATIVE_STARTUP_MODEL_REJECTED_ACTION = const NATIVE_CLEANUP_OPERATOR_RECOVERY_MESSAGE = "Send a new message to continue after Paperclip verifies that the previous provider and its tools have stopped. If cleanup cannot be verified, inspect the run and its environment. Clearing a task session does not resolve this quarantine. Automatic retries are stopped."; -const PROVIDER_DURABLE_EVENT_TYPES = new Set([ - "harness.ready", - "session.started", - "session.resumed", - "session.updated", - "turn.started", - "provider.event", - "provider.rpc_result", -]); +const PROVIDER_DURABLE_EVENT_TYPES = NATIVE_PROVIDER_DURABLE_EVENT_TYPES; type NativeRecoveryMode = "bootstrap_retry" | "exact_checkpoint_resume" | "ambiguous_state"; @@ -14103,11 +14098,27 @@ async function createRunnerdBackendWithinSessionClaim( if (!updated) throw new Error("runner_harness_backup_lease_missing"); }; + // A created lease may restore the backup when Paperclip destroyed its + // source sandbox and no provider work followed the checkpoint. + // The verified backup is reused by the restore, so files are hashed once. + let retiredSourceBackup: VerifiedHarnessBackup | null = null; + const retiredSourceBackupAuthorized = async (): Promise => { + const backup = verifyNativeHarnessBackup({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId }); + const sessionId = nativeSessionKey(input.execution); + if (!backup || !sessionId) return false; + retiredSourceBackup = backup; + return retiredSourceAuthorizesNativeBackupRestore(input.db, { + companyId: input.execution.binding.companyId, currentRunId: input.execution.binding.runId, + normalizedSessionId: sessionId, manifest: backup.manifest, + manifestSha256: nativeHarnessBackupManifestSha256(readFileSync(resolve(backup.root, "manifest.json"))), + }); + }; + const restoreVerifiedHarnessBackup = async () => { if (!remoteTarget || !remoteCommandRunner) { throw new Error("runner_harness_backup_unavailable"); } - const backup = verifyNativeHarnessBackup({ + const backup = retiredSourceBackup ?? verifyNativeHarnessBackup({ root, execution: input.execution, runnerInstanceId: input.runnerInstanceId, @@ -14204,6 +14215,11 @@ async function createRunnerdBackendWithinSessionClaim( "harness_state.bootstrap_claim", claimUntouchedRemoteSession, ); + const migratedSessionContinuation = sandboxLeaseAcquisition?.outcome === "created" && backupAvailable + && remoteTarget.reusableLeaseConfigured === true && await canRestoreMigratedNativeSession(input.db, { + ...input.execution.binding, workspaceId: input.execution.binding.executionWorkspaceId, + sessionId: nativeSessionKey(input.execution), logicalCwd: input.execution.workspace.cwd, remoteCwd: remoteTarget.remoteCwd, + }); const restoreIntoCreatedSandbox = shouldRestoreNativeHarnessBackupIntoSandbox({ acquisitionOutcome: @@ -14211,11 +14227,10 @@ async function createRunnerdBackendWithinSessionClaim( reusableLeaseConfigured: remoteTarget.reusableLeaseConfigured, backupAvailable, - migratedSessionContinuation: sandboxLeaseAcquisition?.outcome === "created" && backupAvailable - && remoteTarget.reusableLeaseConfigured === true && await canRestoreMigratedNativeSession(input.db, { - ...input.execution.binding, workspaceId: input.execution.binding.executionWorkspaceId, - sessionId: nativeSessionKey(input.execution), logicalCwd: input.execution.workspace.cwd, remoteCwd: remoteTarget.remoteCwd, - }), + migratedSessionContinuation, + // Migration already authorizes the restore; skip the second proof. + retiredSourceContinuation: !migratedSessionContinuation && sandboxLeaseAcquisition?.outcome === "created" + && backupAvailable && remoteTarget.reusableLeaseConfigured === true && await retiredSourceBackupAuthorized(), }); await input.trace?.record({ name: "sandbox.lease.acquisition", diff --git a/server/src/services/system-recovery.ts b/server/src/services/system-recovery.ts index 347593dafed..bfc8e36f0fb 100644 --- a/server/src/services/system-recovery.ts +++ b/server/src/services/system-recovery.ts @@ -47,20 +47,33 @@ export async function adoptSystemRecovery(db: Db, source: WorkspaceRecoverySourc }); } +export function exhaustedSystemRecoveryAction(state: SystemRecoveryState): string { + const reason = state.failure?.code ? ` (last failure: ${state.failure.code})` : ""; + return `Automatic recovery of saved work stopped after ${state.attempts} attempts${reason}. ` + + "Retry recovery to export and merge from the retained sandbox again; no provider turn is submitted. " + + "If Retry keeps failing, inspect the run and its environment."; +} + /** CAS protects a newer manual generation from an old worker's late write. * This is bookkeeping authority only; physical writes still require ownership. */ export async function updateSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity, previous: SystemRecoveryState, next: SystemRecoveryState) { const status = next.state === "complete" ? "recovered" : next.state === "exhausted" ? "blocked" : next.state === "running" ? "recovering" : "queued"; + // A spent budget is not system work any more. Hand it to the board with the + // one action that can progress it, instead of leaving an inert system owner. + const exhausted = next.state === "exhausted"; + const handoff = exhausted ? exhaustedSystemRecoveryAction(next) : null; const compatibility = { schema: "paperclip.workspace-recovery.v1", runId: source.runId, status, stage: next.phase === "verify_execution" ? "verifying_stop" : next.phase === "preserve_files" ? "restoring_files" : "saving_receipts", attempt: next.attempts, attemptLimit: 6, retryable: true, retryAt: next.nextAttemptAt ?? undefined, + ...(handoff ? { message: handoff } : {}), }; return writeExecutionControl(db, DELIVERY_QUEUES.workspaceRecovery, async tx => { const [updated] = await tx.update(issueRecoveryActions).set({ - ownerType: "system", ownerAgentId: null, ownerUserId: null, + ownerType: exhausted ? "board" : "system", ownerAgentId: null, ownerUserId: null, + ...(handoff ? { nextAction: handoff } : {}), evidence: sql`${issueRecoveryActions.evidence} || jsonb_build_object('systemRecovery', ${JSON.stringify(next)}::jsonb || ${next.generationId === previous.generationId && next.intentId === previous.intentId ? sql`jsonb_build_object('intentId', ${issueRecoveryActions.evidence}#>'{systemRecovery,intentId}')` @@ -76,6 +89,26 @@ export async function updateSystemRecovery(db: Db, source: WorkspaceRecoverySour }, { companyId: source.companyId }); } +/** Bookkeeping only: an action that was exhausted before the board handoff + * existed still reaches the board. It grants no attempt and keeps the generation. */ +export async function handOffExhaustedSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity, + state: SystemRecoveryState) { + if (state.state !== "exhausted") return false; + const handoff = exhaustedSystemRecoveryAction(state); + const [updated] = await db.update(issueRecoveryActions).set({ + ownerType: "board", ownerAgentId: null, ownerUserId: null, nextAction: handoff, + evidence: sql`jsonb_set(${issueRecoveryActions.evidence}, '{workspaceRecovery,message}', ${JSON.stringify(handoff)}::jsonb)`, + updatedAt: new Date(), + }).where(and(eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.id, source.actionId), + eq(issueRecoveryActions.sourceIssueId, source.issueId), eq(issueRecoveryActions.ownerType, "system"), + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' = 'exhausted'`, + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,generationId}' = ${state.generationId}`, + sql`${issueRecoveryActions.evidence}#>>'{systemRecovery,revision}' = ${String(state.revision)}`, + sql`jsonb_typeof(${issueRecoveryActions.evidence}->'workspaceRecovery') = 'object'`)) + .returning({ id: issueRecoveryActions.id }); + return Boolean(updated); +} + export async function loadSystemRecovery(db: Db, source: WorkspaceRecoverySourceIdentity) { const [row] = await db.select({ evidence: issueRecoveryActions.evidence }).from(issueRecoveryActions).where(and( eq(issueRecoveryActions.companyId, source.companyId), eq(issueRecoveryActions.sourceIssueId, source.issueId), diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index af3ae52399d..e760af1b97d 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -455,7 +455,16 @@ describe("automatic retained workspace recovery", () => { } const before = await action(f.actionId); await recoverRetainedWorkspace(f.input); - expect(await action(f.actionId)).toEqual(before); + const after = await action(f.actionId); + // An exhausted source that cannot be adopted is only handed to the board. + // Its recovery state, evidence and retention stay untouched. + const spent = readSystemRecovery(before.evidence)?.state === "exhausted" && before.ownerType === "system"; + const stable = (row: typeof after) => ({ ...row, updatedAt: null, ownerType: null, nextAction: null, + evidence: { ...row.evidence, workspaceRecovery: { ...(row.evidence.workspaceRecovery as object), message: null } } }); + if (spent) { + expect(after).toMatchObject({ ownerType: "board", nextAction: expect.stringMatching(/Retry recovery/) }); + expect(stable(after)).toEqual(stable(before)); + } else expect(after).toEqual(before); expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)))[0].leasePolicy).toBe("reuse_by_environment"); expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); expect(f.runtime.retryPendingSandboxTeardown).not.toHaveBeenCalled(); @@ -2957,6 +2966,61 @@ describe("automatic retained workspace recovery", () => { } finally { await coordinator.stop(); await worker.stop(); } }, 30_000); + it("hands an exhausted system export repair to the board instead of leaving it system-owned", async () => { + // Regression: the bounded export budget ran out, but the action stayed active and + // system-owned for hours. The board saw no owner, and replies started nothing. + const f = await seed(); + await adoptSystemRecovery(db, f); + const before = await action(f.actionId); + const adopted = readSystemRecovery(before.evidence)!; + const now = Date.now(); + await db.update(issueRecoveryActions).set({ cause: "native_workspace_sync_out_retry_exhausted", + nextAction: "Finalization retry budget exhausted. Retry workspace export and merge from the retained sandbox; do not submit another provider turn.", + evidence: { ...before.evidence, systemRecovery: { ...adopted, revision: adopted.revision + 1, phase: "preserve_files", + state: "scheduled", attempts: 6, retriesUsed: 5, firstFailureAt: now - 3_600_000, deadlineAt: now - 1_000, nextAttemptAt: null, + failure: { code: "export_interrupted", retryClass: "retry_transient", evidenceFingerprint: "f".repeat(64) } } } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await recoverRetainedWorkspace(f.input); + const exhausted = await action(f.actionId); + expect(readSystemRecovery(exhausted.evidence)?.state).toBe("exhausted"); + expect(exhausted).toMatchObject({ status: "active", ownerType: "board" }); + expect(exhausted.nextAction).toMatch(/stopped after 6 attempts/); + const blocker = await getExecutionBlocker(db, f.companyId, f.issueId); + expect(blocker?.workspaceRecovery).toMatchObject({ status: "blocked", canRetry: true }); + expect(blocker?.nextAction).toMatch(/Retry recovery/); + expect(blocker?.nextAction).not.toMatch(/send a (new )?message/i); + // Nothing physical or provider-facing ran after the budget was spent. + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + // The board's Retry starts one new bounded system generation. + await requestWorkspaceRecovery({ ...f.input, idempotencyKey: randomUUID(), actorId: "board" }); + const retried = await action(f.actionId); + expect(retried.ownerType).toBe("system"); + expect(readSystemRecovery(retried.evidence)).toMatchObject({ state: "scheduled", attempts: 0 }); + }, 30_000); + + it("hands a repair that was already exhausted and system-owned to the board without another attempt", async () => { + const f = await seed(); + await adoptSystemRecovery(db, f); + const before = await action(f.actionId); + const spent = failSystemRecovery({ ...readSystemRecovery(before.evidence)!, attempts: 6, retriesUsed: 5 }, + new SystemRecoveryFailure("export_interrupted", "retry_transient")); + expect(spent.state).toBe("exhausted"); + await db.update(issueRecoveryActions).set({ cause: "native_workspace_sync_out_retry_exhausted", ownerType: "system", + evidence: { ...before.evidence, systemRecovery: spent, + workspaceRecovery: { ...readWorkspaceRecovery(before.evidence)!, status: "blocked", attempt: 6, retryable: true } } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await recoverRetainedWorkspace(f.input); + const handed = await action(f.actionId); + expect(handed).toMatchObject({ status: "active", ownerType: "board" }); + expect(readSystemRecovery(handed.evidence)).toEqual(spent); + expect((await getExecutionBlocker(db, f.companyId, f.issueId))?.nextAction).toMatch(/Retry recovery/); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + // Repeat sweeps are idempotent and still grant no attempt. + await recoverRetainedWorkspace(f.input); + expect(readSystemRecovery((await action(f.actionId)).evidence)).toEqual(spent); + }, 30_000); + it("reconstructs a resolved files-only preservation obligation at startup without replay", async () => { const f = await seed("legacy"); await adoptSystemRecovery(db, f); diff --git a/server/src/services/workspace-recovery.ts b/server/src/services/workspace-recovery.ts index e910dd81003..9372a62c97d 100644 --- a/server/src/services/workspace-recovery.ts +++ b/server/src/services/workspace-recovery.ts @@ -28,7 +28,7 @@ import { withNativeWorkspaceFinalizationOwnership } from "./native-runtime/nativ import { readNativeWorkspaceSyncReference, readNativeWorkspaceSyncLocalCwd, resumeNativeWorkspaceSync } from "./native-runtime/native-workspace-sync.js"; import { classifyNativeWorkspaceFailure } from "./native-runtime/native-workspace-failure.js"; -import { adoptSystemRecovery, loadSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; +import { handOffExhaustedSystemRecovery, adoptSystemRecovery, loadSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; import { SYSTEM_RECOVERY_MAX_RETRIES, createSystemRecovery, failSystemRecovery, readSystemRecovery, reserveSystemRecoveryAttempt, SystemRecoveryFailure, type SystemRecoveryState } from "./system-recovery-state.js"; import { inspectWorkspaceRecoverySource, workspaceRecoverySourceEligible, workspaceSourceBindingFailure } from "./workspace-recovery-source.js"; import { preserveWorkspaceSnapshot } from "./workspace-recovery-snapshot.js"; @@ -160,14 +160,24 @@ export async function recoverRetainedWorkspace(input: Source & { db: Db; environ const { db } = input; await adoptSystemRecovery(db, input); const initial = await loadSystemRecovery(db, input); - if (!initial || initial.state === "complete" || (initial.state === "exhausted" && !canAdoptNativeSourceRetention(initial)) || + if (initial?.state === "exhausted" && !canAdoptNativeSourceRetention(initial)) { + await handOffExhaustedSystemRecovery(db, input, initial); + return; + } + if (!initial || initial.state === "complete" || (initial.nextAttemptAt !== null && initial.nextAttemptAt > Date.now())) return; const runAttempt = async (ownership: { token: string; assertHeld(): Promise }) => { let current = await loadSystemRecovery(db, input); if (!current) return; if (current.state === "exhausted") { + const spent = current; current = await retainNativeRecoverySource(db, input, current, ownership); - if (!current) return; + if (!current) { + // Not adoptable now. The board owns the spent budget until Retry or a + // later adoption returns it to the system. + await handOffExhaustedSystemRecovery(db, input, (await loadSystemRecovery(db, input)) ?? spent); + return; + } } const reserved = reserveSystemRecoveryAttempt(current); if (!reserved) {