diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index a37f943e065..9f05c54d926 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2652,7 +2652,12 @@ Retained-file recovery verifies the recorded source strictly, including the reusable-lease chain of its sandbox. When that verification refuses the source permanently (`allocation_ownership_ambiguous`, `source_binding_invalid`, `source_unavailable` or `historical_source_missing`), retrying repeats the -refusal. The recovery worker then continues the task from its last committed +refusal. This applies to export holds and to the no-replay hold of a native run +that failed before provider work (`native_continuation_requires_reconciliation`, +recorded resolved with outcome `blocked`); holds saved by earlier releases are +discovered without migration. Its Blocked receipt is the recorded +`nativeFailureBlock` status version. The refusal text never asks anyone to +wait for a Paperclip update. The recovery worker then continues the task from its last committed workspace save, with the named outcome `retained_source_unverifiable_continued_from_last_saved`. The unverified files are never applied. When the retained root can still be read, the worker copies it @@ -5182,7 +5187,15 @@ and invalidate the session when the selected keys, versions and bytes are unchan For per-turn direct sandbox recovery, the executor reads the retained Runner's suspended lifecycle through the recorded sandbox lease and validates its complete -identity and lack of pending delivery/cleanup. Only a lifecycle projection is +identity and lack of pending delivery/cleanup. The read uses the directory the +retained Runner's own lease recorded. A workspace can move to a new allocation of +the same sandbox, for example after its previous epoch was abandoned and the last +save was restored into a fresh root. That root is a sibling under the sandbox's +`.paperclip-workspaces` with the same execution-workspace ID, and the read still +binds to the source's directory. Such a moved source must be suspended; a Runner +left in the old allocation is never adopted live. Any other directory (another +workspace, another parent, a non-canonical path) stays refused with +`native_controller_recovery_missing_evidence:retained_sandbox_binding`. Only a lifecycle projection is cached locally; provider files remain on the execution host. Prepared tasks use their existing `read_state` contract. The normal snapshot rebinder, tool-contract checks and current admission authority still govern the next turn. diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 49984b10497..56c2e491475 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -1,5 +1,6 @@ import { execFile, spawn } from "node:child_process"; -import { nativeProviderRecoveryEvidence, nativeSessionFailureSourceCode, verifyRemoteRunnerReattachment } from "./native-session-executor.js"; +import { nativeProviderRecoveryEvidence, nativeSessionFailureSourceCode, restoreRemoteSuspendedRunnerCheckpoint, + verifyRemoteRunnerReattachment } from "./native-session-executor.js"; import { readRemoteRunnerRecoveryError } from "./remote-runner-recovery-diagnostics.js"; import { nativeToolContractFingerprintForTarget, rebindNativeSessionCheckpoint } from "./native-session-resume.js"; import { createHash, randomUUID } from "node:crypto"; @@ -317,7 +318,7 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "successor", runIds: [runId] })).toEqual([]); await db.update(nativeRunFinalizations).set({ phase: "committed", leaseOwner: null, leaseExpiresAt: null }) .where(eq(nativeRunFinalizations.runId, fixture.scope.runId)); - await expect(readSandbox()).resolves.toBeUndefined(); + await expect(readSandbox()).resolves.toEqual({ remoteCwd: "/workspace" }); await expect(inspect({ readControlState: () => ({ identity, commands: [{ status: "pending" }], committedEvents: [{ eventType: "run.terminal", envelope: identity }] }) })).rejects.toThrow(denied); const recovered = await inspect(); @@ -329,6 +330,112 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d expect(await claimNativeSessionResumptions({ db, runnerInstanceId: "duplicate", runIds: [runId], now: deadline })).toEqual([]); }); + // The Nov 10 staging shape: a preparation-only run's epoch was abandoned, so + // the warm follow-up restored the last save into a new allocation of the same + // sandbox. The committed source's Runner state stays in its own directory. + it("binds a retained Runner read to the source's directory after the workspace moved within the same sandbox", async () => { + const fixture = await seed(), environmentId = randomUUID(), workspaceId = randomUUID(); + const sourceLeaseId = randomUUID(), currentLeaseId = randomUUID(), providerLeaseId = randomUUID(); + const allocations = "/home/daytona/paperclip-workspace/.paperclip-workspaces"; + const sourceCwd = `${allocations}/${workspaceId}-1-${randomUUID()}`; + const movedCwd = `${allocations}/${workspaceId}-6-${currentLeaseId}`; + await db.insert(environments).values({ id: environmentId, name: `Moved ${environmentId}`, driver: "sandbox" }); + // Saved by the source run (status expired after its reusable handoff). + await db.insert(environmentLeases).values({ id: sourceLeaseId, companyId, environmentId, issueId: fixture.scope.issueId, + heartbeatRunId: fixture.scope.runId, provider: "daytona", providerLeaseId, status: "expired", + leasePolicy: "reuse_by_environment", metadata: { remoteCwd: sourceCwd } }); + const target = { kind: "remote", transport: "sandbox", environmentId, leaseId: currentLeaseId, providerKey: "daytona", + remoteCwd: movedCwd, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId } } as const; + const readSandbox = (override: Record = {}) => assertNativeRetainedSandboxBinding(db, + { ...fixture.scope, target: { ...target, ...override } as never }); + + await expect(readSandbox()).resolves.toEqual({ remoteCwd: sourceCwd }); + await expect(readSandbox({ remoteCwd: sourceCwd })).resolves.toEqual({ remoteCwd: sourceCwd }); + // A root published under the exact workspace ID is the same workspace too. + await expect(readSandbox({ remoteCwd: `${allocations}/${workspaceId}` })).resolves.toEqual({ remoteCwd: sourceCwd }); + const exactRoot = `${allocations}/${workspaceId}`; + await db.update(environmentLeases).set({ metadata: { remoteCwd: exactRoot } }).where(eq(environmentLeases.id, sourceLeaseId)); + await expect(readSandbox()).resolves.toEqual({ remoteCwd: exactRoot }); + await db.update(environmentLeases).set({ metadata: { remoteCwd: sourceCwd } }).where(eq(environmentLeases.id, sourceLeaseId)); + // Strict checks stay strict: another workspace's allocation, another + // parent, a non-canonical path or name, or another sandbox never bind. + for (const remoteCwd of [`${allocations}/${randomUUID()}-6-${currentLeaseId}`, `/elsewhere/${workspaceId}-6-${currentLeaseId}`, + `${allocations}/../.paperclip-workspaces/${workspaceId}-6-${currentLeaseId}`, `${allocations}/${workspaceId}-0-${currentLeaseId}`, + `${allocations}/${workspaceId}-6-${currentLeaseId}-extra`, "/unrelated"]) { + await expect(readSandbox({ remoteCwd })).rejects.toThrow("retained_sandbox_binding"); + } + await expect(readSandbox({ sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: randomUUID() } })) + .rejects.toThrow("retained_sandbox_binding"); + }); + + // The same move, driven through the changed read with sandbox confinement: + // the Runner command may touch only paths inside the sandbox's workspace base. + it.each(["suspended", "ready"] as const)("reads a moved source's %s Runner state from its own directory through a confined sandbox", async lifecycle => { + const fixture = await seed(), environmentId = randomUUID(), workspaceId = randomUUID(), runId = randomUUID(); + const providerLeaseId = randomUUID(), base = join(root, `sandbox-${runId}`, "paperclip-workspace"); + const sourceCwd = join(base, ".paperclip-workspaces", `${workspaceId}-1-${randomUUID()}`); + const movedCwd = join(base, ".paperclip-workspaces", `${workspaceId}-6-${randomUUID()}`); + await mkdir(movedCwd, { recursive: true }); + await db.insert(environments).values({ id: environmentId, name: `Confined ${environmentId}`, driver: "sandbox" }); + await db.insert(environmentLeases).values({ companyId, environmentId, issueId: fixture.scope.issueId, + heartbeatRunId: fixture.scope.runId, provider: "daytona", providerLeaseId, status: "expired", + leasePolicy: "reuse_by_environment", metadata: { remoteCwd: sourceCwd } }); + const identity = { runId: fixture.scope.runId, normalizedSessionId: fixture.scope.normalizedSessionId, + runnerInstanceId: fixture.native.runnerInstanceId, environmentLeaseId: fixture.native.environmentLeaseId, + turnId: fixture.native.turnId, itemId: fixture.native.itemId }; + const runnerDirectory = (cwd: string) => join(cwd, ".paperclip-runtime", "paperclip-runner", "sessions", + createHash("sha256").update(fixture.scope.normalizedSessionId).digest("hex"), "runner"); + await mkdir(runnerDirectory(sourceCwd), { recursive: true }); + await writeFile(join(runnerDirectory(sourceCwd), "runner-state.json"), JSON.stringify({ schema: "paperclip.runner.durable.state.v1", + ...identity, lifecycle, outbox: [], pendingTerminalDelivery: null, pendingProviderCleanup: null, warmTransition: null })); + const commands: string[] = []; + const execute = (request: { command: string; args?: string[] }) => { + const script = (request.args ?? []).join(" "); + commands.push(script); + // Provider confinement: nothing outside the sandbox's workspace base. + const paths = script.match(/'(\/[^']*)'/g)?.map(path => path.slice(1, -1)) ?? []; + if (paths.some(path => !path.startsWith(`${base}/`))) { + return Promise.resolve({ stdout: "", stderr: "outside sandbox", exitCode: 126, timedOut: false }); + } + return new Promise<{ stdout: string; stderr: string; exitCode: number; timedOut: boolean }>(resolve => { + execFile(request.command, request.args ?? [], { cwd: movedCwd }, (error, stdout, stderr) => resolve({ stdout, stderr, + exitCode: error ? typeof error.code === "number" ? error.code : 1 : 0, timedOut: false })); + }); + }; + const execution = parseNativeExecutionInput({ + schema: "paperclip.native-execution-input.v1", provider: { kind: "codex", model: null }, + binding: { companyId, agentId, issueId: fixture.scope.issueId, runId, executionWorkspaceId: workspaceId }, + task: { identifier: "NCD-1", title: "Continue retained work", description: null, + prompt: "Continue the saved work.", workMode: "standard" }, + workspace: { cwd: movedCwd, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: fixture.scope.normalizedSessionId, driverKind: "codex_app_server", + protocolVersion: 1, lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 } }, + completionContract: { id: randomUUID(), sha256: "fixture", schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "1", objective: "Continue the saved work", criteria: [{ id: "objective", requirement: "Save the result" }] } }, + credentialBindings: [], interactionResponses: [], + }); + // The controller already restored the committed source's durable checkpoint. + const controller = join(root, `controller-${runId}`); + await mkdir(join(controller, "control-plane"), { recursive: true }); + await writeFile(join(controller, "control-plane", "control-plane-state.json"), + JSON.stringify({ schema: "paperclip.runner.durable.control-plane-state.v2", identity })); + const target = { kind: "remote", transport: "sandbox", environmentId, leaseId: randomUUID(), providerKey: "daytona", + remoteCwd: movedCwd, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId }, runner: { execute } } as const; + const restore = () => restoreRemoteSuspendedRunnerCheckpoint({ db, execution, runnerExecutionTarget: target as never }, controller); + + if (lifecycle === "suspended") { + await restore(); + expect(commands.some(script => script.includes(runnerDirectory(sourceCwd)))).toBe(true); + expect(commands.some(script => script.includes(movedCwd))).toBe(false); + expect(JSON.parse(await readFile(join(controller, "plugin-runner-state.json"), "utf8"))) + .toMatchObject({ lifecycle: "suspended", ...identity }); + } else { + // A Runner left in the abandoned allocation is never adopted live. + await expect(restore()).rejects.toThrow("native_controller_recovery_missing_evidence:runner_not_settled"); + await expect(readFile(join(controller, "plugin-runner-state.json"))).rejects.toMatchObject({ code: "ENOENT" }); + } + }); + it.each(["live-owner", "expired-owner", "provider-attempt", "stopped", "operator-stop"])("does not publish an admission wait after authority changes: %s", async scenario => { const fixture = await seed(), runId = randomUUID(); const scope = { ...fixture.scope, runId }; diff --git a/server/src/services/native-runtime/native-controller-recovery.ts b/server/src/services/native-runtime/native-controller-recovery.ts index 1b27be232a5..91cad55c06f 100644 --- a/server/src/services/native-runtime/native-controller-recovery.ts +++ b/server/src/services/native-runtime/native-controller-recovery.ts @@ -1,3 +1,4 @@ +import { posix } from "node:path"; import { and, eq, inArray, ne, or, sql } from "drizzle-orm"; import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, nativeRunnerCheckpoints, type Db } from "@paperclipai/db"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; @@ -121,12 +122,35 @@ export async function deferNativeControllerRecovery(db: Db, input: { }); } -/** Bind a read of retained Runner lifecycle to the surviving sandbox. The - * ordinary admission owner still checks current permissions before execution. */ +/** The retained Runner keeps its state under the directory its own lease + * recorded. A workspace can move to a new allocation of the SAME sandbox: its + * prior epoch was abandoned, so the last save was restored into a fresh root + * named for the new lease. Both roots are canonical siblings under the + * sandbox's `.paperclip-workspaces` and carry the same execution-workspace ID. + * Anything else (another workspace, another parent, a non-canonical path) is + * not the retained allocation and stays refused. */ +function sameSandboxWorkspaceAllocation(recorded: string, current: string): boolean { + if (recorded === current) return true; + const canonical = (path: string) => posix.isAbsolute(path) && posix.normalize(path) === path && !path.endsWith("/"); + if (!canonical(recorded) || !canonical(current)) return false; + const parent = posix.dirname(recorded); + if (posix.basename(parent) !== ".paperclip-workspaces" || posix.dirname(current) !== parent) return false; + const workspace = WORKSPACE_ALLOCATION.exec(posix.basename(recorded))?.[1]; + return Boolean(workspace && WORKSPACE_ALLOCATION.exec(posix.basename(current))?.[1] === workspace); +} + +/** The published allocation names (remote-workspaces publishedWorkspaceDirectory): + * exactly ``, or `--`. */ +const UUID = "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"; +const WORKSPACE_ALLOCATION = new RegExp(`^(${UUID})(?:-[1-9][0-9]*-${UUID})?$`); + +/** Bind a read of retained Runner lifecycle to the surviving sandbox and + * return the directory that Runner wrote under. The ordinary admission owner + * still checks current permissions before execution. */ export async function assertNativeRetainedSandboxBinding(db: Db, input: { companyId: string; issueId: string; runId: string; target: Extract; -}): Promise { +}): Promise<{ remoteCwd: string }> { const { target } = input; if (target.transport !== "sandbox" || target.sandboxLeaseAcquisition?.outcome !== "resumed" || !target.providerKey || !target.environmentId || !target.leaseId) { @@ -139,8 +163,9 @@ export async function assertNativeRetainedSandboxBinding(db: Db, input: { eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.issueId, input.issueId), eq(environmentLeases.heartbeatRunId, input.runId), eq(environmentLeases.environmentId, target.environmentId), eq(environmentLeases.provider, target.providerKey), eq(environmentLeases.providerLeaseId, target.sandboxLeaseAcquisition.providerLeaseId))); - if (!owner || !owner.resultId || owner.recoveryState - || leases.length !== 1 || leases[0].metadata?.remoteCwd !== target.remoteCwd + const recordedCwd = leases.length === 1 ? leases[0].metadata?.remoteCwd : undefined; + if (!owner || !owner.resultId || owner.recoveryState || typeof recordedCwd !== "string" + || !sameSandboxWorkspaceAllocation(recordedCwd, target.remoteCwd) || !["active", "released", "expired"].includes(leases[0].status)) { throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding"); } @@ -150,6 +175,7 @@ export async function assertNativeRetainedSandboxBinding(db: Db, input: { if (owner.leaseOwner || owner.phase !== "committed") { throw new Error("native_controller_recovery_missing_evidence:retained_sandbox_binding"); } + return { remoteCwd: recordedCwd }; } /** Only the existing bounded selector chooses history. Never search backwards diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index edab5a5afaa..664ff9b36bd 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6972,8 +6972,9 @@ async function refreshFinishedPerTurnRunnerdProjection( } /** The controller checkpoint and Runner lifecycle have separate authorities. - * Rehydrate a missing suspension receipt before choosing live warm adoption. */ -async function restoreRemoteSuspendedRunnerCheckpoint(input: { + * Rehydrate a missing suspension receipt before choosing live warm adoption. + * Exported for the provider-boundary regression only. */ +export async function restoreRemoteSuspendedRunnerCheckpoint(input: { db: Db; execution: NativeExecutionInput; runnerExecutionTarget?: AdapterExecutionTarget | null; }, root: string): Promise { @@ -6986,6 +6987,9 @@ async function restoreRemoteSuspendedRunnerCheckpoint(input: { || !durableIdentityMatchesSession(prior, input.execution) || prior.runId === input.execution.binding.runId || runnerdAuthorityLifecycle(root, prior) !== "absent") return; let state: Record; + // A Runner left in an abandoned allocation is never adopted live: the new + // execution restored the last save into its own root of the same sandbox. + let movedAllocation = false; if (target.runnerTask) { // The admitted task owner binds this read to the surviving physical task; // its suspended identity still belongs to the previous business turn. @@ -6994,14 +6998,16 @@ async function restoreRemoteSuspendedRunnerCheckpoint(input: { state = result.state; } else { if (!target.runner) throw new Error("native_controller_recovery_dependency_unavailable:runner_state_reader"); - await assertNativeRetainedSandboxBinding(input.db, { ...input.execution.binding, runId: prior.runId, target }); - const stateDirectory = posix.join(target.remoteCwd, ".paperclip-runtime", "paperclip-runner", "sessions", - createHash("sha256").update(nativeSessionKey(input.execution)).digest("hex"), "runner"); - state = await readRemoteRunnerState({ runner: target.runner, stateDirectory }); + // The retained Runner wrote under its own lease's directory, which differs + // from this target when the workspace moved within the same sandbox. + const retained = await assertNativeRetainedSandboxBinding(input.db, { ...input.execution.binding, runId: prior.runId, target }); + movedAllocation = retained.remoteCwd !== target.remoteCwd; + state = await readRemoteRunnerState({ runner: target.runner, + stateDirectory: remoteRunnerStateDirectory(retained.remoteCwd, nativeSessionKey(input.execution)) }); } // A running warm source keeps the existing live-adoption identity, idle, // controller-fencing and physical-process checks. Only suspension is cached. - if (input.execution.session.lifecyclePolicy.mode === "warm" && state.lifecycle !== "suspended") return; + if (!movedAllocation && input.execution.session.lifecyclePolicy.mode === "warm" && state.lifecycle !== "suspended") return; if (state.lifecycle !== "suspended" || !Array.isArray(state.outbox) || state.outbox.length !== 0 || state.pendingTerminalDelivery != null || state.pendingProviderCleanup != null || state.warmTransition != null) { throw new Error("native_controller_recovery_missing_evidence:runner_not_settled"); diff --git a/server/src/services/system-recovery-state.test.ts b/server/src/services/system-recovery-state.test.ts index f57f28cac11..0be11845f5b 100644 --- a/server/src/services/system-recovery-state.test.ts +++ b/server/src/services/system-recovery-state.test.ts @@ -71,7 +71,10 @@ describe("durable system recovery budget", () => { const text = exhaustedSystemRecoveryAction(refused); expect(text).toContain("Paperclip refused to recover the saved files"); expect(text).toContain("Paperclip cannot prove which runs used the sandbox that holds the saved files (allocation_ownership_ambiguous)"); - expect(text).toContain("Retry runs the check again and succeeds only after Paperclip is updated"); + expect(text).toContain("Retry runs the check again. The agent's result is saved"); + // Recovery never waits on a person or a release, and never promises a + // fallback the hold may not get. + expect(text).not.toMatch(/succeeds only after|Paperclip is updated|last saved version/); expect(text).not.toMatch(/after automatic retries|stopped after|Retry to try again/); const once = failSystemRecovery(reserveSystemRecoveryAttempt(createSystemRecovery(randomUUID()), 0)!, diff --git a/server/src/services/system-recovery-state.ts b/server/src/services/system-recovery-state.ts index abf58eca9a9..2e7d99a1865 100644 --- a/server/src/services/system-recovery-state.ts +++ b/server/src/services/system-recovery-state.ts @@ -170,9 +170,11 @@ export function exhaustedSystemRecoveryAction(state: SystemRecoveryState, eviden const cause = state.failure?.cause ? `, cause: ${state.failure.cause.label}` : ""; if (isDeterministicRecoveryRefusal(state)) { const code = state.failure!.code; + // Never promise a release. A retained-file hold the fallback settles + // (workspace-recovery-fallback) replaces this text with its own outcome. return `Paperclip refused to recover the saved files: ${recoveryRefusalReasons[code] ?? "a safety check on the saved files did not pass"} ` + `(${code}${cause}). Automatic recovery does not repeat this check, because the same saved state gives the same result. ` - + "Retry runs the check again and succeeds only after Paperclip is updated. The agent's result is saved, and the agent will not repeat its work."; + + "Retry runs the check again. The agent's result is saved, and the agent will not repeat its work."; } const reason = state.failure?.code ? ` (last failure: ${state.failure.code}${cause})` : ""; const outcome = state.attempts > 1 ? `stopped after ${state.attempts} attempts` : "failed"; diff --git a/server/src/services/workspace-recovery-fallback-discovery.ts b/server/src/services/workspace-recovery-fallback-discovery.ts index 23cc1e26f51..5c679e584f3 100644 --- a/server/src/services/workspace-recovery-fallback-discovery.ts +++ b/server/src/services/workspace-recovery-fallback-discovery.ts @@ -7,6 +7,36 @@ import { isExhaustedStopProofWait, STOP_PROOF_MAX_REARMS, STOP_PROOF_RECOVERY_KE export const UNVERIFIABLE_SOURCE_CODES = ["allocation_ownership_ambiguous", "source_binding_invalid", "source_unavailable", "historical_source_missing"] as const; export const PERMANENT_SOURCE_RETRY_CLASSES = ["service_required", "use_alternative_strategy"] as const; export const RETAINED_SOURCE_FALLBACK_CHECK_KEY = "retainedSourceFallbackCheck"; +/** Holds of an accepted result whose copy-back could not finish. */ +export const EXPORT_HOLD_CAUSES = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"] as const; +/** A native run that failed before provider work keeps its files under a + * no-replay hold. It is recorded resolved-but-blocked: replay stays blocked + * until its files are recovered, or the task continues from the last save. */ +export const NATIVE_NO_REPLAY_HOLD_CAUSE = "native_continuation_requires_reconciliation"; + +type HoldAction = { cause: string; status: string; outcome: string | null; evidence: Record }; +const field = (value: unknown, key: string) => value && typeof value === "object" ? (value as Record)[key] : undefined; + +/** Whether this recovery action is a retained-file hold the fallback may settle. */ +export function isRetainedFileHold(action: HoldAction) { + if ((EXPORT_HOLD_CAUSES as readonly string[]).includes(action.cause)) return ["active", "escalated"].includes(action.status); + const automatic = action.evidence.automaticRecovery; + return action.cause === NATIVE_NO_REPLAY_HOLD_CAUSE + && (["active", "escalated"].includes(action.status) || (action.status === "resolved" && action.outcome === "blocked")) + && field(automatic, "policy") === "preserve_without_replay_v1" && field(automatic, "replay") === "blocked" + && typeof action.evidence.runId === "string" && field(automatic, "runId") === action.evidence.runId; +} + +/** SQL form of isRetainedFileHold. */ +function retainedFileHoldCondition() { + return sql`((${issueRecoveryActions.cause} in (${list(EXPORT_HOLD_CAUSES)}) and ${issueRecoveryActions.status} in ('active', 'escalated')) + or (${issueRecoveryActions.cause} = ${NATIVE_NO_REPLAY_HOLD_CAUSE} + and (${issueRecoveryActions.status} in ('active', 'escalated') + or (${issueRecoveryActions.status} = 'resolved' and ${issueRecoveryActions.outcome} = 'blocked')) + and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,policy}' = 'preserve_without_replay_v1' + and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked' + and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,runId}' = ${issueRecoveryActions.evidence}->>'runId'))`; +} /** Whether an exhausted hold may continue from the last save: its source is * refused permanently, or its stop stayed unproven through every re-arm. */ @@ -77,25 +107,24 @@ const leaseChangedSinceCheck = sql`(jsonb_typeof(${check}->'checkedAt') = 'numbe * Retry or a new generation reconsiders it). */ export function retainedSourceFallbackPendingCondition() { - return sql`(${issueRecoveryActions.status} in ('active', 'escalated') + return sql`((${issueRecoveryActions.status} in ('active', 'escalated') or ${retainedFileHoldCondition()}) and ${issueRecoveryActions.evidence}#>>'{systemRecovery,state}' = 'exhausted' and ${issueRecoveryActions.evidence}->'retainedSourceFallback' is null - and ((${issueRecoveryActions.cause} in ('native_workspace_sync_out_unrecoverable', 'native_workspace_sync_out_retry_exhausted') + and ((${retainedFileHoldCondition()} and ${issueRecoveryActions.evidence}#>>'{systemRecovery,failure,code}' in (${list(UNVERIFIABLE_SOURCE_CODES)}) and ${issueRecoveryActions.evidence}#>>'{systemRecovery,failure,retryClass}' in (${list(PERMANENT_SOURCE_RETRY_CLASSES)}) and exists (select 1 from heartbeat_runs source where source.company_id = ${issueRecoveryActions.companyId} and source.id::text = ${issueRecoveryActions.evidence}->>'runId' and source.runner_profile_json#>>'{remoteWorkspace,state}' = 'prepared')) - or (${issueRecoveryActions.cause} in ('native_workspace_sync_out_unrecoverable', 'native_workspace_sync_out_retry_exhausted') - and ${exhaustedStopProofWait} and ${stopProofRearms} >= ${STOP_PROOF_MAX_REARMS} + or (${retainedFileHoldCondition()} and ${exhaustedStopProofWait} and ${stopProofRearms} >= ${STOP_PROOF_MAX_REARMS} and exists (select 1 from heartbeat_runs source where source.company_id = ${issueRecoveryActions.companyId} and source.id::text = ${issueRecoveryActions.evidence}->>'runId' and source.runner_profile_json#>>'{remoteWorkspace,state}' = 'prepared')) - or exists (select 1 from issue_recovery_actions sibling + or (${issueRecoveryActions.status} in ('active', 'escalated') and exists (select 1 from issue_recovery_actions sibling where sibling.company_id = ${issueRecoveryActions.companyId} and sibling.source_issue_id = ${issueRecoveryActions.sourceIssueId} and sibling.id <> ${issueRecoveryActions.id} and sibling.evidence->>'runId' = ${issueRecoveryActions.evidence}->>'runId' and sibling.evidence#>>'{systemRecovery,state}' = 'complete' - and sibling.evidence#>>'{workspaceRecovery,status}' = 'recovered')) + and sibling.evidence#>>'{workspaceRecovery,status}' = 'recovered'))) and (${check} is null or ${check}->>'generationId' is distinct from ${issueRecoveryActions.evidence}#>>'{systemRecovery,generationId}' or ${leaseChangedSinceCheck} diff --git a/server/src/services/workspace-recovery-fallback.test.ts b/server/src/services/workspace-recovery-fallback.test.ts index 931eb3e42b2..82e18edbe48 100644 --- a/server/src/services/workspace-recovery-fallback.test.ts +++ b/server/src/services/workspace-recovery-fallback.test.ts @@ -319,6 +319,82 @@ describe("retained source fallback", () => { expect.arrayContaining([expect.objectContaining({ action: expect.objectContaining({ id: f.actionId }) })])); }, 60_000); + /** + * The Nov 10 staging shape: a native run failed before provider work, so its + * prepared files sit under a no-replay hold that is recorded resolved but + * blocked. Its reusable sandbox history made the source unverifiable + * (allocation_ownership_ambiguous) and the task stayed blocked on a Retry + * that could never pass. The worker continues it from the last save. + */ + it("continues a resolved-but-blocked native no-replay hold from the last save, discovered by the worker", async () => { + const f = await unverifiableRetainedSource(); + const source = await run(f.runId); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", resultId: null, recoveryState: "blocked", + failureCode: "native_continuation_requires_reconciliation" }).where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(heartbeatRuns).set({ errorCode: "native_continuation_requires_reconciliation", + error: "native_controller_recovery_missing_evidence:retained_sandbox_binding", + resultJson: { ...source.resultJson, executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }) + .where(eq(heartbeatRuns.id, f.runId)); + const [task] = await db.update(issues).set({ status: "blocked", lastStatusDecisionId: null }) + .where(eq(issues.id, f.issueId)).returning(); + const held = await action(f.actionId); + const refused = "Paperclip refused to recover the saved files: Paperclip cannot prove which runs used the sandbox that holds the saved files " + + "(allocation_ownership_ambiguous). Automatic recovery does not repeat this check, because the same saved state gives the same result. " + + "Retry runs the check again and succeeds only after Paperclip is updated. The agent's result is saved, and the agent will not repeat its work."; + // The exact row the Nov 10 release saved: resolved with outcome blocked, + // replay blocked, and the task's Blocked status version recorded. + await db.update(issueRecoveryActions).set({ kind: "active_run_watchdog", cause: "native_continuation_requires_reconciliation", + status: "resolved", outcome: "blocked", resolvedAt: new Date(), ownerType: "board", nextAction: refused, + evidence: { ...held.evidence, + automaticRecovery: { runId: f.runId, policy: "preserve_without_replay_v1", replay: "blocked", + recordedAt: new Date().toISOString(), actionOutcome: "unknown" }, + nativeFailureBlock: { runId: f.runId, statusVersion: Number(task.statusVersion) }, + workspaceRestoreRecovery: { schema: "paperclip.workspace-restore-recovery.v1", leaseIds: [f.lease.id] }, + workspaceRestoreFailure: "restore_failed", continuationDelivery: "invalidated", + workspaceRecovery: { ...(held.evidence.workspaceRecovery as object), message: refused } } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + + expect(await nextWorkspaceRecoveryAt(db)).toBeLessThanOrEqual(Date.now()); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => undefined, true) ?? []); + + const settled = await action(f.actionId); + expect(settled).toMatchObject({ status: "resolved", outcome: "restored", ownerType: "system", + resolutionNote: RETAINED_SOURCE_FALLBACK_OUTCOME, returnOwnerAgentId: source.agentId }); + expect(settled.evidence.continuationDelivery).toBe("pending"); + expect(settled.evidence.automaticRecovery).toBeUndefined(); + for (const text of [settled.nextAction, readWorkspaceRecovery(settled.evidence)?.message]) + expect(String(text)).not.toMatch(/succeeds only after|Paperclip is updated/); + expect(readRemoteWorkspaceAuthority((await workspace(f.workspaceId)).metadata)).toMatchObject({ writerRunId: null, dirty: false, + location: null, snapshotId: f.snapshotId }); + // The hold owned the Blocked status, so the task is ready again. + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("todo"); + // Never a second executor: the abandoned source is never replaced. + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(coordinator.failureDetail).toMatchObject({ replacementDenied: RETAINED_SOURCE_FALLBACK_OUTCOME }); + const wake = vi.fn(async () => ({ id: randomUUID() })); + await deliverReconciledExecutions(db, wake as never); + await deliverReconciledExecutions(db, wake as never); + expect(wake.mock.calls.filter(call => (call as unknown[])[0] === source.agentId)).toHaveLength(1); + expect((await retainedWorkspaceCandidates(db, undefined, false, true)).map(row => row.action.id)).not.toContain(f.actionId); + }, 60_000); + + it("does not settle a no-replay hold whose task moved on to a newer Blocked decision", async () => { + const f = await unverifiableRetainedSource(); + const [task] = await db.update(issues).set({ status: "blocked" }).where(eq(issues.id, f.issueId)).returning(); + const held = await action(f.actionId); + await db.update(issueRecoveryActions).set({ kind: "active_run_watchdog", cause: "native_continuation_requires_reconciliation", + status: "resolved", outcome: "blocked", resolvedAt: new Date(), evidence: { ...held.evidence, + automaticRecovery: { runId: f.runId, policy: "preserve_without_replay_v1", replay: "blocked" }, + // A newer status decision replaced the Blocked status this run's failure set. + nativeFailureBlock: { runId: f.runId, statusVersion: Number(task.statusVersion) - 1 } } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await recoverRetainedWorkspace(f.input); + const settled = await action(f.actionId); + expect(settled).toMatchObject({ resolutionNote: RETAINED_SOURCE_FALLBACK_OUTCOME }); + expect(settled.evidence.continuationDelivery).toBe("not_requested"); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0].status).toBe("blocked"); + }, 60_000); + it("waits until a deferred transient check is due before the worker visits the hold again", async () => { const f = await unverifiableRetainedSource(); const source = await run(f.runId); diff --git a/server/src/services/workspace-recovery-fallback.ts b/server/src/services/workspace-recovery-fallback.ts index 00278498886..82f1a86682f 100644 --- a/server/src/services/workspace-recovery-fallback.ts +++ b/server/src/services/workspace-recovery-fallback.ts @@ -26,7 +26,8 @@ import { abandonUnverifiedRemoteWorkspaceEpoch } from "./remote-workspace-epoch- import { hasStopOnlyCleanup, prepareHistoricalSandboxStopVerification, prepareSandboxStopAndRetain, restoreSandboxStopAfterFailedRestart, settleStopOnlyCleanup } from "./sandbox-stop-and-retain.js"; import { nativeWorkspaceFinalizationBlockSchema } from "./native-runtime/preserved-native-result-finalization.js"; import { withNativeWorkspaceFinalizationOwnership } from "./native-runtime/native-workspace-finalization-ownership.js"; -import { PERMANENT_SOURCE_RETRY_CLASSES, RETAINED_SOURCE_FALLBACK_CHECK_KEY, UNVERIFIABLE_SOURCE_CODES, retainedHoldFallbackEligible } from "./workspace-recovery-fallback-discovery.js"; +import { PERMANENT_SOURCE_RETRY_CLASSES, RETAINED_SOURCE_FALLBACK_CHECK_KEY, UNVERIFIABLE_SOURCE_CODES, isRetainedFileHold, + retainedHoldFallbackEligible } from "./workspace-recovery-fallback-discovery.js"; import { captureReadOnlyWorkspaceArchive, sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; import { restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; import { readSystemRecovery, type SystemRecoveryState } from "./system-recovery-state.js"; @@ -44,7 +45,6 @@ export const RETAINED_SOURCE_FALLBACK_SCHEMA = "paperclip.retained-source-fallba export const SIBLING_WORKSPACE_RECOVERY_SCHEMA = "paperclip.sibling-workspace-recovery.v1" as const; export const RETAINED_SOURCE_QUARANTINE_PHASE = "workspace_recovery_quarantine"; -const exportCauses = ["native_workspace_sync_out_unrecoverable", "native_workspace_sync_out_retry_exhausted"]; const terminal = ["failed", "interrupted", "timed_out", "cancelled", "succeeded"]; // A check deferred for a transient reason runs again after this delay, a bounded number of times. const FALLBACK_RECHECK_MS = 5 * 60_000; @@ -113,7 +113,7 @@ async function inspectFallback(db: Db, input: Source, lock: boolean, requireStop if (!issue || !run || !action) return refuse("source_changed"); if (object(action.evidence.retainedSourceFallback).schema === RETAINED_SOURCE_FALLBACK_SCHEMA) return { kind: "settled" as const }; const system = readSystemRecovery(action.evidence); - if (!exportCauses.includes(action.cause) || !["active", "escalated"].includes(action.status) || action.evidence.runId !== run.id + if (!isRetainedFileHold(action) || action.evidence.runId !== run.id || action.evidence.executionReconciliation || system?.state !== "exhausted" || system.sourceRunId !== run.id || !retainedHoldFallbackEligible(action.evidence, system)) return refuse("not_unverifiable_source_hold"); if (run.runtimeMode !== "native" || !terminal.includes(run.status) || !run.finishedAt @@ -347,12 +347,16 @@ export async function continueFromLastSavedWorkspace(input: Source & { // failure that nothing changed since) still names it. Otherwise a newer // decision owns the block: it is kept and no continuation is started. const block = nativeWorkspaceFinalizationBlockSchema.safeParse(action.evidence.workspaceFinalizationBlock); + // A native no-replay hold records the Blocked status version its run's failure set. + const failureBlock = object(action.evidence.nativeFailureBlock); const blockedAt = issue.blockedTransitionAt?.getTime(); const ownsBlock = issue.status === "blocked" && (block.success ? block.data.issueId === issue.id && block.data.runId === run.id && block.data.statusVersion === Number(issue.statusVersion) && block.data.priorDecisionId === issue.lastStatusDecisionId - : action.evidence.workspaceFinalizationBlock === undefined && issue.lastStatusDecisionId === null && blockedAt !== undefined - && blockedAt >= run.finishedAt!.getTime() - 1_000 && blockedAt <= action.createdAt.getTime() + 60_000); + : action.evidence.nativeFailureBlock !== undefined + ? failureBlock.runId === run.id && failureBlock.statusVersion === Number(issue.statusVersion) + : action.evidence.workspaceFinalizationBlock === undefined && issue.lastStatusDecisionId === null && blockedAt !== undefined + && blockedAt >= run.finishedAt!.getTime() - 1_000 && blockedAt <= action.createdAt.getTime() + 60_000); const dispatchable = issue.status !== "blocked" || ownsBlock; const userText = quarantine.state === "captured" ? "Paperclip could not verify the previous run's unsaved files, so the task continues from the last saved version. Those files are attached to the task, unapplied."