diff --git a/packages/paperclip-runner/docs/durable-recovery.md b/packages/paperclip-runner/docs/durable-recovery.md index 4de69c15a29..5d021ed8172 100644 --- a/packages/paperclip-runner/docs/durable-recovery.md +++ b/packages/paperclip-runner/docs/durable-recovery.md @@ -227,6 +227,16 @@ protected launch argument fails with the closed classification controller never opens a replacement session after either class, whatever the checkpoint's recovery policy, and stops the run for an operator. +A remote session whose saved working directory is not the run's assigned +workspace root (its workspace epoch was abandoned, so the saved version was +restored into a new root) cannot resume there. The driver declines it before +any Runner or provider process starts. An idle session reports +`provider_state_incompatible`: a replacement opens in the assigned root when the +checkpoint allows it, and otherwise the run fails with +`native_provider_state_incompatible`. A session with an unfinished provider turn +or a pending runtime request reports `provider_side_effect_pending` and stops +for an operator, whatever its recovery policy. + A restart can kill the runner after it journaled a run's own cold `run.attach` and before the attachment finished. The restored runner settles that command as `execution_indeterminate` and never executes it again. When the journal shows no diff --git a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts index afaadd9defa..134c048e310 100644 --- a/packages/paperclip-runner/src/backends/native-backend-factory.test.ts +++ b/packages/paperclip-runner/src/backends/native-backend-factory.test.ts @@ -32,6 +32,7 @@ import { WORKSPACE, } from "../drivers/codex/codex-app-server-driver.test-support.js"; import { createNativeSessionBackend } from "../index.js"; +import { PROVIDER_SIDE_EFFECT_PENDING, PROVIDER_STATE_INCOMPATIBLE } from "../contracts/harness-driver.js"; import { createCodexNativeSessionBackend, createRunnerdNativeSessionBackend } from "./codex-native-backend.js"; import { createOpenCodeNativeSessionBackend } from "./opencode-native-backend.js"; @@ -409,9 +410,11 @@ describe("native backend factory", () => { // An abandoned remote workspace epoch is restored into a new sibling root // under `.paperclip-workspaces`, while the follow-up inherits a checkpoint - // that names the old root. Resume there is impossible, but it is an ordinary - // resume failure: the runtime's replacement policy must decide, not a thrown - // admission error that ends the run as native_session_interrupted. + // that names the old root. Resume there is impossible, and retrying cannot + // move the saved session: the runner reports provider state incompatible. + // The runtime then replaces the session when the checkpoint allows it, and + // otherwise ends the run with the permanent named code + // native_provider_state_incompatible, never a generic retryable failure. it.each([ ["Codex", execution()], ["ACPX Claude", acpxExecution("claude")], @@ -421,6 +424,7 @@ describe("native backend factory", () => { expect(recovered).toEqual({ recovered: false, reason: "persisted session working directory is not the assigned remote workspace", + failureClass: PROVIDER_STATE_INCOMPATIBLE, }); // No Runner or provider process is started for a session that cannot resume here. expect(factory).not.toHaveBeenCalled(); @@ -430,15 +434,38 @@ describe("native backend factory", () => { it.each([ ["an active turn", { activeTurnId: "turn-in-flight" }], ["a pending runtime request", { pendingRuntimeRequests: [{ requestId: "request-1" }] }], - ])("keeps refusing a moved-root session that holds %s", async (_label, inFlight) => { + ])("stops a moved-root session that holds %s for reconciliation, never replacing it", async (_label, inFlight) => { const { factory, backend, snapshot } = movedRootRecovery(acpxExecution("claude")); await expect(backend.recoverSession!({ ...snapshot, ...inFlight } as never, { signal: new AbortController().signal })) - .rejects.toThrow("Remote Codex working directory does not match the assigned workspace"); + .resolves.toEqual({ + recovered: false, + reason: "persisted session working directory is not the assigned remote workspace, and a provider turn or runtime request was in flight", + failureClass: PROVIDER_SIDE_EFFECT_PENDING, + }); + expect(factory).not.toHaveBeenCalled(); + }); + + it("treats a moved-root active turn that already has a terminal record as finished", async () => { + const { factory, backend, snapshot } = movedRootRecovery(acpxExecution("claude")); + await expect(backend.recoverSession!({ ...snapshot, activeTurnId: "turn-1", + terminalTurns: [{ turnId: "turn-1", fingerprint: "{\"status\":\"completed\"}" }] } as never, { signal: new AbortController().signal })) + .resolves.toMatchObject({ recovered: false, failureClass: PROVIDER_STATE_INCOMPATIBLE }); + expect(factory).not.toHaveBeenCalled(); + }); + + it("names the agent generically when a remote Claude session opens outside its workspace", async () => { + const { factory, backend } = movedRootRecovery(acpxExecution("claude")); + const failure = await backend.openSession({ + identity: { runId: "run", sessionId: "session", companyId: "company", issueId: "issue", agentId: "agent" }, + workingDirectory: "/home/daytona/paperclip-workspace/.paperclip-workspaces/elsewhere", + }).catch((error: unknown) => error); + // A Claude run must not be told its working directory is a Codex one. + expect((failure as Error).message).toBe("Remote agent working directory does not match the assigned workspace"); expect(factory).not.toHaveBeenCalled(); }); it.each([ - ["a filesystem root", "/", "Codex working directory cannot be a filesystem root"], + ["a filesystem root", "/", "Agent working directory cannot be a filesystem root"], ["an unnormalized path", "/home/daytona/../root", "normalized absolute path"], ])("keeps refusing a saved session whose working directory is %s", async (_label, workingDirectory, message) => { const { factory, backend, snapshot } = movedRootRecovery(acpxExecution("claude")); @@ -533,7 +560,7 @@ describe("native backend factory", () => { it.each([ ["thread read", { read: "/projects/other", resume: "/home/agent", reason: "provider read a different working directory" }], - ["thread resume", { read: "/home/agent", resume: "/projects/other", reason: "Error: Codex thread response changed the assigned working directory" }], + ["thread resume", { read: "/home/agent", resume: "/projects/other", reason: "Error: Provider thread response changed the assigned working directory" }], ])("does not recover when the %s reports another cwd", async (_label, cwds) => { const transport = new PreparedLaunchTransport(cwds.resume); transport.readResponse = { thread: { id: transport.threadId, sessionId: transport.providerSessionId, cwd: cwds.read, turns: [] } }; @@ -552,7 +579,7 @@ describe("native backend factory", () => { codexTransportFactory: () => new PreparedLaunchTransport("/home/agent"), workingDirectoryAuthority: "remote_runner", environment: {}, }); await expect(backend.openSession({ identity, workingDirectory: "/home/agent" })) - .rejects.toThrow("Remote Codex working directory requires an assigned workspace"); + .rejects.toThrow("Remote agent working directory requires an assigned workspace"); }); it("does not allow prepared launch authority without runnerd", () => { diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index 7be7eccfc9a..406523d3b5f 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -498,7 +498,9 @@ export interface PersistedHarnessSession { } /** Runner proof that saved provider state cannot be restored by this runner - * build. Restoring the same state again fails the same way. */ + * build, or in the workspace assigned to this run (for example, a remote + * workspace root that moved after its epoch was abandoned). Restoring the same + * state again fails the same way. */ export const PROVIDER_STATE_INCOMPATIBLE = "provider_state_incompatible"; /** Runner refused to attach a new run to the saved session (for example, a * protected launch argument or the durable provider profile would change). diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 52641c405ed..a006348de21 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -20,7 +20,7 @@ import type { PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; -import { HarnessReconciliationError, runnerRecoveryFailureClass } from "../../contracts/harness-driver.js"; +import { HarnessReconciliationError, PROVIDER_SIDE_EFFECT_PENDING, PROVIDER_STATE_INCOMPATIBLE, runnerRecoveryFailureClass } from "../../contracts/harness-driver.js"; import { CODEX_CODEX_PROTOCOL_VERSION, CODEX_SKILLLESS_BASE_INSTRUCTIONS, @@ -400,20 +400,35 @@ export class CodexAppServerDriver implements HarnessDriver { // in the newly assigned root. When it holds no in-flight turn or runtime // request (a follow-up's inherited checkpoint, for example), decline // before any Runner or provider process starts, so the runtime's recovery - // policy decides: a replacement in the assigned workspace, or a named - // refusal. In-flight provider work keeps the stricter admission refusal, - // because nothing here can prove its side effects settled. + // policy decides: a replacement in the assigned workspace, or the + // permanent named refusal when the checkpoint requires the same session. + // Retrying cannot move the saved session, so this is incompatible state, + // not a transient failure. A session with an in-flight turn or runtime + // request may have an external effect nothing here can prove settled, so + // it is never replaced: it stops by name for an operator to reconcile. if ( snapshot.workingDirectory && - !snapshot.activeTurnId && - !(snapshot.pendingRuntimeRequests?.length) && this.#options.workingDirectoryAuthority === "remote_runner" && remoteRunnerWorkingDirectoryMoved(snapshot.workingDirectory, this.#options.environment) ) { - return { - recovered: false, - reason: "persisted session working directory is not the assigned remote workspace", - }; + // A crash can save a turn's terminal record before clearing its active + // marker; that turn finished and is not in flight. + const activeTurnIsTerminal = (snapshot.terminalTurns ?? []).some( + (terminal) => terminal.turnId === snapshot.activeTurnId, + ); + const inFlight = (Boolean(snapshot.activeTurnId) && !activeTurnIsTerminal) + || Boolean(snapshot.pendingRuntimeRequests?.length); + return inFlight + ? { + recovered: false, + reason: "persisted session working directory is not the assigned remote workspace, and a provider turn or runtime request was in flight", + failureClass: PROVIDER_SIDE_EFFECT_PENDING, + } + : { + recovered: false, + reason: "persisted session working directory is not the assigned remote workspace", + failureClass: PROVIDER_STATE_INCOMPATIBLE, + }; } const transport = this.#transport({ workingDirectory: snapshot.workingDirectory diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts index 8904234549d..845eec9d141 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts @@ -550,7 +550,7 @@ describe("Codex app-server Codex driver", () => { await expect(driver.recoverSession?.(snapshot)).resolves.toEqual({ recovered: false, reason: expect.stringContaining( - "Codex working directory is outside the assigned workspace", + "Agent working directory is outside the assigned workspace", ), }); }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts index cd36389978f..67039c32797 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts @@ -53,7 +53,7 @@ export function validateCodexWorkingDirectory( authority: CodexWorkingDirectoryAuthority = "local_filesystem", ): string { if (workingDirectory.trim().length === 0) { - throw new Error("Codex working directory is required"); + throw new Error("Agent working directory is required"); } if (authority === "remote_runner") { return validateRemoteRunnerWorkingDirectory(workingDirectory, environment); @@ -66,24 +66,24 @@ export function validateCodexWorkingDirectory( try { resolved = realpathSync.native(requested); if (!statSync(resolved).isDirectory()) { - throw new Error("Codex working directory must be a directory"); + throw new Error("Agent working directory must be a directory"); } } catch (error) { const code = (error as NodeJS.ErrnoException).code; if (code === "ENOENT") { throw new Error( - "Codex working directory must exist before provider admission", + "Agent working directory must exist before provider admission", ); } throw error; } if (resolved === parse(resolved).root) { - throw new Error("Codex working directory cannot be a filesystem root"); + throw new Error("Agent working directory cannot be a filesystem root"); } const configuredRoot = environment.PAPERCLIP_WORKSPACE_CWD; const hostHome = canonicalConfiguredPath(environment.HOME); if (hostHome && pathContains(resolved, hostHome)) { - throw new Error("Codex working directory cannot contain the host HOME"); + throw new Error("Agent working directory cannot contain the host HOME"); } if ( hostHome && @@ -92,7 +92,7 @@ export function validateCodexWorkingDirectory( ) ) { throw new Error( - "Codex working directory cannot overlap sensitive host HOME state", + "Agent working directory cannot overlap sensitive host HOME state", ); } if ( @@ -101,7 +101,7 @@ export function validateCodexWorkingDirectory( (configuredRoot === undefined || configuredRoot.trim().length === 0) ) { throw new Error( - "Codex working directory inside the host HOME requires an assigned workspace", + "Agent working directory inside the host HOME requires an assigned workspace", ); } const codexHome = canonicalConfiguredPath(environment.CODEX_HOME); @@ -110,7 +110,7 @@ export function validateCodexWorkingDirectory( pathContains(resolved, codexHome) || pathContains(codexHome, resolved) ) { - throw new Error("Codex working directory cannot overlap host CODEX_HOME"); + throw new Error("Agent working directory cannot overlap host CODEX_HOME"); } } if (configuredRoot !== undefined && configuredRoot.trim().length > 0) { @@ -122,7 +122,7 @@ export function validateCodexWorkingDirectory( isAbsolute(pathFromRoot) ) { throw new Error( - "Codex working directory is outside the assigned workspace", + "Agent working directory is outside the assigned workspace", ); } } @@ -154,7 +154,7 @@ export function validatePreparedLaunchWorkingDirectory( ); } if (workingDirectory === posix.parse(workingDirectory).root) { - throw new Error("Codex working directory cannot be a filesystem root"); + throw new Error("Agent working directory cannot be a filesystem root"); } return workingDirectory; } @@ -170,10 +170,10 @@ export function codexThreadWorkingDirectory( authority: CodexWorkingDirectoryAuthority = "local_filesystem", ): string { if (authority !== "runner_prepared_launch") { - if (expected === null) throw new Error("Codex working directory is required"); + if (expected === null) throw new Error("Agent working directory is required"); const returned = typeof reported === "string" ? reported : expected; if (resolve(returned) !== expected) { - throw new Error("Codex thread response changed the assigned working directory"); + throw new Error("Provider thread response changed the assigned working directory"); } return expected; } @@ -182,7 +182,7 @@ export function codexThreadWorkingDirectory( } const returned = validatePreparedLaunchWorkingDirectory(reported); if (expected !== null && returned !== expected) { - throw new Error("Codex thread response changed the assigned working directory"); + throw new Error("Provider thread response changed the assigned working directory"); } return returned; } @@ -217,16 +217,16 @@ function validateRemoteRunnerWorkingDirectory( ): string { if (!isNormalizedRemotePath(workingDirectory)) { throw new Error( - "Remote Codex working directory must be a normalized absolute path", + "Remote agent working directory must be a normalized absolute path", ); } if (workingDirectory === posix.parse(workingDirectory).root) { - throw new Error("Codex working directory cannot be a filesystem root"); + throw new Error("Agent working directory cannot be a filesystem root"); } const configuredRoot = environment.PAPERCLIP_WORKSPACE_CWD?.trim(); if (!configuredRoot) { throw new Error( - "Remote Codex working directory requires an assigned workspace", + "Remote agent working directory requires an assigned workspace", ); } if ( @@ -242,7 +242,7 @@ function validateRemoteRunnerWorkingDirectory( // canonical identity inside the authoritative filesystem before launch. if (workingDirectory !== configuredRoot) { throw new Error( - "Remote Codex working directory does not match the assigned workspace", + "Remote agent working directory does not match the assigned workspace", ); } return workingDirectory; diff --git a/packages/paperclip-runner/src/live/runner-prepared-launch-workspace.test.ts b/packages/paperclip-runner/src/live/runner-prepared-launch-workspace.test.ts index 6112d7f87bb..cef217f8f7c 100644 --- a/packages/paperclip-runner/src/live/runner-prepared-launch-workspace.test.ts +++ b/packages/paperclip-runner/src/live/runner-prepared-launch-workspace.test.ts @@ -71,7 +71,7 @@ it("reproduces the controller-side remote cwd requirement for plugin-owned launc try { const { backend, prepareLaunch, launches } = await pluginBackend(root, root, "remote_runner"); await expect(backend.openSession({ identity: sessionIdentity, workingDirectory: "/controller/workspace" })) - .rejects.toThrow("Remote Codex working directory requires an assigned workspace"); + .rejects.toThrow("Remote agent working directory requires an assigned workspace"); expect(prepareLaunch).not.toHaveBeenCalled(); expect(launches).toEqual([]); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 0dfda06c42f..0f30e117452 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -6664,6 +6664,67 @@ describe("executeNativeSession recovery", () => { } }); + // A remote workspace epoch was abandoned, so the run was assigned a new + // sibling root while its checkpoint still names the old one. A checkpoint + // that requires the same session (an unfinished goal) ends with the + // permanent named code, never a generic retryable failure; a checkpoint that + // allows replacement continues in a fresh session in the assigned root. + it.each([ + ["same_session_only", false, false], + ["allow_replacement_after_governed_wait", false, false], + ["allow_replacement_after_resume_failure", true, false], + // An in-flight turn may have an unsettled external effect: never replaced, under any policy. + ["same_session_only", false, true], + ["allow_replacement_after_resume_failure", false, true], + ] as const)("handles a %s checkpoint whose remote workspace root moved (replaces=%s, in flight=%s)", async (providerRecoveryPolicy, replaces, inFlight) => { + const parent = "/home/daytona/paperclip-workspace/.paperclip-workspaces"; + const savedRoot = `${parent}/a5a34403-04fe-4e7e-97cf-d8f371a802a4-1-80214561-2151-4112-a43a-7d65f3071b56`; + const assignedRoot = `${parent}/a5a34403-04fe-4e7e-97cf-d8f371a802a4-5-ef79592c-6784-438f-8a17-9800dd2fa2fc`; + const checkpoint: PersistedNativeSession = { + backendKind: "runner", sessionId: "thread-1", identity, providerSessionId: "provider-session-1", + workingDirectory: savedRoot, providerRecoveryPolicy, cursor: "0", activeTurnId: inFlight ? "turn-in-flight" : null, + terminalTurns: [], pendingRuntimeRequests: [], lineage: [], + }; + const transportFactory = vi.fn(() => new FakeCodexTransport()); + const backend = createCodexNativeSessionBackend({ ...input, workspace: { ...input.workspace, cwd: assignedRoot } }, { + runnerInstanceId: "runner-recovery", transportFactory, + workingDirectoryAuthority: "remote_runner", environment: { PAPERCLIP_WORKSPACE_CWD: assignedRoot }, + }); + const replacementAttempted = new Error("replacement attempted"); + const openReplacementSession = vi.fn(async (sessionInput: { workingDirectory?: string }) => { + expect(sessionInput.workingDirectory).toBe(assignedRoot); + throw replacementAttempted; + }); + const openRun = vi.fn(async () => undefined); + const failure = await executeNativeSession({ + input: { ...input, workspace: { ...input.workspace, cwd: assignedRoot } }, + backend: { descriptor: () => backend.descriptor(), openSession: sessionInput => backend.openSession(sessionInput), + openReplacementSession, recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options) }, + controlPlane: { + openRun, appendEvent: vi.fn(), completeRun: vi.fn(), + loadSessionCheckpoint: async () => structuredClone(checkpoint), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), + }, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }).catch((error: unknown) => error); + if (replaces) { + expect(failure).toBe(replacementAttempted); + expect(openReplacementSession).toHaveBeenCalledOnce(); + } else { + if (inFlight) { + expect(failure).toBeInstanceOf(NativeProviderSideEffectPendingError); + expect((failure as NativeProviderSideEffectPendingError).code).toBe("native_provider_side_effect_pending"); + } else { + expect(failure).toBeInstanceOf(NativeProviderStateIncompatibleError); + expect((failure as NativeProviderStateIncompatibleError).code).toBe("native_provider_state_incompatible"); + } + expect(openReplacementSession).not.toHaveBeenCalled(); + expect(openRun).not.toHaveBeenCalled(); + } + // No Runner or provider process starts for a session that cannot resume here. + expect(transportFactory).not.toHaveBeenCalled(); + }); + it("never replaces a session after a deterministic, non-destructive recovery refusal", async () => { const outcomes = [ { failureClass: PROVIDER_ATTACH_REFUSED, reason: "run.attach cannot change protected launch arguments", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index fff7e2b6e6f..e138de3c335 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8549,7 +8549,7 @@ export function nativeSessionFailureSourceCode( } const NATIVE_PROVIDER_SIDE_EFFECT_PENDING_ACTION = - "The saved provider session has a tool call whose result never reached the runner, so Paperclip did not replace the session or retry it. Reconcile the action: check whether it ran and record the outcome. The task then continues in a fresh provider session in the same workspace. The saved session and workspace are unchanged."; + "The saved provider session has an unfinished provider turn, or a tool call whose result never reached the runner, so Paperclip did not replace the session or retry it. Reconcile the action: check whether it ran and record the outcome. The task then continues in a fresh provider session in the same workspace. The saved session and workspace are unchanged."; const NATIVE_PROVIDER_STATE_INCOMPATIBLE_ACTION = "The current runner cannot restore this task's saved provider session, so retrying this run cannot succeed. Retry the task to continue in a fresh provider session with the same workspace. The earlier provider conversation is not carried over.";