From 004859561043128b4afc83cc2c57e32fb37e52ea Mon Sep 17 00:00:00 2001 From: Paperclip Date: Fri, 6 Nov 2026 12:58:01 -0800 Subject: [PATCH] Reject failover backups behind the accepted event cursor Check supported controller journals before copying a verified backup to a replacement sandbox. Preserve native_event_replay_conflict when a runner exit wraps the error, so stale state does not enter a retry loop. Also supply the normal image-only input in the workflow target fixture used by current master. Verified 1,171 native session tests and nine workflow target tests. The new stale-cursor and classification cases fail on unchanged master; the workflow fixture reproduces and fixes the unset input error. Broader CI and exact-head Apex review remain required. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- doc/recoverable-disk-cleanup.md | 9 ++ .../native-session-executor.test.ts | 86 +++++++++++++++++++ .../native-runtime/native-session-executor.ts | 36 +++++++- tests/runner-e2e/workflow-target.test.ts | 2 +- 4 files changed, 129 insertions(+), 4 deletions(-) diff --git a/doc/recoverable-disk-cleanup.md b/doc/recoverable-disk-cleanup.md index d268de28f16..95adcd1ac05 100644 --- a/doc/recoverable-disk-cleanup.md +++ b/doc/recoverable-disk-cleanup.md @@ -63,6 +63,15 @@ directories, and data Codex downloads again on startup: remote plugin bundles un install staging. After a restore, remote plugins can be unavailable to a turn until Codex finishes downloading them. +A turn that ends without a proven Runner suspension writes no new copy, but the +controller can still acknowledge the events Runner produced after the last copy. +Before it restores a copy into a replacement sandbox, the server compares the +copy's produced event cursor with the controller's acknowledged cursor. If the +copy is behind, the run fails with `native_event_replay_conflict` and automatic +recovery stops. No files are copied to the sandbox. Resuming that copy would make +Runner reject the controller's acknowledgement, and it could repeat work the +controller already accepted. + An hourly background sweep expires the copy once the session can only resume by reopening closed work. It checks at most 200 sessions and expires at most 50 per batch, starting two minutes after readiness. A copy expires when all of these hold: diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c568d5350f2..d3e76df91ba 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -18089,6 +18089,92 @@ describe("runnerd provider runtime wiring", () => { } }); + it.each([ + { backup: "stale", nextSourceSeq: 120, controlSchema: "paperclip.runner.durable.control-plane-state.v1" }, + { backup: "stale", nextSourceSeq: 120, controlSchema: "paperclip.runner.durable.control-plane-state.v2" }, + { backup: "current", nextSourceSeq: 1518, controlSchema: "paperclip.runner.durable.control-plane-state.v2" }, + ])("checks the controller cursor before restoring a $backup failover backup ($controlSchema)", async ({ backup, nextSourceSeq, controlSchema }) => { + const remoteCwd = join(isolatedStateDirectory, "remote"); + await mkdir(join(remoteCwd, ".paperclip-runtime", "paperclip-runner"), { recursive: true }); + const syncIn = vi.fn(async () => undefined); + const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { + if (command.args?.[0] === "--build-metadata") return { + exitCode: 0, timedOut: false, stdout: JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], + prpTransportModes: ["listen_ws"], + }), stderr: "", + }; + if (command.args?.[0] === "--version") return { + exitCode: 0, timedOut: false, stdout: "codex-cli 0.160.0", stderr: "", + }; + if (command.args?.[1]?.includes("base64")) return { + exitCode: 1, timedOut: false, stdout: "", stderr: "", + }; + return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; + }); + const runnerInstanceId = "runner-failover-cursor"; + await createRunnerdBackend({ + db: leaseDb(execution), execution, runnerInstanceId, + runnerIngressAuthorized: true, + runnerExecutionTarget: { + kind: "remote", transport: "sandbox", remoteCwd, environmentId: "environment", + leaseId: "lease-replacement", providerKey: "daytona", + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "sandbox-replacement", reason: "not_found" }, + runner: { execute: remoteExecute, syncIn }, + } as never, + }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { + prepareExternalRunnerState: () => Promise; + }; + const root = options.stateDirectory!; + await mkdir(join(root, "control-plane"), { recursive: true }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify({ + schema: controlSchema, + identity: options.prpIdentity, ackedSourceSeq: 1517, connectionCount: 1, committedEvents: [], commands: [], + })); + const current = join(root, "failover-backups", "current"); + await mkdir(join(current, "runner"), { recursive: true }); + await mkdir(join(current, "codex-home", "sessions"), { recursive: true }); + await writeFile(join(current, "runner", "runner-state.json"), JSON.stringify({ + schema: "paperclip.runner.durable.state.v1", ...options.prpIdentity, runnerInstanceId, + lifecycle: "suspended", nextSourceSeq, ackedSourceSeq: nextSourceSeq - 1, outbox: [], + })); + await writeFile(join(current, "codex-home", "sessions", "thread.jsonl"), "thread-state"); + await writeFile(join(current, "manifest.json"), JSON.stringify(buildNativeHarnessBackupManifest({ + backupRoot: current, execution, runnerInstanceId, + providerSessionIdentity: { providerSessionId: "thread-1", providerBackendSessionId: "session-1", providerSessionIdentity: null }, + sourceProviderLeaseId: "sandbox-previous", + }))); + + const prepared = options.prepareExternalRunnerState(); + if (backup === "stale") { + await expect(prepared).rejects.toThrow( + "native_event_replay_conflict: failover backup is behind the controller cursor (backup produced 119, controller acknowledged 1517)", + ); + expect(syncIn).not.toHaveBeenCalled(); + const error = await prepared.catch((caught: unknown) => caught); + expect(nativeSessionFailureSourceCode(new Error(`native_session_recovery_failed: ${String(error)}`))) + .toBe("native_event_replay_conflict"); + expect(nativeSessionFailureDisposition(1, new Date(), "native_event_replay_conflict").phase).toBe("terminal_failure"); + } else { + await expect(prepared).rejects.toThrow("runner_harness_state_mismatch: restored_provider_identity_changed"); + expect(syncIn).toHaveBeenCalled(); + } + }); + + it("keeps an integrity conflict permanent when a runner exit wraps it", () => { + expect(nativeSessionFailureSourceCode(new Error( + "native_session_recovery_failed: runnerd exited unexpectedly: native_event_replay_conflict: failover backup is behind the controller cursor", + ))).toBe("native_event_replay_conflict"); + expect(nativeSessionFailureSourceCode(new Error( + "native_session_recovery_failed: Error: runnerd exited while waiting for runner.drain", + ))).toBe("native_runner_process_exited"); + }); + it.each([ ...["current", "preinstalled-exact", "preinstalled-mismatch", "preinstalled-error", "preinstalled-timeout", "stale", "missing", "retained", "retained-mismatch", "retained-error", "retained-timeout", "retained-explicit"] .map((image) => ({ image, version: "0.160.0" })), diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c7fddc8388b..0e8b4a60a79 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7005,6 +7005,34 @@ export function verifyNativeHarnessBackup(input: { return null; } +export function nativeHarnessBackupCursorConflict(input: { + backupRoot: string; + stateRoot: string; +}): string | null { + const read = (path: string, maxBytes: number) => { + try { + return existsSync(path) + ? record(JSON.parse(readBoundedNativeFile(path, maxBytes, "runner_durable_identity_too_large").toString("utf8"))) + : null; + } catch { + return null; + } + }; + const control = read(resolve(input.stateRoot, "control-plane", "control-plane-state.json"), + NATIVE_CONTROL_PLANE_STATE_MAX_BYTES); + if (!control || !RUNNERD_CONTROL_PLANE_STATE_SCHEMAS.has(String(control.schema))) return null; + const runner = read(resolve(input.backupRoot, "runner", "runner-state.json"), NATIVE_RUNNER_STATE_MAX_BYTES); + if (!runner) return null; + const acked = control.ackedSourceSeq; + const next = runner.nextSourceSeq; + if (!Number.isSafeInteger(acked) || !Number.isSafeInteger(next) || (next as number) < 1 || + record(control.identity).runnerInstanceId !== runner.runnerInstanceId) return null; + const produced = (next as number) - 1; + return (acked as number) > produced + ? `native_event_replay_conflict: failover backup is behind the controller cursor (backup produced ${produced}, controller acknowledged ${acked as number})` + : null; +} + function nativeSessionCheckpointDirectory(): string { const directory = resolve( resolvePaperclipInstanceRoot(), @@ -7412,6 +7440,9 @@ export function nativeSessionFailureSourceCode( ) { return "provider_transport_failed"; } + if (/native_event_replay_conflict/i.test(message)) { + return "native_event_replay_conflict"; + } if ( /native_runner_process_exited|runnerd exited|runner process failed/i.test( message, @@ -7422,9 +7453,6 @@ export function nativeSessionFailureSourceCode( if (/planning_mode_unsupported/i.test(message)) { return "planning_mode_unsupported"; } - if (/native_event_replay_conflict/i.test(message)) { - return "native_event_replay_conflict"; - } if (message.toLowerCase().includes(NATIVE_CURRENT_WAKE_COMMENTS_CHANGED_AFTER_READ.toLowerCase())) { return NATIVE_CURRENT_WAKE_COMMENTS_CHANGED_AFTER_READ; } @@ -14346,6 +14374,8 @@ async function createRunnerdBackendWithinSessionClaim( runnerInstanceId: input.runnerInstanceId, }); if (!backup) throw new Error("runner_harness_backup_unavailable"); + const cursorConflict = nativeHarnessBackupCursorConflict({ backupRoot: backup.root, stateRoot: root }); + if (cursorConflict) throw new Error(cursorConflict); await measureNativeRunnerSpan( input.trace, "harness_state.failover_restore", diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.ts index a4e1c57998b..f98b85c128b 100644 --- a/tests/runner-e2e/workflow-target.test.ts +++ b/tests/runner-e2e/workflow-target.test.ts @@ -17,7 +17,7 @@ async function resolveTarget(event: string, cadence: string, branch = "master") const result = spawnSync("bash", ["-c", `gh() { printf '%s\\n' "$MOCK_BRANCH_SHA"; }\n${script}`], { encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence, TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha, - MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" }, + MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip", IMAGE_ONLY: "false" }, }); return { status: result.status, stderr: result.stderr, output: await readFile(output, "utf8").catch(() => "") };