diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index b059505c464..636028ec192 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -4881,7 +4881,12 @@ transport failures keep the 30-second budget. At the launch of a new remote runner, a launch call that stalls this way is not retried. The controller reads the identity marker for the launch nonce instead, so the one launched runner is adopted or cleaned up by its nonce. The identity read uses the same 3-minute -stall budget. Each answer that the sandbox gives keeps its current meaning. Monitoring checks the saved process generation when available. +stall budget. Each answer that the sandbox gives keeps its current meaning. + +After the controller adopts a remote runner, it checks the runner's liveness every 250 ms. A check that +fails without an answer does not fail the run while the runner holds its authenticated connection. The +bound is the remote runner reconnect grace (7 minutes). A check that reports the process gone still fails +the run at once. A local adopted runner, which has no reconnect grace, keeps the old rule. Monitoring checks the saved process generation when available. Only a matching ownership marker and confirmed process exit permit automatic replacement. Missing or conflicting identity evidence, failed reads, and an exhausted probe budget retain the existing ownership and recovery safeguards. diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts index 664f7db4a44..859329993c1 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts @@ -28,6 +28,7 @@ async function verifyLiveRunnerAdoption( uncertainAtClose = false, semanticRecoveryMode?: "denied-before-cancel" | "cleanup-only", stopExitProofFault?: "missing" | "false", + livenessUnverifiedWhileConnected?: "within-grace" | "past-grace" | "reset-by-success", ) { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-live-adopt-")); const pendingCompletion = toolHandlerOrder?.startsWith("completion-") === true; @@ -214,6 +215,7 @@ async function verifyLiveRunnerAdoption( ...options, ...(preparedTask && !adoptionAuthorization.startsWith("legacy") ? { authorizeRunnerAdoption } : {}), ...(uncertainAtClose ? { closeGraceMs: 2_000 } : {}), + ...(livenessUnverifiedWhileConnected ? { runnerReconnectGraceMs: livenessUnverifiedWhileConnected === "within-grace" ? 60_000 : 2_000 } : {}), ...(semanticRecoveryMode === "cleanup-only" ? { recoveryPurpose: "stop_cleanup" as const } : {}), // Hash different stable bytes without replacing the real runner artifact // used by concurrent tests. Adoption must never execute this path. @@ -311,6 +313,31 @@ async function verifyLiveRunnerAdoption( }), ); if (toolHandlerOrder?.endsWith("after")) adopted.transport.setServerRequestHandler(redispatchedTool); + if (livenessUnverifiedWhileConnected) { + const pause = (ms: number) => new Promise(resolveWait => setTimeout(resolveWait, ms)); + expect(authority?.activeRunnerConnectionCount()).toBe(1); + rejectLiveness = true; + if (livenessUnverifiedWhileConnected === "reset-by-success") { + await pause(1_400); + rejectLiveness = false; + await pause(500); + rejectLiveness = true; + await pause(1_400); + } else { + await pause(livenessUnverifiedWhileConnected === "within-grace" ? 1_500 : 3_000); + } + rejectLiveness = false; + if (livenessUnverifiedWhileConnected === "past-grace") { + await expect(adopted.transport.request("thread/read", {})).rejects.toThrow("native_adopted_runner_identity_unverifiable"); + } else { + await expect(adopted.transport.request("thread/read", {})).resolves.toEqual( + expect.objectContaining({ thread: expect.objectContaining({ id: "codex-thread-1" }) })); + expect(adopted.evidence().diagnostics.join("\n")).not.toContain("native_adopted_runner_identity_unverifiable"); + } + expect(() => process.kill(runnerPid!, 0)).not.toThrow(); + expect(signal).not.toHaveBeenCalled(); + if (livenessUnverifiedWhileConnected === "past-grace") return; + } if (semanticRecoveryMode) { const retained = authority as DurablePrpControlPlane | null; expect(retained).not.toBeNull(); @@ -520,6 +547,10 @@ it("adopts a prepared runner after losing its controller launch cache without pr it.each(["deny", "legacy", "legacy-missing", "missing-binding"] as const)("preserves adoption authorization with %s", mode => verifyLiveRunnerAdoption(false, false, false, false, false, true, mode), 30_000); +it.each(["within-grace", "past-grace", "reset-by-success"] as const)( + "bounds a rejected liveness probe of a connected adopted runner by the reconnect grace (%s)", + mode => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, false, undefined, undefined, mode), 30_000); + it("retains authenticated adopted-runner ownership when a close probe rejects", () => verifyLiveRunnerAdoption(false, false, false, false, false, false, "allow", undefined, true), 30_000); it( diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 2d33e6a8182..64b9d87f167 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -8605,11 +8605,13 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { >, ): void { let checking = false; + let unverifiedSince: number | null = null; this.#adoptedRunnerMonitor = setInterval(() => { if (checking || this.#closed || (this.#releaseEpisode && !this.#releaseEpisode.claimed)) return; checking = true; void Promise.resolve(adoptedRunner.isAlive()) .then((alive) => { + unverifiedSince = null; if (alive || this.#closed || (this.#releaseEpisode && !this.#releaseEpisode.claimed)) return; this.#evidence.runnerExited = true; this.#publish(); @@ -8620,6 +8622,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); }) .catch(() => { + const now = Date.now(); + unverifiedSince ??= now; + if ((this.#core?.activeRunnerConnectionCount() ?? 0) > 0 + && now - unverifiedSince < (this.options.runnerReconnectGraceMs ?? 0)) return; if (!this.#closed && !(this.#releaseEpisode && !this.#releaseEpisode.claimed)) { this.#failTransport( new Error(