diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 9e87d7bad89..1c129fe075a 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -21304,9 +21304,13 @@ describe("runnerd provider runtime wiring", () => { "interrupted_bootstrap_unprepared_other_run", "interrupted_bootstrap_unprepared_suspended", "interrupted_bootstrap_unprepared_backup", "interrupted_bootstrap_unprepared_unrecorded", "interrupted_bootstrap_unprepared_connected"] .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), + ...["interrupted_bootstrap_unlaunched", "interrupted_bootstrap_unlaunched_marker", "interrupted_bootstrap_unlaunched_backup", + "interrupted_bootstrap_unlaunched_archived", "interrupted_bootstrap_unlaunched_nested_state"] + .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), { outcome: "created", scenario: "interrupted_bootstrap", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unprepared", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unprepared_unrecorded", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unlaunched", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_symlink_parent", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened_active_turn", homeMode: "isolated" }, @@ -21324,6 +21328,24 @@ describe("runnerd provider runtime wiring", () => { await symlink(symlinkTarget, join(runtimeRoot, "sessions")); } if (scenario.startsWith("interrupted_bootstrap")) await mkdir(sessionRoot, { recursive: true }); + if (scenario.startsWith("interrupted_bootstrap_unlaunched")) { + // The detached controller staged launch files but never launched a Runner. + await mkdir(join(sessionRoot, "codex-home"), { recursive: true }); + await writeFile(join(sessionRoot, "codex-home", "config.toml"), "fixture-staged-config"); + if (scenario.endsWith("_marker")) { + await mkdir(join(sessionRoot, "runner"), { recursive: true }); + await writeFile(join(sessionRoot, "runner", "runner-process.identity"), "launchednonce\n4242\n"); + } + if (scenario.endsWith("_archived")) { + // An authority rotation archived the live Runner state of a real session. + await mkdir(join(sessionRoot, "authority-epochs", "epoch-previous"), { recursive: true }); + await writeFile(join(sessionRoot, "authority-epochs", "epoch-previous", "runner-state.json"), "{}"); + } + if (scenario.endsWith("_nested_state")) { + await mkdir(join(sessionRoot, "filesystem", "elsewhere"), { recursive: true }); + await writeFile(join(sessionRoot, "filesystem", "elsewhere", "runner-state.json"), "{}"); + } + } const syncIn = vi.fn(async () => undefined); const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { if (command.args?.[2] === "paperclip-runner-claim-unstarted-session") { @@ -21354,6 +21376,12 @@ describe("runnerd provider runtime wiring", () => { try { execFileSync("sh", command.args, { stdio: "pipe" }); return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; } catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } } + if (command.args?.[2] === "paperclip-runner-unlaunched-bootstrap") { + try { execFileSync("sh", command.args, { stdio: "pipe" }); return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; } + catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } + } + if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unlaunched")) return { + exitCode: 1, timedOut: false, stdout: "", stderr: "" }; if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unprepared") && command.args[1].includes("codex-provider-state.json")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; if (command.args?.[2] === "paperclip-runner-recovery-identity" && scenario === "interrupted_bootstrap_unprepared_unrecorded") return { @@ -21461,6 +21489,29 @@ describe("runnerd provider runtime wiring", () => { { type: scenario === "pending_turn" ? "turn.start" : "session.open", status: "pending" }], })); } + if (scenario.startsWith("interrupted_bootstrap_unlaunched")) { + // A closing server detached this bootstrap after staging launch files and + // before launching its Runner; nothing else proves the Runner's absence. + if (scenario.endsWith("_backup")) { + await mkdir(join(options.stateDirectory!, "failover-backups", "current"), { recursive: true }); + await writeFile(join(options.stateDirectory!, "failover-backups", "current", "manifest.json"), "{}"); + } + const restartAsides = async () => (await readdir(dirname(sessionRoot))) + .filter(name => name.startsWith(`${basename(sessionRoot)}.bootstrap-restart-`)); + if (scenario === "interrupted_bootstrap_unlaunched") { + await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + expect(await restartAsides()).toHaveLength(1); + expect(await readdir(sessionRoot)).toEqual([]); + expect(remoteExecute.mock.calls.filter(([command]) => command.args?.[2] === "paperclip-runner-claim-unstarted-session")).toHaveLength(1); + } else { + // A launch marker, any saved Runner state or a failover backup keeps the run fail-closed. + await expect(options.prepareExternalRunnerState()).rejects.toThrow("runner_harness_state_mismatch: resumed_unavailable"); + expect(await restartAsides()).toEqual([]); + expect(remoteExecute.mock.calls.some(([command]) => command.args?.[2] === "paperclip-runner-bootstrap-restart")).toBe(false); + } + takeRemoteBootstrapRestartProof(execution.binding.runId); + return; + } if (scenario === "missing_control_plane") { await rm(join(options.stateDirectory!, "control-plane", "control-plane-state.json")); } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 461462c6b7d..4a7bf8ff57b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -16663,6 +16663,29 @@ async function createRunnerdBackendWithinSessionClaim( const restartableBootstrapState = (state: { incompleteReason: string | null; notSuspendedRunId?: string | null }) => (state.incompleteReason === "not_suspended" || state.incompleteReason === "provider_unopened") && provenOwnBootstrapRestart(state.notSuspendedRunId); + /** + * A controller detached mid-bootstrap can stop after staging launch files + * into the session root but before launching its Runner. The launch writes + * the process identity marker before the Runner starts, and the Runner + * writes its state file, so a real session root holding neither never had a + * Runner and no provider work. Only the caller's same-run claim, granted + * after the previous controller's lease ended, may act on this. + */ + const remoteBootstrapNeverLaunchedRunner = async (): Promise => { + if (!remoteCommandRunner || !remoteSessionRoot || !remoteStateDirectory) return false; + const probe = await remoteCommandRunner.execute({ + command: "sh", + args: ["-c", + // An archived authority epoch holds a real session's Runner state, so + // any Runner state or launch marker anywhere under the root fails closed. + 'set -eu; test -d "$1" && test ! -L "$1" || exit 1; for path in "$2" "$3" "$1/authority-epochs"; do if test -e "$path" || test -L "$path"; then exit 1; fi; done; found=$(find "$1" \\( -name runner-state.json -o -name runner-process.identity \\) -print); test -z "$found"', + "paperclip-runner-unlaunched-bootstrap", remoteSessionRoot, + posix.join(remoteStateDirectory, "runner-state.json"), posix.join(remoteStateDirectory, "runner-process.identity")], + bypassSession: true, + timeoutMs: 10_000, + }).catch(() => null); + return probe?.exitCode === 0 && !probe.timedOut; + }; const inspectOwnUnpreparedBootstrap = async (): Promise => { const runId = input.execution.binding.runId; if (!remoteCommandRunner || !remoteStateDirectory || archivedRotationRunnerStatePath @@ -16670,6 +16693,11 @@ async function createRunnerdBackendWithinSessionClaim( || harnessBackupCandidates(root).some(candidate => existsSync(resolve(candidate, "manifest.json")))) return null; const runnerState = await readRemoteRunnerState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory }) .then(record, () => null); + if (!runnerState && await remoteBootstrapNeverLaunchedRunner()) { + recordRemoteBootstrapRestartProof(runId, "absent_confirmed"); + await input.onLog?.("stdout", "[paperclip-runner] The interrupted bootstrap never started its runner; restarting the bootstrap.\n"); + return provenOwnBootstrapRestart(runId) ? runId : null; + } if (!runnerState || runnerState.runnerInstanceId !== input.runnerInstanceId || runnerState.runId !== runId || runnerState.normalizedSessionId !== nativeSessionKey(input.execution) || typeof runnerState.lifecycle !== "string" || runnerState.lifecycle === "suspended") return null;