diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 1bf347ec6ba..b0450623921 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -21109,10 +21109,12 @@ describe("runnerd provider runtime wiring", () => { ...["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_unproven", "interrupted_bootstrap_symlink_parent", "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_unproven", "interrupted_bootstrap_unopened_other_run", "interrupted_bootstrap_unopened_active_turn", "interrupted_bootstrap_unprepared", "interrupted_bootstrap_unprepared_unproven", - "interrupted_bootstrap_unprepared_other_run", "interrupted_bootstrap_unprepared_suspended", "interrupted_bootstrap_unprepared_backup"] + "interrupted_bootstrap_unprepared_other_run", "interrupted_bootstrap_unprepared_suspended", "interrupted_bootstrap_unprepared_backup", + "interrupted_bootstrap_unprepared_unrecorded", "interrupted_bootstrap_unprepared_connected"] .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), { outcome: "created", scenario: "interrupted_bootstrap", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unprepared", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unprepared_unrecorded", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_symlink_parent", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened_active_turn", homeMode: "isolated" }, @@ -21162,6 +21164,11 @@ describe("runnerd provider runtime wiring", () => { } if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unprepared") && command.args[1].includes("codex-provider-state.json")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; + if (command.args?.[2] === "paperclip-runner-recovery-identity" && scenario === "interrupted_bootstrap_unprepared_unrecorded") return { + exitCode: 0, timedOut: false, stderr: "", + stdout: "exitednonce\n4242\n2026-11-12T00:32:10.725Z\nrunner-new-in-retained-workspace\nlinux:abcd-1234:100\n" }; + if (command.args?.[2] === "paperclip-runner-recovery-check" && scenario === "interrupted_bootstrap_unprepared_unrecorded") return { + exitCode: 45, timedOut: false, stdout: "", stderr: "" }; if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unopened") && command.args[1].includes("codex-provider-state.json") && !command.args[1].includes("runner-state.json")) return { exitCode: 0, timedOut: false, stderr: "", stdout: Buffer.from(JSON.stringify({ @@ -21175,7 +21182,10 @@ describe("runnerd provider runtime wiring", () => { runId: scenario.endsWith("_other_run") ? "previous-warm-run" : execution.binding.runId, lifecycle: scenario.startsWith("interrupted_bootstrap_unopened") || scenario.endsWith("_suspended") ? "suspended" : scenario.startsWith("interrupted_bootstrap_unprepared") ? "connecting" : "ready", - outbox: [], pendingTerminalDelivery: null })).toString("base64"), + outbox: [], pendingTerminalDelivery: null, + ...(scenario.startsWith("interrupted_bootstrap_unprepared") ? { lastControllerCommandSeq: 0, reconnectCount: 0, + processedCommands: {}, lastConnectionProtocolVersion: scenario.endsWith("_connected") ? 2 : null, + environmentLeaseId: "lease-resumed", turnId: `turn-${execution.binding.runId}`, itemId: `item-${execution.binding.runId}` } : {}) })).toString("base64"), }; if (command.args?.[1]?.includes("base64")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "", @@ -21272,7 +21282,8 @@ describe("runnerd provider runtime wiring", () => { "interrupted_bootstrap_unprepared_backup"].includes(scenario)) { noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped"); } - if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened" || scenario === "interrupted_bootstrap_unprepared") { + if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened" || scenario === "interrupted_bootstrap_unprepared" + || scenario === "interrupted_bootstrap_unprepared_unrecorded") { await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); const siblings = await readdir(dirname(sessionRoot)); expect(siblings.filter(name => name.startsWith(`${basename(sessionRoot)}.bootstrap-restart-`))).toHaveLength(1); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7fbcab9816e..e1e347fed8d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -14686,6 +14686,14 @@ export async function readRemoteRunnerState(input: { } } +export function remoteRunnerStateNeverConnected(state: Record): boolean { + const empty = (value: unknown) => value !== null && typeof value === "object" && !Array.isArray(value) + && Object.keys(value).length === 0; + return state.lastConnectionProtocolVersion === null && state.lastControllerCommandSeq === 0 + && state.reconnectCount === 0 && Array.isArray(state.outbox) && state.outbox.length === 0 + && state.pendingTerminalDelivery === null && empty(state.processedCommands); +} + async function readRemoteRunnerProviderState(input: { runner: CommandManagedRuntimeRunner; stateDirectory: string; @@ -16551,16 +16559,20 @@ async function createRunnerdBackendWithinSessionClaim( (state.incompleteReason === "not_suspended" || state.incompleteReason === "provider_unopened") && provenOwnBootstrapRestart(state.notSuspendedRunId); const inspectOwnUnpreparedBootstrap = async (): Promise => { + const runId = input.execution.binding.runId; if (!remoteCommandRunner || !remoteStateDirectory || archivedRotationRunnerStatePath - || !provenOwnBootstrapRestart(input.execution.binding.runId) + || input.restartRecovery?.kind !== "bootstrap_incomplete" || input.restartRecovery.runId !== runId || harnessBackupCandidates(root).some(candidate => existsSync(resolve(candidate, "manifest.json")))) return null; const runnerState = await readRemoteRunnerState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory }) .then(record, () => null); - if (!runnerState || runnerState.runnerInstanceId !== input.runnerInstanceId + if (!runnerState || runnerState.runnerInstanceId !== input.runnerInstanceId || runnerState.runId !== runId || runnerState.normalizedSessionId !== nativeSessionKey(input.execution) - || typeof runnerState.lifecycle !== "string" || runnerState.lifecycle === "suspended" - || !provenOwnBootstrapRestart(typeof runnerState.runId === "string" ? runnerState.runId : null)) return null; - return input.execution.binding.runId; + || typeof runnerState.lifecycle !== "string" || runnerState.lifecycle === "suspended") return null; + if (!hasRemoteBootstrapRestartProof(runId) && !(remoteRunnerStateNeverConnected(runnerState) + && recordRemoteBootstrapRestartProof(runId, await stopStaleRemoteRunnerBeforeBootstrap({ + execution: input.execution, runnerExecutionTarget: input.runnerExecutionTarget, identity: runnerState, onLog: input.onLog, + })))) return null; + return provenOwnBootstrapRestart(runId) ? runId : null; }; const inspectRemoteHarnessState = async ( source: "live" | "host_preparation" = "live",