diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index a7444964fc5..d5dfd2a44ea 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -401,6 +401,10 @@ import { createGovernedWaitEventObservation, createRemoteRunnerProcessLauncher, createRunnerdBackend, + noteRemoteBootstrapRestartProven, + recordRemoteBootstrapRestartProof, + hasRemoteBootstrapRestartProof, + takeRemoteBootstrapRestartProof, unidentifiedLocalRunnerProvenAbsent, executePaperclipNativeSession, detachNativeSessionsForRestart, @@ -627,10 +631,23 @@ describe("remote controller restart adoption", () => { }); it("escalates to SIGKILL when the runner ignores SIGTERM", async () => { const { execute, target } = fixture(); + const live = execute.getMockImplementation()!; + let killed = false; + execute.mockImplementation(async request => { + if (killed && request.args[1]!.includes("expected_pid")) return { stdout: "", stderr: "", exitCode: 45, timedOut: false }; + if (request.args.at(-1) === "KILL") killed = true; + return live(request); + }); await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, { waitMs: 20, pollMs: 1 })).resolves.toBe("killed"); expect(signalled(execute)).toEqual(["TERM", "KILL"]); }); + it("reports a runner that outlives SIGKILL as not stopped", async () => { + const { execute, target } = fixture(); + await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, + { waitMs: 20, pollMs: 1 })).resolves.toBe("stop_failed"); + expect(signalled(execute)).toEqual(["TERM", "KILL"]); + }); it("signals nothing when the sandbox runner belongs to another identity", async () => { const { execute, target } = fixture({ runnerInstanceId: "different" }); await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, @@ -643,7 +660,21 @@ describe("remote controller restart adoption", () => { execute.mockImplementation(async request => request.args[1]!.includes("expected_pid") ? { stdout: "", stderr: "", exitCode: 45, timedOut: false } : live(request)); await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, - { waitMs: 20, pollMs: 1 })).resolves.toBe("no_prior_runner"); + { waitMs: 20, pollMs: 1 })).resolves.toBe("absent_confirmed"); + expect(signalled(execute)).toEqual([]); + }); + it.each([["absent_confirmed", true], ["stopped", true], ["killed", true], ["stop_failed", false], ["no_prior_runner", false]] as const)( + "records a bootstrap restart proof only for a confirmed stop: %s", (stopped, proven) => { + noteRemoteBootstrapRestartProven("run", "stopped"); + expect(recordRemoteBootstrapRestartProof("run", stopped)).toBe(proven); + expect(hasRemoteBootstrapRestartProof("run")).toBe(proven); + takeRemoteBootstrapRestartProof("run"); + }); + it("does not confirm absence when the process probe is uncertain", async () => { + const { execute, target } = fixture(); + const verify = async () => { throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "probe_timeout" } as never); }; + await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, + { waitMs: 20, pollMs: 1, verify: verify as never })).resolves.toBe("no_prior_runner"); expect(signalled(execute)).toEqual([]); }); }); @@ -19570,6 +19601,8 @@ describe("runnerd provider runtime wiring", () => { ["fresh", "external_checkpoint", "external_pending_turn", "external_wrong_identity", "external_unregistered", "existing_state", "symlink_parent", "wrong_identity", "wrong_runner", "wrong_lease", "missing_control_plane", "connected", "committed_event", "committed_command", "pending_turn", "remote_probe_failed", "backup_present"] .map((scenario) => ({ outcome, scenario, homeMode: "isolated" }))), ...["resumed", "replacement"].map(outcome => ({ outcome, scenario: "fresh", homeMode: "preconfigured" })), + ...["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_unproven", "interrupted_bootstrap_symlink_parent"] + .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), ])( "bootstraps only an untouched provider session in a $outcome workspace lease: $scenario ($homeMode)", async ({ outcome, scenario, homeMode }) => { const remoteCwd = join(isolatedStateDirectory, "remote"); @@ -19577,7 +19610,12 @@ describe("runnerd provider runtime wiring", () => { await mkdir(runtimeRoot, { recursive: true }); const sessionRoot = join(runtimeRoot, "sessions", createHash("sha256").update(execution.session.normalizedSessionId!).digest("hex")); if (scenario === "existing_state") await mkdir(sessionRoot, { recursive: true }); - if (scenario === "symlink_parent") await symlink(isolatedStateDirectory, join(runtimeRoot, "sessions")); + const symlinkTarget = scenario === "interrupted_bootstrap_symlink_parent" ? join(isolatedStateDirectory, "elsewhere") : isolatedStateDirectory; + if (scenario === "symlink_parent" || scenario === "interrupted_bootstrap_symlink_parent") { + await mkdir(symlinkTarget, { recursive: true }); + await symlink(symlinkTarget, join(runtimeRoot, "sessions")); + } + if (scenario.startsWith("interrupted_bootstrap")) await mkdir(sessionRoot, { recursive: true }); const syncIn = vi.fn(async () => undefined); const remoteExecute = vi.fn(async (command: { command: string; args?: string[] }) => { if (command.args?.[2] === "paperclip-runner-claim-unstarted-session") { @@ -19604,6 +19642,16 @@ describe("runnerd provider runtime wiring", () => { if (command.args?.[2] === "paperclip-runner-launch") { throw new Error("fixture_stop_after_launch_staging"); } + if (command.args?.[2] === "paperclip-runner-bootstrap-restart") { + try { execFileSync("sh", command.args, { stdio: "pipe" }); return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; } + catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } + } + if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap")) return { + exitCode: 0, timedOut: false, stderr: "", stdout: Buffer.from(JSON.stringify({ + runnerInstanceId: "runner-new-in-retained-workspace", normalizedSessionId: execution.session.normalizedSessionId, + runId: scenario === "interrupted_bootstrap_other_run" ? "previous-warm-run" : execution.binding.runId, + lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64"), + }; if (command.args?.[1]?.includes("base64")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "", }; @@ -19621,6 +19669,9 @@ describe("runnerd provider runtime wiring", () => { const backend = await createRunnerdBackend({ db: leaseDb(executionWithContext), execution: executionWithContext, runnerInstanceId: "runner-new-in-retained-workspace", ...(scenario.startsWith("external_") ? { runnerRemoteBinaryPath: controllerArtifact, runnerRemoteCodexPath: controllerArtifact } : {}), + ...(scenario.startsWith("interrupted_bootstrap") ? { restartRecovery: { kind: "bootstrap_incomplete" as const, + runId: execution.binding.runId, leaseOwner: "controller", controllerGeneration: 3, providerAttempt: 1, + restartKind: "hard" as const, recoveryRequestId: null } } : {}), trace: createNativeRunTrace({ runId: execution.binding.runId, onEvent: onTraceEvent }), runnerIngressAuthorized: true, runnerExecutionTarget: { @@ -19690,7 +19741,16 @@ describe("runnerd provider runtime wiring", () => { await mkdir(join(options.stateDirectory!, "failover-backups", "current"), { recursive: true }); await writeFile(join(options.stateDirectory!, "failover-backups", "current", "manifest.json"), "{}"); } - if (scenario === "fresh" || scenario === "external_checkpoint") { + if (["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_symlink_parent"].includes(scenario)) { + noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped"); + } + if (scenario === "interrupted_bootstrap") { + await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + const siblings = await readdir(dirname(sessionRoot)); + expect(siblings.filter(name => name.startsWith(`${basename(sessionRoot)}.bootstrap-restart-`))).toHaveLength(1); + expect((await lstat(sessionRoot)).isDirectory()).toBe(true); + expect(remoteExecute.mock.calls.filter(([command]) => command.args?.[2] === "paperclip-runner-claim-unstarted-session")).toHaveLength(1); + } else if (scenario === "fresh" || scenario === "external_checkpoint") { await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); expect(onTraceEvent).toHaveBeenCalledWith(expect.objectContaining({ payload: expect.objectContaining({ @@ -19748,6 +19808,13 @@ describe("runnerd provider runtime wiring", () => { ); expect(remoteExecute.mock.calls.some(([command]) => command.args?.[1]?.includes("install -d"))).toBe(false); expect(syncIn).not.toHaveBeenCalled(); + if (scenario === "interrupted_bootstrap_symlink_parent") { + expect((await readdir(symlinkTarget)).some(name => name.includes(".bootstrap-restart-"))).toBe(false); + expect((await lstat(sessionRoot)).isDirectory()).toBe(true); + } else { + expect(remoteExecute.mock.calls.some(([command]) => command.args?.[2] === "paperclip-runner-bootstrap-restart")).toBe(false); + } + takeRemoteBootstrapRestartProof(execution.binding.runId); } }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index df2df509da7..e31831b35ba 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -5748,7 +5748,11 @@ async function migrateRunnerdStateRootForExecution(input: { // holds no provider work (RC-29d). if (runnerdStateProvesIncompleteBootstrap(scoped, { localProcesses }) || (remoteSandbox && runnerdStateProvesNoRemoteProviderTurn(scoped))) { - if (remoteSandbox) await stopStaleRemoteRunnerBeforeBootstrap({ ...input, identity }); + if (remoteSandbox) { + takeRemoteBootstrapRestartProof(input.execution.binding.runId); + recordRemoteBootstrapRestartProof(input.execution.binding.runId, + await stopStaleRemoteRunnerBeforeBootstrap({ ...input, identity })); + } quarantineRunnerdStateRoot(scoped, "identity_indeterminate"); return; } @@ -6287,6 +6291,28 @@ const STALE_REMOTE_RUNNER_STOP_WAIT_MS = 10_000; * runner can be verified there is nothing to stop, and a remaining conflict is * still caught by the bounded same-run bootstrap retry. */ +const provenRemoteBootstrapRestarts = new Map(); + +export function noteRemoteBootstrapRestartProven(runId: string, stopped: "absent_confirmed" | "stopped" | "killed"): void { + provenRemoteBootstrapRestarts.set(runId, stopped); +} + +export function recordRemoteBootstrapRestartProof(runId: string, + stopped: Awaited>): boolean { + provenRemoteBootstrapRestarts.delete(runId); + if (stopped !== "absent_confirmed" && stopped !== "stopped" && stopped !== "killed") return false; + provenRemoteBootstrapRestarts.set(runId, stopped); + return true; +} + +export function hasRemoteBootstrapRestartProof(runId: string): boolean { + return provenRemoteBootstrapRestarts.has(runId); +} + +export function takeRemoteBootstrapRestartProof(runId: string): boolean { + return provenRemoteBootstrapRestarts.delete(runId); +} + export async function stopStaleRemoteRunnerBeforeBootstrap(input: { execution: NativeExecutionInput; runnerExecutionTarget?: AdapterExecutionTarget | null; @@ -6296,7 +6322,7 @@ export async function stopStaleRemoteRunnerBeforeBootstrap(input: { verify?: typeof verifyRemoteRunnerReattachment; waitMs?: number; pollMs?: number; -} = {}): Promise<"no_prior_runner" | "stopped" | "killed" | "stop_failed"> { +} = {}): Promise<"no_prior_runner" | "absent_confirmed" | "stopped" | "killed" | "stop_failed"> { const target = input.runnerExecutionTarget; if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner) return "no_prior_runner"; const providerLeaseId = target.sandboxLeaseAcquisition?.providerLeaseId; @@ -6309,9 +6335,10 @@ export async function stopStaleRemoteRunnerBeforeBootstrap(input: { claim: { runId, remote: { providerLeaseId, remoteCwd: target.remoteCwd } }, target, identity: input.identity, runId, normalizedSessionId: nativeSessionKey(input.execution), }); - } catch { + } catch (error) { // No runner with this run's exact identity and generation is alive here. - return "no_prior_runner"; + return error instanceof RemoteRunnerRecoveryError && error.remoteRunnerRecovery.reason === "process_not_running" + ? "absent_confirmed" : "no_prior_runner"; } try { let outcome: "stopped" | "killed" = "stopped"; @@ -6323,6 +6350,11 @@ export async function stopStaleRemoteRunnerBeforeBootstrap(input: { if (await runner.isAlive()) { await runner.signal("SIGKILL"); outcome = "killed"; + const killDeadline = Date.now() + (options.waitMs ?? STALE_REMOTE_RUNNER_STOP_WAIT_MS); + while (Date.now() < killDeadline && await runner.isAlive()) { + await new Promise(done => setTimeout(done, options.pollMs ?? 500)); + } + if (await runner.isAlive()) return "stop_failed"; } } await input.onLog?.("stdout", `[paperclip-runner] Stopped the interrupted runner ${runner.pid} before restarting the bootstrap.\n`); @@ -15073,6 +15105,7 @@ async function createRunnerdBackendWithinSessionClaim( runnerState: Record | null; providerSessionIdentity: Record | null; incompleteReason: "unavailable" | "not_suspended" | null; + notSuspendedRunId?: string | null; }> => { if (!remoteCommandRunner || !remoteStateDirectory) { return { @@ -15140,6 +15173,7 @@ async function createRunnerdBackendWithinSessionClaim( runnerState: null, providerSessionIdentity: null, incompleteReason: "not_suspended", + notSuspendedRunId: typeof runnerState.runId === "string" ? runnerState.runId : null, }; } let providerState: Record; @@ -15206,6 +15240,20 @@ async function createRunnerdBackendWithinSessionClaim( return probe.exitCode === 0 && !probe.timedOut; }; + const moveInterruptedBootstrapSessionAside = async (): Promise => { + if (!remoteCommandRunner || !remoteRuntimeRoot || !remoteSessionRoot) return false; + const moved = await remoteCommandRunner.execute({ + command: "sh", + args: ["-c", + 'set -eu; for directory in "$1" "$2" "$3"; do test -d "$directory" && test ! -L "$directory" || exit 1; done; mv -- "$3" "$4"', + "paperclip-runner-bootstrap-restart", remoteRuntimeRoot, posix.dirname(remoteSessionRoot), remoteSessionRoot, + `${remoteSessionRoot}.bootstrap-restart-${Date.now()}-${randomUUID()}`], + bypassSession: true, + timeoutMs: 30_000, + }); + return moved.exitCode === 0 && !moved.timedOut; + }; + const recordInPlaceHarnessReuse = async ( providerSessionIdentity: Record, startedAtMs = Date.now(), @@ -15481,7 +15529,13 @@ async function createRunnerdBackendWithinSessionClaim( !state.runnerState || !state.providerSessionIdentity ) { - if (state.incompleteReason !== "unavailable" || backupAvailable || + const provenRestart = state.incompleteReason === "not_suspended" && !backupAvailable + && input.restartRecovery?.kind === "bootstrap_incomplete" + && state.notSuspendedRunId === input.execution.binding.runId + && hasRemoteBootstrapRestartProof(input.execution.binding.runId) + && await moveInterruptedBootstrapSessionAside() + && takeRemoteBootstrapRestartProof(input.execution.binding.runId); + if ((state.incompleteReason !== "unavailable" && !provenRestart) || backupAvailable || !(await claimUntouchedRemoteSession())) { throw new Error(`runner_harness_state_mismatch: resumed_${state.incompleteReason}`); }