diff --git a/server/src/services/native-runtime/native-bootstrap-claim-time-proof.integration.test.ts b/server/src/services/native-runtime/native-bootstrap-claim-time-proof.integration.test.ts new file mode 100644 index 00000000000..296a17c2251 --- /dev/null +++ b/server/src/services/native-runtime/native-bootstrap-claim-time-proof.integration.test.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { eq } from "drizzle-orm"; +import { expect, it } from "vitest"; +import { nativeRunFinalizations } from "@paperclipai/db"; +import { DurablePrpControlPlane } from "../../vendor/paperclip-runner/index.js"; +import { describeEmbeddedPostgres, nativeRestartFixture } from "../../__tests__/helpers/native-restart.js"; +import { nativeRunnerCheckpointRemoteBootstrapProof, registerNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { captureClaimTimeRemoteBootstrapProof, claimTimeRemoteBootstrapProofHolds } from "./native-session-executor.js"; + +describeEmbeddedPostgres("claim-time remote bootstrap proof", () => { + const environment = nativeRestartFixture(); + + it.each([ + { label: "turn-sent", oldCommands: ["run.prepare", "session.open", "turn.start"], holds: false }, + { label: "bootstrap-only", oldCommands: ["run.prepare", "session.open"], holds: true }, + ])("keeps the interrupted controller's proof after the new controller registers: $label", async ({ label, oldCommands, holds }) => { + const { db, runId, issueId, native } = await environment.seedRun(`CLAIM-PROOF-${label}`); + const { companyId, successor, runtimeRoot } = environment.current; + const binding = { companyId, issueId, runId }; + const identity = { runnerInstanceId: native.runnerInstanceId, environmentLeaseId: native.environmentLeaseId, runId, + normalizedSessionId: native.normalizedSessionId, turnId: native.turnId, itemId: native.itemId }; + const controller = (directory: string) => new DurablePrpControlPlane({ stateDirectory: join(runtimeRoot, directory), identity, + expectedRunnerVersion: "fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + const own = async (generation: number) => db.update(nativeRunFinalizations).set({ controllerBootId: successor.bootId, + controllerGeneration: generation, leaseOwner: randomUUID(), leaseExpiresAt: new Date(Date.now() + 60_000) }) + .where(eq(nativeRunFinalizations.runId, runId)); + const execution = { binding, session: { normalizedSessionId: native.normalizedSessionId } } as never; + const claim = { kind: "bootstrap_incomplete" as const, runId, leaseOwner: "recovered", controllerGeneration: 2, + providerAttempt: 1, restartKind: "hard" as const, recoveryRequestId: null }; + const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace" } as never; + const interrupted = controller(`${runId}-interrupted`); + const replacement = controller(`${runId}-replacement`); + try { + await own(1); + for (const type of oldCommands) interrupted.queueCommand(type, {}); + interrupted.store.state.connectionCount = 1; + await registerNativeRunnerCheckpoint({ db, authority: interrupted, bootId: successor.bootId, binding }); + await interrupted.stop(); + + await own(2); + await captureClaimTimeRemoteBootstrapProof({ db, execution, restartRecovery: claim, runnerExecutionTarget: target }); + expect(claimTimeRemoteBootstrapProofHolds(runId, native.runnerInstanceId)).toBe(holds); + + replacement.queueCommand("run.prepare", {}); + replacement.queueCommand("session.open", { reuse: "same_session" }); + await registerNativeRunnerCheckpoint({ db, authority: replacement, bootId: successor.bootId, binding }); + + expect(await nativeRunnerCheckpointRemoteBootstrapProof(db, binding, + { runnerInstanceId: native.runnerInstanceId, normalizedSessionId: native.normalizedSessionId })).toBe("bootstrap_incomplete"); + expect(claimTimeRemoteBootstrapProofHolds(runId, native.runnerInstanceId)).toBe(holds); + expect(claimTimeRemoteBootstrapProofHolds(runId, "another-runner")).toBe(false); + } finally { + await interrupted.stop(); + await replacement.stop(); + } + }); +}); diff --git a/server/src/services/native-runtime/native-runner-checkpoint.ts b/server/src/services/native-runtime/native-runner-checkpoint.ts index 91f3ac28609..9bbb5e95257 100644 --- a/server/src/services/native-runtime/native-runner-checkpoint.ts +++ b/server/src/services/native-runtime/native-runner-checkpoint.ts @@ -302,6 +302,17 @@ export type NativeRemoteBootstrapProof = "bootstrap_incomplete" | "session_opene export async function nativeRunnerCheckpointRemoteBootstrapProof( db: Pick, binding: Binding, expected: { runnerInstanceId: string; normalizedSessionId: string }, ): Promise { + const evidence = await nativeRunnerCheckpointRemoteBootstrapEvidence(db, binding, expected.normalizedSessionId); + return evidence?.runnerInstanceId === expected.runnerInstanceId ? evidence.proof : null; +} + +/** + * The same proof, with the runner instance the checkpoint names. A caller that + * must hold the proof across a new controller's registration reads it first. + */ +export async function nativeRunnerCheckpointRemoteBootstrapEvidence( + db: Pick, binding: Binding, normalizedSessionId: string, +): Promise<{ proof: NativeRemoteBootstrapProof; runnerInstanceId: string } | null> { try { const limit = BOOTSTRAP_CHECKPOINT_PARTS.size + MAX_BOOTSTRAP_CHECKPOINT_COMMANDS + MAX_BOOTSTRAP_CHECKPOINT_EVENTS; const parts = await db.select({ part: nativeRunnerCheckpoints.part }).from(nativeRunnerCheckpoints).where(and( @@ -314,8 +325,8 @@ export async function nativeRunnerCheckpointRemoteBootstrapProof( || parts.some(row => !BOOTSTRAP_CHECKPOINT_PARTS.has(row.part) && !/^(?:event|command):\d+$/.test(row.part))) return null; const checkpoint = await loadNativeRunnerCheckpoint(db, binding); if (!checkpoint || checkpoint.predecessor !== null - || checkpoint.recoveryEvidence.runnerInstanceId !== expected.runnerInstanceId - || checkpoint.recoveryEvidence.normalizedSessionId !== expected.normalizedSessionId) return null; + || checkpoint.recoveryEvidence.normalizedSessionId !== normalizedSessionId) return null; + const runnerInstanceId = checkpoint.recoveryEvidence.runnerInstanceId; // The decoder above validated these canonical v2 records. Rebuild the // controller state in memory without writing files under recovery locks. const read = (part: string) => { @@ -335,8 +346,8 @@ export async function nativeRunnerCheckpointRemoteBootstrapProof( .map(seq => read(`event:${seq}`)), }; if ((state.identity as { runId?: unknown } | undefined)?.runId !== binding.runId) return null; - if (runnerdStateIsIncompleteBootstrap(state)) return "bootstrap_incomplete"; - return runnerdStateHasNoProviderTurn(state) ? "session_opened_without_turn" : null; + if (runnerdStateIsIncompleteBootstrap(state)) return { proof: "bootstrap_incomplete", runnerInstanceId }; + return runnerdStateHasNoProviderTurn(state) ? { proof: "session_opened_without_turn", runnerInstanceId } : null; } catch { return null; } diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b61053f502c..c5a7a79eb32 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -21462,7 +21462,7 @@ describe("runnerd provider runtime wiring", () => { "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_unproven", "interrupted_bootstrap_unopened_other_run", "interrupted_bootstrap_unopened_active_turn", "interrupted_bootstrap_unprepared", "interrupted_bootstrap_unprepared_unproven", "interrupted_bootstrap_unprepared_other_run", "interrupted_bootstrap_unprepared_suspended", "interrupted_bootstrap_unprepared_backup", - "interrupted_bootstrap_unprepared_unrecorded", "interrupted_bootstrap_unprepared_connected"] + "interrupted_bootstrap_unprepared_unrecorded", "interrupted_bootstrap_unprepared_connected", "interrupted_bootstrap_connected_checkpoint_proven"] .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), ...["interrupted_bootstrap_unlaunched", "interrupted_bootstrap_unlaunched_marker", "interrupted_bootstrap_unlaunched_backup", "interrupted_bootstrap_unlaunched_archived", "interrupted_bootstrap_unlaunched_nested_state"] @@ -21544,10 +21544,12 @@ describe("runnerd provider runtime wiring", () => { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unprepared") && command.args[1].includes("codex-provider-state.json")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; - if (command.args?.[2] === "paperclip-runner-recovery-identity" && scenario === "interrupted_bootstrap_unprepared_unrecorded") return { + if (command.args?.[2] === "paperclip-runner-recovery-identity" && ["interrupted_bootstrap_unprepared_unrecorded", + "interrupted_bootstrap_connected_checkpoint_proven"].includes(scenario)) return { exitCode: 0, timedOut: false, stderr: "", stdout: "exitednonce\n4242\n2026-11-12T00:32:10.725Z\nrunner-new-in-retained-workspace\nlinux:abcd-1234:100\n" }; - if (command.args?.[2] === "paperclip-runner-recovery-check" && scenario === "interrupted_bootstrap_unprepared_unrecorded") return { + if (command.args?.[2] === "paperclip-runner-recovery-check" && ["interrupted_bootstrap_unprepared_unrecorded", + "interrupted_bootstrap_connected_checkpoint_proven"].includes(scenario)) return { exitCode: 45, timedOut: false, stdout: "", stderr: "" }; if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unopened") && command.args[1].includes("codex-provider-state.json") && !command.args[1].includes("runner-state.json")) return { @@ -21563,8 +21565,10 @@ describe("runnerd provider runtime wiring", () => { lifecycle: scenario.startsWith("interrupted_bootstrap_unopened") || scenario.endsWith("_suspended") ? "suspended" : scenario.startsWith("interrupted_bootstrap_unprepared") ? "connecting" : "ready", outbox: [], pendingTerminalDelivery: null, - ...(scenario.startsWith("interrupted_bootstrap_unprepared") ? { lastControllerCommandSeq: 0, reconnectCount: 0, - processedCommands: {}, lastConnectionProtocolVersion: scenario.endsWith("_connected") ? 2 : null, + ...(scenario.startsWith("interrupted_bootstrap_unprepared") || scenario === "interrupted_bootstrap_connected_checkpoint_proven" + ? { lastControllerCommandSeq: scenario === "interrupted_bootstrap_connected_checkpoint_proven" ? 2 : 0, reconnectCount: 0, + processedCommands: {}, lastConnectionProtocolVersion: scenario.endsWith("_connected") + || scenario === "interrupted_bootstrap_connected_checkpoint_proven" ? 2 : null, environmentLeaseId: "lease-resumed", turnId: `turn-${execution.binding.runId}`, itemId: `item-${execution.binding.runId}` } : {}) })).toString("base64"), }; if (command.args?.[1]?.includes("base64")) return { @@ -21572,6 +21576,10 @@ describe("runnerd provider runtime wiring", () => { }; return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; }); + const controllerProof = vi.spyOn(runnerCheckpoint, "nativeRunnerCheckpointRemoteBootstrapEvidence") + .mockResolvedValue(scenario === "interrupted_bootstrap_connected_checkpoint_proven" + ? { proof: "session_opened_without_turn", runnerInstanceId: "runner-new-in-retained-workspace" } : null); + onTestFinished(() => controllerProof.mockRestore()); const runtimeContext = nativeRuntimeContextFixture(); const executionWithContext = homeMode === "preconfigured" ? { ...execution, schema: "paperclip.native-execution-input.v7" as const, @@ -21686,7 +21694,7 @@ describe("runnerd provider runtime wiring", () => { noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped"); } if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened" || scenario === "interrupted_bootstrap_unprepared" - || scenario === "interrupted_bootstrap_unprepared_unrecorded") { + || scenario === "interrupted_bootstrap_unprepared_unrecorded" || scenario === "interrupted_bootstrap_connected_checkpoint_proven") { await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); const siblings = await readdir(dirname(sessionRoot)); expect(siblings.filter(name => name.startsWith(`${basename(sessionRoot)}.bootstrap-restart-`))).toHaveLength(1); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 9e2c3380d83..fd2e9d55671 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -161,7 +161,7 @@ import { import { lstat, readFile, readdir, readlink } from "node:fs/promises"; import { hostname, tmpdir, userInfo } from "node:os"; import { DatabaseSync } from "node:sqlite"; -import { NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES, loadNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader, loadNativeUnclaimedBootstrapCheckpoint, nativeRunnerCheckpointPersistence, nativeRunnerNeverAuthenticated, nativeRunnerCheckpointRegistration, restoreNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; +import { NATIVE_CHECKPOINT_HEADER_INSPECTION_MAX_BYTES, loadNativeRunnerCheckpoint, loadNativeRunnerCheckpointHeader, loadNativeUnclaimedBootstrapCheckpoint, nativeRunnerCheckpointPersistence, nativeRunnerCheckpointRemoteBootstrapEvidence, nativeRunnerNeverAuthenticated, nativeRunnerCheckpointRegistration, restoreNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; import { REMOTE_CONTROLLER_LEASE_MS, REMOTE_RUNNER_RECONNECT_GRACE_MS, localRunnerProcessReconnectGraceMs } from "./native-runner-recovery-contract.js"; import type { NativeTaskSessionPublicationContext } from "./native-task-session-publication.js"; import { basename, dirname, join, posix, resolve } from "node:path"; @@ -6283,6 +6283,7 @@ async function migrateRunnerdStateRootForExecution(input: { if (input.restartRecovery?.kind === "reattach_remote_runner" && !existsSync(scoped)) { throw new Error("runner_state_identity_mismatch: remote_reattach_root_missing"); } + await captureClaimTimeRemoteBootstrapProof(input); await restoreRemoteBootstrapCheckpointForRecovery({ ...input, root: scoped }); if (existsSync(scoped)) { if (!isSafeNativeStateDirectory(scoped)) { @@ -6926,6 +6927,34 @@ export function recordRemoteBootstrapRestartProof(runId: string, return true; } +/** + * The controller checkpoint proof read before this server registers its own + * controller for the run. A new controller rewrites that checkpoint with its + * own bootstrap commands, so a later read would no longer describe what the + * interrupted runner received. + */ +const claimTimeRemoteBootstrapProofs = new Map(); + +export async function captureClaimTimeRemoteBootstrapProof(input: { + db: Pick; + execution: NativeExecutionInput; + restartRecovery?: NativeRestartRecoveryClaim; + runnerExecutionTarget?: AdapterExecutionTarget | null; +}): Promise { + const runId = input.execution.binding.runId; + claimTimeRemoteBootstrapProofs.delete(runId); + const target = input.runnerExecutionTarget; + if (input.restartRecovery?.kind !== "bootstrap_incomplete" || input.restartRecovery.runId !== runId + || target?.kind !== "remote" || target.transport !== "sandbox" || target.runnerTask) return; + const evidence = await nativeRunnerCheckpointRemoteBootstrapEvidence(input.db, input.execution.binding, + nativeSessionKey(input.execution)); + if (evidence) claimTimeRemoteBootstrapProofs.set(runId, evidence.runnerInstanceId); +} + +export function claimTimeRemoteBootstrapProofHolds(runId: string, runnerInstanceId: unknown): boolean { + return typeof runnerInstanceId === "string" && claimTimeRemoteBootstrapProofs.get(runId) === runnerInstanceId; +} + export function hasRemoteBootstrapRestartProof(runId: string): boolean { return provenRemoteBootstrapRestarts.has(runId); } @@ -16796,7 +16825,8 @@ async function createRunnerdBackendWithinSessionClaim( if (!runnerState || runnerState.runnerInstanceId !== input.runnerInstanceId || runnerState.runId !== runId || runnerState.normalizedSessionId !== nativeSessionKey(input.execution) || typeof runnerState.lifecycle !== "string" || runnerState.lifecycle === "suspended") return null; - if (!hasRemoteBootstrapRestartProof(runId) && !(remoteRunnerStateNeverConnected(runnerState) + if (!hasRemoteBootstrapRestartProof(runId) && !((remoteRunnerStateNeverConnected(runnerState) + || claimTimeRemoteBootstrapProofHolds(runId, runnerState.runnerInstanceId)) && recordRemoteBootstrapRestartProof(runId, await stopStaleRemoteRunnerBeforeBootstrap({ execution: input.execution, runnerExecutionTarget: input.runnerExecutionTarget, identity: runnerState, onLog: input.onLog, })))) return null; @@ -16882,12 +16912,16 @@ async function createRunnerdBackendWithinSessionClaim( throw new Error("runner_harness_state_mismatch: runner_identity"); } if (runnerState.lifecycle !== "suspended") { + const notSuspendedRunId = typeof runnerState.runId === "string" ? runnerState.runId : null; + if (source === "host_preparation" && notSuspendedRunId === input.execution.binding.runId) { + await inspectOwnUnpreparedBootstrap(); + } return { complete: false, runnerState: null, providerSessionIdentity: null, incompleteReason: "not_suspended", - notSuspendedRunId: typeof runnerState.runId === "string" ? runnerState.runId : null, + notSuspendedRunId, }; } let providerState: Record;