diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7fe7b47478f..adef62fa6af 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2781,6 +2781,8 @@ The goal-less regression in `native-session-resumption.test.ts` holds provider cleanup open and then exercises a normal follow-up and an explicit session reset. Control-port tests also cover terminal snapshots before accepted results and result-replay contention with the status writer's coordinator-first lock order. +Remote adoption tests must invoke actual external-state preparation. Inspecting +transport options alone does not exercise the resumed-lease harness checks. Run it against an isolated database with `PAPERCLIP_TEST_DATABASE_URL` when the host cannot start embedded Postgres: diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0db9d0df81c..6239b21ca02 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -2711,6 +2711,13 @@ The first production release containing recovery readers and redaction for both private recovery fields is the disk-independent rollback floor. Earlier development-reader probes do not establish a published rollback target. +Active remote adoption and compatible idle warm reuse carry an existing verified +Runner proof. Preparation revalidates that proof and the exact process's liveness, +then preserves its in-place harness state. The later protocol attachment still +authenticates the durable PRP authority. A provider lease acquisition described as +`resumed` does not establish that this already-running harness was suspended. +Ordinary resumed leases retain their verified suspension and backup checks. + For per-turn direct sandbox recovery, the executor reads the retained Runner's suspended lifecycle through the recorded sandbox lease and validates its complete identity and lack of pending delivery/cleanup. Only a lifecycle projection is diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 12c3d8eb3c3..c0cc1cf41d0 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -12611,6 +12611,53 @@ describe("runnerd provider runtime wiring", () => { runnerArtifact: artifact, adoptExistingRunner: { pid: 123, startedAt: "2026-09-19T10:00:00.000Z" } }); expect(state.resolveRemoteRunnerBinary).not.toHaveBeenCalled(); expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") || request.args.includes("--build-metadata"))).toBe(false); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise }; + await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") + || request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false); + }); + + it.each(["verified", "no-proof", "dead", "stale"] as const)("reuses only verified idle warm process during actual remote preparation: %s", async mode => { + const previous = scopedNativeExecution(`warm-live-preparation-${mode}`); + previous.session.lifecyclePolicy = { mode: "warm", idleTimeoutMs: 300_000 }; + const current = { ...previous, binding: { ...previous.binding, runId: `${previous.binding.runId}-next` } }; + const identity = { runId: previous.binding.runId, normalizedSessionId: previous.session.normalizedSessionId!, + runnerInstanceId: "retained-warm-runner", environmentLeaseId: "retained-lease", turnId: "prior-turn", itemId: "prior-item" }; + const root = await initializeScopedRunnerRoot(previous, identity.runnerInstanceId); + await writeRunnerAuthority(root, identity, "suspended"); + const projections = vi.spyOn(controllerRecovery, "resolveNativeControllerRecovery").mockResolvedValue({ kind: "no_prior_execution" }); + onTestFinished(() => projections.mockRestore()); + const db = leaseDb(current), ordinarySelect = db.select.bind(db); + db.select = ((fields?: Record) => fields?.status === heartbeatRuns.status && fields?.runnerProfileJson === heartbeatRuns.runnerProfileJson + ? priorNativeRunDb(previous).select(fields as never) : ordinarySelect(fields as never)) as Db["select"]; + let alive = true; + const execute = vi.fn(async (request: { args: string[] }) => ({ exitCode: request.args[2] === "paperclip-runner-recovery-check" && !alive ? 45 : 0, + timedOut: false, stderr: "", stdout: request.args[0] === "--build-metadata" ? JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + }) : request.args[0] === "--version" ? "codex-cli 0.160.0" : request.args[1]?.includes("base64") + ? Buffer.from(JSON.stringify({ ...identity, lifecycle: "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64") + : request.args[1]?.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" })); + const revalidate = vi.fn(async () => undefined); + const syncIn = vi.fn(async () => undefined); + await createRunnerdBackend({ db, execution: current, runnerInstanceId: identity.runnerInstanceId, + runnerIngressAuthorized: true, + ...(mode === "no-proof" ? {} : { remoteWarmRecovery: { identity, reuseProcess: true, previousExecution: previous, + checkpoint: {} as never, remote: { providerLeaseId: "retained-sandbox", remoteCwd: "/workspace" }, revalidate } }), + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment", + leaseId: identity.environmentLeaseId, providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "retained-sandbox" }, runner: { execute, syncIn } } as never }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise }; + if (mode === "dead") alive = false; + if (mode === "stale") revalidate.mockRejectedValueOnce(new Error("fixture_warm_owner_changed")); + if (mode === "verified") await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + else await expect(options.prepareExternalRunnerState()).rejects.toThrow(mode === "no-proof" ? "resumed_not_suspended" + : mode === "dead" ? "runner_remote_process_identity_unavailable" : "fixture_warm_owner_changed"); + expect(syncIn).not.toHaveBeenCalled(); + expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch") + || request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false); }); it.each(["claude", "codex"].flatMap(provider => (["per_turn", "warm"] as const).flatMap(policy => ["canonical", "resumed", "resumed-timeout", "local-receipt", "local-receipt-missing-terminal", "local-receipt-pending-command", "quarantined", "empty-runner", "empty-runner-quarantined", "nonempty-runner", "wrong-identity", "active", "foreign-scope", "no-stop-proof", "not-suspended", "pending-outbox", "pending-command", "missing-terminal", "ambiguous", "contradictory"].filter(mode => policy === "per_turn" || !mode.startsWith("resumed")).map(mode => [provider, policy, mode] as const))))( diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 133ae2fc2b6..e6418e282ce 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -13075,7 +13075,20 @@ async function createRunnerdBackendWithinSessionClaim( bytesTransferred: 0, }, }); - if (sandboxLeaseAcquisition?.outcome === "resumed") { + if (sandboxLeaseAcquisition?.outcome === "resumed" && adoptedProcess && + (input.restartRecovery?.kind === "reattach_remote_runner" || input.remoteWarmRecovery)) { + // Active recovery and a verified idle warm handoff adopt the + // exact surviving process, rather than restoring a suspended + // per-turn backup. Identity/lease/PID generation were verified + // above; refresh that proof after any preparation wait. The + // transport still authenticates the durable PRP authority. + await measureNativeRunnerSpan(input.trace, "sandbox.lease.resume", async () => { + await input.remoteWarmRecovery?.revalidate(); + if (!await adoptedProcess.isAlive()) { + throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }); + } + }); + } else if (sandboxLeaseAcquisition?.outcome === "resumed") { const reuseStartedAtMs = Date.now(); const state = await measureNativeRunnerSpan( input.trace,