diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c08d42ab0c9..b23ef186ef2 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -651,7 +651,7 @@ describe("remote controller restart adoption", () => { it("signals nothing when the sandbox runner belongs to another identity", async () => { const { execute, target } = fixture({ runnerInstanceId: "different" }); await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, - { waitMs: 20, pollMs: 1 })).resolves.toBe("no_prior_runner"); + { waitMs: 20, pollMs: 1 })).resolves.toBe("stop_unproven"); expect(signalled(execute)).toEqual([]); }); it("signals nothing when no prior runner is alive", async () => { @@ -663,7 +663,7 @@ describe("remote controller restart adoption", () => { { waitMs: 20, pollMs: 1 })).resolves.toBe("absent_confirmed"); expect(signalled(execute)).toEqual([]); }); - it.each([["absent_confirmed", true], ["stopped", true], ["killed", true], ["stop_failed", false], ["no_prior_runner", false]] as const)( + it.each([["absent_confirmed", true], ["stopped", true], ["killed", true], ["stop_failed", false], ["stop_unproven", false], ["no_prior_runner", false]] as const)( "records a bootstrap restart proof only for a confirmed stop: %s", (stopped, proven) => { noteRemoteBootstrapRestartProven("run", "stopped"); expect(recordRemoteBootstrapRestartProof("run", stopped)).toBe(proven); @@ -674,7 +674,7 @@ describe("remote controller restart adoption", () => { const { execute, target } = fixture(); const verify = async () => { throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "probe_timeout" } as never); }; await expect(stopStaleRemoteRunnerBeforeBootstrap({ execution, runnerExecutionTarget: target as never, identity }, - { waitMs: 20, pollMs: 1, verify: verify as never })).resolves.toBe("no_prior_runner"); + { waitMs: 20, pollMs: 1, verify: verify as never })).resolves.toBe("stop_unproven"); expect(signalled(execute)).toEqual([]); }); }); @@ -2608,7 +2608,16 @@ describe("remote provider pack manifest", () => { next.digest = "sha256:" + createHash("sha256").update(canonical(next.payload)).digest("hex"); await writeFile(state.bundledPackManifestPath!, JSON.stringify(next)); await writeFile(state.bundledRunnerBinary!, Buffer.concat([daemon, Buffer.from("new release")])); + let recoveredRunnerStopped = false; const recoveryExecute = vi.fn(async (request: { args: string[] }) => { + if (request.args[2] === "paperclip-runner-recovery-check" && request.args[1]?.includes('kill -"$6"')) { + const exitCode = recoveredRunnerStopped ? 45 : 0; + recoveredRunnerStopped = true; + return { exitCode, timedOut: false, stderr: "", stdout: "" }; + } + if (request.args[2] === "paperclip-runner-recovery-check" && recoveredRunnerStopped) { + return { exitCode: 45, timedOut: false, stderr: "", stdout: "" }; + } if (request.args[0] === "--build-metadata") return { exitCode: 0, timedOut: false, stderr: "", stdout: JSON.stringify({ schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, @@ -2625,7 +2634,8 @@ describe("remote provider pack manifest", () => { runnerInstanceId: identity.runnerInstanceId, runnerIngressAuthorized: true, ...(opencode ? { runnerEnvironment: configuredEnvironmentProjection({ PAPERCLIP_OPENCODE_PERFORMANCE_DIAGNOSTICS: "1" }) } : {}), runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", - environmentId: "environment", leaseId: "lease", providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true }, runner: { execute: recoveryExecute }, + environmentId: "environment", leaseId: "lease", providerKey: "daytona", effectiveCapabilities: { runnerWebSocketIngress: true }, + runner: { execute: recoveryExecute, syncIn: vi.fn(async () => undefined), syncOut: vi.fn(async () => undefined) }, sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never, restartRecovery: { kind: "reattach_remote_runner", runId: remoteExecution.binding.runId, leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "graceful", recoveryRequestId: null, @@ -2684,9 +2694,14 @@ describe("remote provider pack manifest", () => { expect(admittedNativeProviderPack(knownResult.session!, nativeIdentity)).toEqual(coldAdmission); if (pinned) { const replacement = { identity: () => nativeIdentity } as never; - Object.assign(state.createBackend.mock.results.at(-1)!.value, { openSession: async () => replacement }); + const recoveredBackendMock = state.createBackend.mock.results.at(-1)!.value; + state.createBackend.mockReturnValueOnce({ kind: "test", openSession: async () => replacement } as never); + const freshPreparation = vi.spyOn(nativeExecutionDriver, "prepareNativeRemoteRunner").mockResolvedValue(undefined as never); + onTestFinished(() => freshPreparation.mockRestore()); const replaced = await knownBackend.openReplacementSession!({ identity: nativeIdentity }, checkpoint); + expect(freshPreparation).toHaveBeenCalled(); expect(replaced).toBe(replacement); + expect(state.createBackend.mock.results.at(-1)!.value).not.toBe(recoveredBackendMock); expect(admittedNativeProviderPack(replaced, nativeIdentity)).toBeNull(); } } @@ -16225,6 +16240,218 @@ describe("runnerd provider runtime wiring", () => { || request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false); }); + it.each(["codex", "acpx"] as const)("opens a remote %s replacement as a new runner, not the surviving runner it archived", async provider => { + const base = scopedNativeExecution(`replacement-after-reattach-${provider}`); + const current = { ...base, ...(provider === "acpx" ? { provider: { kind: "acpx", agent: "claude", model: null }, + session: { ...base.session, driverKind: "acpx_runtime" } } : {}) } as NativeExecutionInputV1; + const driver = vi.spyOn(nativeExecutionDriver, "createNativeRunnerExecutionDriver"); + onTestFinished(() => driver.mockRestore()); + const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, + runnerInstanceId: "admitted-codex", environmentLeaseId: "admitted-lease", + turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId); + await mkdir(join(root, "control-plane"), { recursive: true }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity))); + let installed: unknown = null; + if (provider === "acpx") { + const packRoot = await mkdtemp(join(tmpdir(), "paperclip-replacement-pack-")); + onTestFinished(() => rm(packRoot, { recursive: true, force: true })); + const digest = "sha256:" + "a".repeat(64); + const canonical = (value: unknown): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => + `${JSON.stringify(key)}:${canonical((value as Record)[key])}`).join(",")}}` : JSON.stringify(value); + const artifacts = Object.fromEntries(Object.entries({ + grokLauncher: "dist/providers/grok/launcher.cjs", nodeCommand: "node_modules/node/bin/node", + productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", + opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", + opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", + }).map(([key, path]) => [key, { path, sha256: digest }])); + const payload = { compatibility: { opencode: 1, acpx: 1 }, contractDigest: digest, + target: { platform: "linux", architecture: "x64" }, runnerSourceRevision: "a".repeat(40), + pins: { nodeMinimum: "24.11.0", codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion, + opencode: "1.18.34", acpx: "0.13.1", grok: QUALIFIED_ACPX_PROFILES.grok.agentRuntimeVersion, + claudeAcp: QUALIFIED_ACPX_PROFILES.claude.agentServerVersion, codexAcp: QUALIFIED_ACPX_PROFILES.codex.agentServerVersion }, + acpxProfileDigests: Object.fromEntries(["grok", "claude", "codex"].map(agent => + [agent, QUALIFIED_ACPX_PROFILES[agent as "grok" | "claude" | "codex"].commandDigest])), + distDigest: digest, artifacts, + bridgeDigest: "sha256:" + createHash("sha256").update(digest + "\n" + digest + "\n" + digest).digest("hex") }; + installed = { schema: "paperclip-runner/remote-provider-pack/v1", payload, + digest: "sha256:" + createHash("sha256").update(canonical(payload)).digest("hex") }; + state.bundledPackManifestPath = join(packRoot, "provider-pack.json"); + await writeFile(state.bundledPackManifestPath, JSON.stringify(installed)); + state.bundledRunnerBinary = join(packRoot, "linux-paperclip-runnerd"); + const daemon = Buffer.alloc(64); daemon.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); daemon.writeUInt16LE(62, 18); + await writeFile(state.bundledRunnerBinary, daemon); + state.resolveRemoteRunnerBinary.mockResolvedValue(state.bundledRunnerBinary); + state.resolveRunnerBinary.mockReturnValue(state.bundledRunnerBinary); + } + let runnerStopped = false, sessionArchived = false; + const execute = vi.fn(async (request: { args: string[] }) => { + const script = request.args[1] ?? ""; + if (script.includes(".continuity-break-")) sessionArchived = true; + if (request.args[2] === "paperclip-runner-recovery-check") { + const signalled = script.includes('kill -"$6"'); + const exitCode = runnerStopped ? 45 : 0; + if (signalled) runnerStopped = true; + return { exitCode, timedOut: false, stderr: "", stdout: "" }; + } + if (sessionArchived && script.includes("runner-state.json")) return { exitCode: 1, timedOut: false, stderr: "", stdout: "" }; + return { exitCode: 0, timedOut: false, stderr: "", + stdout: request.args[0] === "--build-metadata" ? JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + providerPacks: installed ? [{ root: "/opt/paperclip-runner/provider-pack", manifest: installed, immutable: true }] : [], + }) : script.includes("base64") ? Buffer.from(JSON.stringify({ ...identity, lifecycle: "suspended" })).toString("base64") + : script.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }; + }); + state.createBackend.mockClear(); state.createTransport.mockClear(); + const nativeIdentity = { ...current.binding, sessionId: current.session.normalizedSessionId! }; + const openSession = vi.fn(async () => ({ close: async () => undefined, identity: () => nativeIdentity })); + const freshOpenSession = vi.fn(async () => ({ close: async () => undefined, identity: () => nativeIdentity })); + state.createBackend.mockReturnValueOnce({ kind: "test", openSession, descriptor: () => ({ name: "reattached" }) } as never) + .mockReturnValueOnce({ kind: "test", openSession: freshOpenSession, descriptor: () => ({ name: "replacement" }) } as never); + const backend = await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: identity.runnerInstanceId, + runnerIngressAuthorized: true, + controllerPersistence: { bootId: "successor", runnerArtifact: { version: "admitted-version", digest: `sha256:${"a".repeat(64)}` } }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + environmentId: "environment", leaseId: identity.environmentLeaseId, providerKey: "daytona", runner: { execute }, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never, + restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId, leaseOwner: "controller", + controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } }, + }); + const factory = state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!; + factory(); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ prpIdentity: identity, adoptExistingRunner: { pid: 123 } }); + const spawnedBefore = driver.mock.calls.at(-1)![0].processStarted; + expect(backend.descriptor()).toEqual({ name: "reattached" }); + const replaced = await backend.openReplacementSession!({ identity: nativeIdentity } as never, {} as never); + expect(runnerStopped).toBe(true); + const signalIndex = execute.mock.calls.findIndex(([request]) => request.args[1]?.includes('kill -"$6"')); + const archiveIndex = execute.mock.calls.findIndex(([request]) => request.args[1]?.includes(".continuity-break-")); + expect(signalIndex).toBeGreaterThanOrEqual(0); + expect(signalIndex).toBeLessThan(archiveIndex); + expect(backend.descriptor()).toEqual({ name: "replacement" }); + expect(backend.bindManagedSession(replaced)).toBe(replaced); + expect(openSession).not.toHaveBeenCalled(); + expect(freshOpenSession).toHaveBeenCalledOnce(); + expect(state.createBackend).toHaveBeenCalledTimes(2); + expect(execute.mock.calls.some(([request]) => request.args[1]?.includes(".continuity-break-"))).toBe(true); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const replacement = state.createTransport.mock.calls.at(-1)![0]; + expect(replacement.adoptExistingRunner).toBeUndefined(); + expect(replacement.prpIdentity).toMatchObject({ runId: current.binding.runId, turnId: `turn-${current.binding.runId}`, + itemId: `item-${current.binding.runId}` }); + const spawnedAfter = driver.mock.calls.at(-1)![0].processStarted!; + expect(spawnedAfter).not.toBe(spawnedBefore); + expect(await Promise.race([spawnedAfter.then(() => "spawned"), Promise.resolve("pending")])).toBe("pending"); + if (provider === "acpx") { + expect(replacement).toMatchObject({ providerNodeCommand: expect.any(String), acpxSidecarPath: expect.any(String) }); + expect(admittedNativeProviderPack(replaced, nativeIdentity)).toMatchObject({ + authority: "core_verified_selection", providerPackDigest: (installed as { digest: string }).digest }); + } + }); + + it("refuses a remote replacement and keeps the saved state when the adopted runner's stop is unproven", async () => { + const provider = "codex" as "codex" | "acpx"; + const base = scopedNativeExecution(`replacement-unproven-stop`); + const current = { ...base, ...(provider === "acpx" ? { provider: { kind: "acpx", agent: "claude", model: null }, + session: { ...base.session, driverKind: "acpx_runtime" } } : {}) } as NativeExecutionInputV1; + const driver = vi.spyOn(nativeExecutionDriver, "createNativeRunnerExecutionDriver"); + onTestFinished(() => driver.mockRestore()); + const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, + runnerInstanceId: "admitted-codex", environmentLeaseId: "admitted-lease", + turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId); + await mkdir(join(root, "control-plane"), { recursive: true }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity))); + let installed: unknown = null; + if (provider === "acpx") { + const packRoot = await mkdtemp(join(tmpdir(), "paperclip-replacement-pack-")); + onTestFinished(() => rm(packRoot, { recursive: true, force: true })); + const digest = "sha256:" + "a".repeat(64); + const canonical = (value: unknown): string => Array.isArray(value) ? `[${value.map(canonical).join(",")}]` + : value && typeof value === "object" ? `{${Object.keys(value).sort().map(key => + `${JSON.stringify(key)}:${canonical((value as Record)[key])}`).join(",")}}` : JSON.stringify(value); + const artifacts = Object.fromEntries(Object.entries({ + grokLauncher: "dist/providers/grok/launcher.cjs", nodeCommand: "node_modules/node/bin/node", + productionLock: "pnpm-lock.yaml", opencodeCommand: "node_modules/.bin/opencode", + opencodeExecutable: "node_modules/opencode-ai/bin/opencode.exe", + opencodeProxy: "dist/cli/opencode-app-server-proxy.cjs", acpxSidecar: "dist/cli/acpx-runtime-sidecar.cjs", + }).map(([key, path]) => [key, { path, sha256: digest }])); + const payload = { compatibility: { opencode: 1, acpx: 1 }, contractDigest: digest, + target: { platform: "linux", architecture: "x64" }, runnerSourceRevision: "a".repeat(40), + pins: { nodeMinimum: "24.11.0", codex: QUALIFIED_ACPX_PROFILES.codex.agentRuntimeVersion, + opencode: "1.18.34", acpx: "0.13.1", grok: QUALIFIED_ACPX_PROFILES.grok.agentRuntimeVersion, + claudeAcp: QUALIFIED_ACPX_PROFILES.claude.agentServerVersion, codexAcp: QUALIFIED_ACPX_PROFILES.codex.agentServerVersion }, + acpxProfileDigests: Object.fromEntries(["grok", "claude", "codex"].map(agent => + [agent, QUALIFIED_ACPX_PROFILES[agent as "grok" | "claude" | "codex"].commandDigest])), + distDigest: digest, artifacts, + bridgeDigest: "sha256:" + createHash("sha256").update(digest + "\n" + digest + "\n" + digest).digest("hex") }; + installed = { schema: "paperclip-runner/remote-provider-pack/v1", payload, + digest: "sha256:" + createHash("sha256").update(canonical(payload)).digest("hex") }; + state.bundledPackManifestPath = join(packRoot, "provider-pack.json"); + await writeFile(state.bundledPackManifestPath, JSON.stringify(installed)); + state.bundledRunnerBinary = join(packRoot, "linux-paperclip-runnerd"); + const daemon = Buffer.alloc(64); daemon.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); daemon.writeUInt16LE(62, 18); + await writeFile(state.bundledRunnerBinary, daemon); + state.resolveRemoteRunnerBinary.mockResolvedValue(state.bundledRunnerBinary); + state.resolveRunnerBinary.mockReturnValue(state.bundledRunnerBinary); + } + let runnerStopped = false, sessionArchived = false; + const execute = vi.fn(async (request: { args: string[] }) => { + const script = request.args[1] ?? ""; + if (script.includes(".continuity-break-")) sessionArchived = true; + if (request.args[2] === "paperclip-runner-recovery-check") { + const signalled = script.includes('kill -"$6"'); + const exitCode = runnerStopped ? 45 : 0; + if (signalled) runnerStopped = true; + return { exitCode, timedOut: false, stderr: "", stdout: "" }; + } + if (sessionArchived && script.includes("runner-state.json")) return { exitCode: 1, timedOut: false, stderr: "", stdout: "" }; + return { exitCode: 0, timedOut: false, stderr: "", + stdout: request.args[0] === "--build-metadata" ? JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + providerPacks: installed ? [{ root: "/opt/paperclip-runner/provider-pack", manifest: installed, immutable: true }] : [], + }) : script.includes("base64") ? Buffer.from(JSON.stringify({ ...identity, lifecycle: "suspended" })).toString("base64") + : script.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }; + }); + state.createBackend.mockClear(); state.createTransport.mockClear(); + const nativeIdentity = { ...current.binding, sessionId: current.session.normalizedSessionId! }; + const openSession = vi.fn(async () => ({ close: async () => undefined, identity: () => nativeIdentity })); + const freshOpenSession = vi.fn(async () => ({ close: async () => undefined, identity: () => nativeIdentity })); + state.createBackend.mockReturnValueOnce({ kind: "test", openSession, descriptor: () => ({ name: "reattached" }) } as never); + const backend = await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: identity.runnerInstanceId, + runnerIngressAuthorized: true, + controllerPersistence: { bootId: "successor", runnerArtifact: { version: "admitted-version", digest: `sha256:${"a".repeat(64)}` } }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + environmentId: "environment", leaseId: identity.environmentLeaseId, providerKey: "daytona", runner: { execute }, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never, + restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId, leaseOwner: "controller", + controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } }, + }); + const factory = state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!; + factory(); + expect(state.createTransport.mock.calls.at(-1)![0]).toMatchObject({ prpIdentity: identity, adoptExistingRunner: { pid: 123 } }); + const spawnedBefore = driver.mock.calls.at(-1)![0].processStarted; + execute.mockImplementation(async (request: { args: string[] }) => request.args[2] === "paperclip-runner-recovery-check" + ? { exitCode: 0, timedOut: true, stderr: "", stdout: "" } + : { exitCode: 0, timedOut: false, stderr: "", stdout: "" }); + await expect(backend.openReplacementSession!({ identity: nativeIdentity } as never, {} as never)) + .rejects.toThrow("native_runner_replacement_stop_unproven"); + expect(execute.mock.calls.some(([request]) => request.args[1]?.includes(".continuity-break-"))).toBe(false); + expect(execute.mock.calls.some(([request]) => request.args[1]?.includes('kill -"$6"'))).toBe(false); + expect(freshOpenSession).not.toHaveBeenCalled(); + expect(state.createBackend).toHaveBeenCalledTimes(1); + expect(existsSync(join(root, "control-plane", "control-plane-state.json"))).toBe(true); + }); + it.each(["failed", "timeout", "succeeded"] as const)("retains the local controller checkpoint until remote continuity archiving succeeds: %s", async mode => { const current = scopedNativeExecution("failed-remote-continuity-archive"); state.createBackend.mockClear(); state.createTransport.mockClear(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index aec710c4219..452ee43d4ce 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6438,7 +6438,7 @@ export async function stopStaleRemoteRunnerBeforeBootstrap(input: { verify?: typeof verifyRemoteRunnerReattachment; waitMs?: number; pollMs?: number; -} = {}): Promise<"no_prior_runner" | "absent_confirmed" | "stopped" | "killed" | "stop_failed"> { +} = {}): Promise<"no_prior_runner" | "absent_confirmed" | "stopped" | "killed" | "stop_failed" | "stop_unproven"> { const target = input.runnerExecutionTarget; if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner) return "no_prior_runner"; const providerLeaseId = target.sandboxLeaseAcquisition?.providerLeaseId; @@ -6454,7 +6454,7 @@ export async function stopStaleRemoteRunnerBeforeBootstrap(input: { } catch (error) { // No runner with this run's exact identity and generation is alive here. return error instanceof RemoteRunnerRecoveryError && error.remoteRunnerRecovery.reason === "process_not_running" - ? "absent_confirmed" : "no_prior_runner"; + ? "absent_confirmed" : "stop_unproven"; } try { let outcome: "stopped" | "killed" = "stopped"; @@ -16801,11 +16801,15 @@ async function createRunnerdBackendWithinSessionClaim( await recordAdmittedRemoteProviderPack(providerPackAuthority, message => input.onLog?.("stderr", message)); } }; + let replacementBackend: ManagedRunnerdBackend | null = null; return { - bindManagedSession: wrapManagedSession, - providerAdmissionRequired: () => Boolean(artifactStorage - && (providerArtifacts || !legacyProviderContinuity && !liveProviderRecovery)), + bindManagedSession: (session, options) => replacementBackend + ? replacementBackend.bindManagedSession(session, options) : wrapManagedSession(session), + providerAdmissionRequired: () => replacementBackend?.providerAdmissionRequired + ? replacementBackend.providerAdmissionRequired() + : Boolean(artifactStorage && (providerArtifacts || !legacyProviderContinuity && !liveProviderRecovery)), providerAdmissionPersisted: artifactStorage ? async () => { + if (replacementBackend) return replacementBackend.providerAdmissionPersisted?.(); const backupOwner = sessionHarnessBackupOwners.get(authorityEpoch); if ((liveProviderRecovery || legacyProviderContinuity) && !providerArtifacts) return; if (!backupOwner || !expectedProviderPackManifest || !remoteCommandRunner) { @@ -16823,6 +16827,7 @@ async function createRunnerdBackendWithinSessionClaim( source => retainNativeProviderArtifacts({ db: input.db, storage: artifactStorage, owner: backupOwner, source, signal })); } : undefined, bindDiagnosticSession: session => { + if (replacementBackend) return replacementBackend.bindDiagnosticSession?.(session); const retained = nativeSessionDiagnostics(session); if (retained && retained !== diagnostics) { const lease = retained.activate(diagnosticOwner, input.onLog); @@ -16830,6 +16835,7 @@ async function createRunnerdBackendWithinSessionClaim( } else diagnostics.bind(session); }, sessionCheckpointPersisted: snapshot => { + if (replacementBackend) return replacementBackend.sessionCheckpointPersisted?.(snapshot); const staged = runnerControlPlane?.store.state.stagedRunAttachment; if (staged && snapshot.identity.runId === staged.targetIdentity.runId && snapshot.identity.sessionId === staged.targetIdentity.normalizedSessionId) { @@ -16838,7 +16844,7 @@ async function createRunnerdBackendWithinSessionClaim( } } }, - descriptor: () => backend.descriptor(), + descriptor: () => (replacementBackend ?? backend).descriptor(), openSession: async (sessionInput) => { cursorCloudRuntime?.register(); try { @@ -16859,12 +16865,33 @@ async function createRunnerdBackendWithinSessionClaim( } catch (error) { cursorCloudRuntime?.unregister(); throw error; } }, openReplacementSession: async (sessionInput, previous) => { + const reattachClaim = input.restartRecovery?.kind === "reattach_remote_runner" ? input.restartRecovery : null; + if (!reattachClaim && input.remoteWarmRecovery) { + throw new Error("native_runner_replacement_requires_fresh_launch"); + } + if (reattachClaim) { + const stopped = await stopStaleRemoteRunnerBeforeBootstrap({ execution: input.execution, + runnerExecutionTarget: input.runnerExecutionTarget, identity: durableIdentity ?? {}, onLog: input.onLog }); + if (stopped === "stop_failed" || stopped === "stop_unproven") { + throw new Error(`native_runner_replacement_${stopped}`); + } + } await measureNativeRunnerSpan( input.trace, "provider.session.archive_before_replacement", () => archiveContinuityState(previous), { parentName: "native.session.execute" }, ); + if (reattachClaim) { + const { remote: _remote, stagedAttachment: _staged, ...claim } = reattachClaim; + const fresh = await createRunnerdBackendWithinSessionClaim({ + ...input, + remoteWarmRecovery: undefined, + restartRecovery: { ...claim, kind: "bootstrap_incomplete" }, + }, sessionScopeId); + replacementBackend = fresh; + return fresh.openSession(sessionInput); + } cursorCloudRuntime?.register(); try { const session = await backend.openSession(sessionInput);