diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 9a3f342ef43..4db05f6b55a 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -405,6 +405,68 @@ it("preserves the cause when a replacement runner's health cannot be verified", } }, 15_000); +it("retries a replacement whose staging failed before any process launched", async () => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-replacement-staging-")); + const stagingError = new Error("runtime context upload: no space left on device"); + const children: ReturnType[] = []; + const exits: Promise[] = []; + const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => { + // The first replacement fails during staging: nothing was dispatched. + if (launch.mock.calls.length === 2) return { + child: { pid: undefined, exitCode: null, kill: () => false }, + completion: Promise.reject(stagingError), + isExitConfirmed: () => true, + }; + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore" }); + children.push(child); + exits.push(new Promise(resolveExit => child.once("exit", () => resolveExit()))); + return { + child, + completion: new Promise((resolveExit, rejectExit) => { + child.once("error", rejectExit); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); + }), + isExitConfirmed: () => child.exitCode !== null || child.signalCode !== null, + }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(directory), stateDirectory: directory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, runnerProcessLauncher: launch, + }); + try { + const opened = await bundle.transport.request("thread/start", { cwd: directory }) as { thread: { id: string } }; + let failure: unknown = null; + const pump = (async () => { + for await (const _event of bundle.transport.notifications()) {} + })().catch(error => { failure = error; }); + children[0]!.kill("SIGKILL"); + // Recovery reports restoration only after the replacement authenticates + // as the single active connection to the durable PRP authority. + await vi.waitFor( + () => expect(bundle.evidence().diagnostics).toContain("runner process restored its durable PRP session"), + { timeout: 4_000 }, + ); + expect(launch).toHaveBeenCalledTimes(3); + expect(children).toHaveLength(2); + expect(bundle.evidence().runnerPid).toBe(children[1]!.pid); + expect(failure).toBeNull(); + // The restored runner still serves the same provider thread. + const resumed = await bundle.transport.request("thread/resume", { threadId: opened.thread.id }) as { thread: { id: string } }; + expect(resumed.thread.id).toBe(opened.thread.id); + expect(failure).toBeNull(); + void pump; + } finally { + const providerPid = bundle.evidence().codexPid; + if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } + await bundle.transport.detachControllerForRestart!(); + for (const child of children) child.kill("SIGKILL"); + await Promise.all(exits); + await rm(directory, { recursive: true, force: true }); + } +}, 15_000); + it.each(["alive", "pending_liveness", "pending_registration"] as const)( "bounds adopted runner authentication at its exact deadline with %s evidence", async (mode) => { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 440341a6d13..cfad7a837cb 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -993,6 +993,43 @@ describe("remote runner process supervision", () => { } }); + it("confirms no replacement process exists when staging fails before the launch RPC", async () => { + // A full sandbox disk rejects the runtime-context upload before dispatch. + const stagingError = new Error( + "/home/daytona/paperclip-workspace/.paperclip-upload-1.tar.gz: create: no space left on device", + ); + const execute = vi.fn(); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, + remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + ensureArtifact: async () => { throw stagingError; }, + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + await expect(handle.completion).rejects.toBe(stagingError); + expect(execute).not.toHaveBeenCalled(); + expect(handle.child.pid).toBeUndefined(); + expect(handle.isExitConfirmed?.()).toBe(true); + }); + + it.each(["failed", "timed out"] as const)("keeps a %s launch RPC unconfirmed because it may have started runnerd", async (outcome) => { + const execute = vi.fn(async () => ({ + exitCode: outcome === "failed" ? 1 : null, signal: null, timedOut: outcome === "timed out", stdout: "", stderr: "", + })); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, + remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + ensureArtifact: async () => undefined, + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + await expect(handle.completion).rejects.toThrow( + outcome === "failed" ? "runner_remote_process_launch_failed" : "runner_remote_process_launch_timed_out", + ); + expect(execute).toHaveBeenCalledTimes(1); + expect(handle.isExitConfirmed?.()).toBe(false); + }); + it("defers an in-flight launch signal until its exact identity is available", async () => { let enter!: () => void; let release!: () => void; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e923de72117..70d41a6cc0b 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -12574,6 +12574,9 @@ export function createRemoteRunnerProcessLauncher(input: { const runner = input.runner; return (spec) => { let exitConfirmed = false; + // Set immediately before the launch RPC. Until then no sandbox command + // could have started runnerd, so a staging failure proves no process. + let launchDispatched = false; let pendingSignal: NodeJS.Signals | null = null; let launchedIdentity: { nonce: string; @@ -12687,6 +12690,7 @@ export function createRemoteRunnerProcessLauncher(input: { // into its own session instead; its own bounded diagnostics directory and // durable PRP state remain the authorities, and the controller monitors // the exact persisted process identity below. + launchDispatched = true; const launchResult = await runner.execute({ command: "sh", args: [ @@ -12841,7 +12845,14 @@ export function createRemoteRunnerProcessLauncher(input: { stderr: diagnosticTail || stateSummary, }; } - })(); + })().catch((error: unknown) => { + // A failure before dispatch (for example a full sandbox disk rejecting + // the staged upload) has no process to supervise. Report the exit as + // confirmed so reconnect recovery can retry within its grace window + // instead of failing closed on an unverifiable replacement. + if (!launchDispatched) exitConfirmed = true; + throw error; + }); return { child, completion,