fix(native): resume a remote Runner after a settled suspension

fix(native): resume a remote Runner after a settled suspension

A scheduled checkpoint retry can retain a remote reattach claim after
closing its failed predecessor cleanly suspends and exits the Runner.
Resume only that exact settled source after confirmed exit and a fresh
identity/state check. Preserve pending work and uncertain-exit fences.

Admit the last already-charged provider attempt and share one bounded
recovery deadline with the final state read, retaining closed diagnostic
labels on read failures. Cover fresh ACPX Runner relaunch on attempts
two and three through actual processes and the durable close barrier.

Six affected suites pass 1,579 tests, with 14 environment-gated skips;
all server typecheck layers pass on Node 25.6.1. The complete provider
turn continuation failure remains preserved as follow-up #19751.

Refs: #19366

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Paperclip <noreply@paperclip.ing>
Co-Authored-By: Paperclip <Paperclip-Paperclip@users.noreply.github.com>

server/src/services/__tests__/run-failure-diagnostics.test.ts

@@ -239,6 +239,18 @@ describe("run failure diagnostics", () => {
       resultJson: { reviewedChatBinding: { denialReason: "owner_mismatch" } } }), {}).execution).toEqual({});
   });
 
+  it("exports how an exited remote runner last left its durable state", () => {
+    const thrown = new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running",
+      runnerLifecycle: "suspended", runnerSettled: true });
+    expect(sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown })).execution).toEqual({
+      remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running",
+      remoteRunnerLifecycle: "suspended", remoteRunnerSettled: true,
+    });
+    expect(collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", resultJson: {
+      remoteRunnerRecovery: { stage: "process_verification", reason: "process_not_running", runnerLifecycle: "private-state", runnerSettled: "yes" },
+    } }), {}).execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" });
+  });
+
   it("exports the remote recovery stage and reason from an error thrown before the result was saved", () => {
     const thrown = Object.assign(new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }),
       { path: "private-path", pid: 123 });

server/src/services/native-runtime/native-controller-recovery.integration.test.ts

@@ -150,7 +150,8 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d
     const sourceClaim = { runId: fixture.scope.runId, remote: { providerLeaseId, remoteCwd } };
     const failure = await verifyRemoteRunnerReattachment({ claim: sourceClaim, target: target as never, identity,
       runId: fixture.scope.runId, normalizedSessionId: fixture.scope.normalizedSessionId }).catch(error => error);
-    expect(readRemoteRunnerRecoveryError(failure)).toEqual({ stage: "process_verification", reason: "process_not_running" });
+    expect(readRemoteRunnerRecoveryError(failure)).toEqual({ stage: "process_verification", reason: "process_not_running",
+      runnerLifecycle: "ready", runnerSettled: true });
     expect(nativeSessionFailureSourceCode(failure)).toBe("native_session_interrupted");
     await expect(verifyRemoteRunnerReattachment({ claim: { ...sourceClaim, runId }, target: target as never,
       identity, runId, normalizedSessionId: fixture.scope.normalizedSessionId })).rejects.toThrow("native_remote_recovery_lease_mismatch");

server/src/services/native-runtime/native-restart-recovery.ts

@@ -60,6 +60,8 @@ export type NativeRestartKind = "hot" | "hard" | "graceful";
 /** A recovery claim for one run failed unexpectedly and was deferred. */
 export const NATIVE_RESTART_RECOVERY_FAILED = "restart_recovery_failed";
 
+export const NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT = 3;
+
 export type NativeRestartRecoveryClaim =
   | {
       kind: "reattach_existing_runner";
@@ -643,7 +645,7 @@ export function classifyNativeRunnerRecoveryEvidence(input: {
     };
   }
   // Continuing an attempt already counted needs no budget; a new one does.
-  if (input.resumeChargesAttempt !== false && (input.providerAttempt ?? 0) >= 3)
+  if (input.resumeChargesAttempt !== false && (input.providerAttempt ?? 0) >= NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT)
     return { claimKind: null, reason: "execution_recovery_budget_exhausted" };
   const checkpointIdentityMatches =
     input.checkpointIdentityMatches ?? input.hasCheckpoint;

server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts

@@ -3,13 +3,15 @@ import { enableRunFailureReporting } from "../../__tests__/helpers/run-failure-r
 import { NativeControllerRecoveryWaitingError } from "./native-controller-recovery.js";
 import { claimNativeSessionResumptions } from "./native-finalization-reconciler.js";
 import { hasNativeLocalProcessStop } from "../native-local-process-stop.js";
-import { randomUUID } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { createRequire } from "node:module";
-import { spawn } from "node:child_process";
+import { execFile, spawn } from "node:child_process";
 import { existsSync } from "node:fs";
-import { mkdir, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readFile, readdir, rename, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { resolve } from "node:path";
+import { join, resolve } from "node:path";
+import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, QUALIFIED_ACPX_PROFILES } from "@paperclipai/paperclip-runner";
+import { createCapabilityRunnerdCodexTransport, defaultCapabilityRunnerdBinary, readRunnerdArtifactBinding, type CapabilityRunnerdCodexTransportOptions } from "@paperclipai/paperclip-runner/live";
 import { and, eq, inArray, sql } from "drizzle-orm";
 import {
   activityLog,
@@ -63,6 +65,7 @@ import {
   restoreRemoteBootstrapCheckpointForRecovery,
   runnerdStateProvesIncompleteBootstrap,
   runnerdStateProvesNoRemoteProviderTurn,
+  verifyRemoteRunnerOrSettledExit,
 } from "./native-session-executor.js";
 import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
 import {
@@ -121,6 +124,148 @@ describeEmbeddedPostgres("native runner restart recovery with real processes", (
   let successor: typeof environment.current.successor;
   beforeAll(() => { ({ temporary, runtimeRoot, companyId, agentId, successor } = environment.current); });
 
+  realProcessIt.each([2, 3])("relaunches scheduled attempt %i after a fresh ACPX Runner rejects turn.start and closes cleanly", async providerAttempt => {
+    const root = await mkdtemp(join(tmpdir(), "fresh-runner-settled-retry-"));
+    const fault = join(root, "reject-turn"), journal = join(root, "sidecar-calls"), shim = join(root, "sidecar.cjs");
+    const binary = defaultCapabilityRunnerdBinary();
+    const fake = resolve(import.meta.dirname, "../../../../packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar");
+    const digest = (bytes: string | Buffer) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`;
+    const script = `const { spawn } = require("node:child_process");
+const { createInterface } = require("node:readline");
+const { existsSync, appendFileSync } = require("node:fs");
+const child = spawn(${JSON.stringify(fake)}, ["--mode", "source-reconcile", "--profile-digest", ${JSON.stringify(QUALIFIED_ACPX_PROFILES.claude.commandDigest)}, "--journal", ${JSON.stringify(journal)}, "--source-fault", ${JSON.stringify(fault)}]);
+child.stderr.pipe(process.stderr);
+createInterface({ input: child.stdout }).on("line", line => {
+  const response = JSON.parse(line);
+  if (response.result?.sidecarPid) response.result.sidecarPid = process.pid;
+  process.stdout.write(JSON.stringify(response) + "\\n");
+});
+createInterface({ input: process.stdin }).on("line", line => {
+  const request = JSON.parse(line);
+  if (request.command === "turn.start" && existsSync(${JSON.stringify(fault)})) {
+    appendFileSync(${JSON.stringify(journal)}, JSON.stringify(request) + "\\n");
+    process.stdout.write(JSON.stringify({ protocolVersion: request.protocolVersion, id: request.id, ok: false,
+      error: { code: "SESSION_RESUME_REQUIRED", message: "Resume the provider session again.", retryable: true } }) + "\\n");
+  } else child.stdin.write(line + "\\n");
+}).on("close", () => child.stdin.end());
+child.on("exit", code => process.exit(code ?? 1));
+process.on("SIGTERM", () => child.kill("SIGTERM"));
+`;
+    await writeFile(shim, script);
+    const nodeDigest = digest(await readFile(process.execPath));
+    const identity = { runId: "fresh-resume-run", normalizedSessionId: "checkpointed-session", runnerInstanceId: "fresh-runner",
+      environmentLeaseId: "same-lease", turnId: "same-turn", itemId: "same-item" };
+    const stateDirectory = join(root, ".paperclip-runtime", "paperclip-runner", "sessions",
+      createHash("sha256").update(identity.normalizedSessionId).digest("hex"));
+    const bundles: ReturnType<typeof createCapabilityRunnerdCodexTransport>[] = [];
+    const children: ReturnType<typeof spawn>[] = [];
+    const launches = vi.fn();
+    const create = (directory: string, resume?: CapabilityRunnerdCodexTransportOptions["resumeProviderSession"],
+      adoptExistingRunner?: CapabilityRunnerdCodexTransportOptions["adoptExistingRunner"]) => {
+      const bundle = createCapabilityRunnerdCodexTransport({
+        runnerBinary: binary, runnerArtifact: readRunnerdArtifactBinding(binary), stateDirectory: directory,
+        provider: "acpx", acpxAgent: "claude", prpIdentity: identity, resumeProviderSession: resume, adoptExistingRunner,
+        lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, closeGraceMs: 3_000,
+        controlPlaneRegistration: async authority => {
+          await authority.start();
+          return { connectUrl: authority.connectUrl, release: () => authority.stop() };
+        },
+        prepareLaunch: async input => ({ configurationId: "qualified-fixture", requirementsDigest: runnerLaunchRequirementsDigest(input),
+          provider: createRunnerProviderLaunch(input, { options: {}, root: directory, cwd: root, codexHome: root,
+            codexCommand: "", providerNodeCommand: process.execPath, opencodeProxyPath: "", acpxSidecarPath: shim }) }),
+        runnerProcessLauncher: spec => {
+          launches(adoptExistingRunner);
+          const child = spawn(binary, [...spec.args, "--acpx-launch-authority-digest", digest("fixture authority"),
+            "--acpx-sidecar-command", process.execPath, "--acpx-sidecar-command-sha256", nodeDigest,
+            "--acpx-sidecar-script", shim, "--acpx-sidecar-script-sha256", digest(script)],
+          { env: { PATH: process.env.PATH, ...spec.environment }, stdio: ["ignore", "ignore", "pipe"], detached: true });
+          children.push(child);
+          let stderr = "";
+          child.stderr?.on("data", bytes => { stderr = (stderr + bytes).slice(-16_384); });
+          return { child, completion: new Promise((resolveExit, reject) => {
+            child.once("error", reject);
+            child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr }));
+          }) };
+        },
+      });
+      bundles.push(bundle);
+      return bundle;
+    };
+    const expectCompleted = async (bundle: ReturnType<typeof create>) => {
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      try {
+        await Promise.race([
+          (async () => {
+            for await (const event of bundle.transport.notifications()) if (event.method === "turn/completed") return;
+            throw new Error("Fixture turn ended without completion");
+          })(),
+          new Promise<never>((_, reject) => { timer = setTimeout(() => reject(new Error("Fixture turn completion timed out")), 5_000); }),
+        ]);
+      } finally { clearTimeout(timer); }
+    };
+    try {
+      const seedDirectory = stateDirectory;
+      const seed = create(seedDirectory);
+      const opened = await seed.transport.request("thread/start", { cwd: root, model: "fixture-model", dynamicTools: [] });
+      const thread = opened.thread as { id: string; sessionId: string };
+      await seed.transport.request("turn/start", { input: [{ type: "text", text: "Save the provider checkpoint." }] });
+      await expectCompleted(seed);
+      await seed.transport.close();
+      const provider = JSON.parse(await readFile(join(seedDirectory, "runner", "acpx-provider-state.json"), "utf8"));
+      const resume = { driverSessionId: thread.id, providerSessionId: thread.sessionId, providerIdentity: provider.identity };
+      await writeFile(fault, "reject-turn-start");
+      const fresh = create(stateDirectory, resume);
+      await fresh.transport.request("thread/read", { threadId: thread.id, includeTurns: false });
+      await fresh.transport.request("thread/resume", { threadId: thread.id, cwd: root, model: "fixture-model", dynamicTools: [] });
+      expect(launches.mock.calls).toEqual([[undefined], [undefined]]);
+      const child = children.at(-1)!;
+      const pid = child.pid!;
+      expect(pid).not.toBe(children[0]!.pid);
+      let fingerprint = "linux:abcd-1234:100";
+      if (process.platform === "linux") {
+        const stat = await readFile(`/proc/${pid}/stat`, "utf8");
+        fingerprint = `linux:${(await readFile("/proc/sys/kernel/random/boot_id", "utf8")).trim()}:${stat.slice(stat.lastIndexOf(") ") + 2).trim().split(/\s+/)[19]}`;
+      }
+      await writeFile(join(stateDirectory, "runner", "runner-process.identity"),
+        `fixture_nonce\n${pid}\n${new Date().toISOString()}\n${identity.runnerInstanceId}\n${fingerprint}\n`);
+      await expect(fresh.transport.request("turn/start", { input: [{ type: "text", text: "Resume saved work." }] }))
+        .rejects.toThrow("classification=session_resume_required");
+      await fresh.transport.close();
+      await vi.waitFor(() => expect(child.exitCode).not.toBeNull(), { timeout: 5_000 });
+      const resumedControl = JSON.parse(await readFile(join(stateDirectory, "control-plane", "control-plane-state.json"), "utf8"));
+      expect(resumedControl.committedEvents).toEqual(expect.arrayContaining([expect.objectContaining({ eventType: "session.resumed" })]));
+      const saved = JSON.parse(await readFile(join(stateDirectory, "runner", "runner-state.json"), "utf8"));
+      expect(saved).toMatchObject({ ...identity, lifecycle: "suspended", outbox: [] });
+      expect(saved.pendingTerminalDelivery ?? null).toBeNull();
+      expect(saved.pendingProviderCleanup ?? null).toBeNull();
+      expect(saved.warmTransition ?? null).toBeNull();
+      const runner = { execute: async (request: { command: string; args: string[]; timeoutMs?: number }) =>
+        new Promise<{ stdout: string; stderr: string; exitCode: number; timedOut: boolean }>(resolveResult => {
+          execFile(request.command, request.args, { timeout: request.timeoutMs }, (error, stdout, stderr) => resolveResult({
+            stdout, stderr, exitCode: error ? Number(error.code) : 0, timedOut: error?.killed ?? false,
+          }));
+        }) };
+      const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: root, runner,
+        sandboxLeaseAcquisition: { providerLeaseId: "same-sandbox", outcome: "resumed" as const } };
+      const adopted = await verifyRemoteRunnerOrSettledExit({
+        claim: { runId: identity.runId, remote: { providerLeaseId: "same-sandbox", remoteCwd: root } },
+        target: target as never, identity, runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, providerAttempt,
+      });
+      expect(adopted).toBeUndefined();
+      await rm(fault);
+      const retried = create(stateDirectory, resume, adopted);
+      await retried.transport.request("thread/read", { threadId: thread.id, includeTurns: false });
+      await retried.transport.request("thread/resume", { threadId: thread.id, cwd: root, model: "fixture-model", dynamicTools: [] });
+      expect(launches.mock.calls).toEqual([[undefined], [undefined], [undefined]]);
+      expect(retried.evidence().runnerPid).toBeGreaterThan(0);
+      expect(retried.evidence().runnerPid).not.toBe(pid);
+    } finally {
+      await Promise.allSettled(bundles.map(bundle => bundle.transport.close()));
+      for (const child of children) if (child.pid && child.exitCode === null && child.signalCode === null) await stopOwnedProcessGroup(child.pid, root);
+      await rm(root, { recursive: true, force: true });
+    }
+  }, 45_000);
+
   it.each(["issue", "run", "coordinator"] as const)("defers a busy %s row without blocking replacement startup", async kind => {
     const fixture = await seedRun(`OVERLAP-BUSY-${kind}`);
     let acquired!: () => void, release!: () => void;

server/src/services/native-runtime/native-session-executor.test.ts

@@ -428,6 +428,7 @@ import {
   REMOTE_RUNNER_AUTHORITY_ARCHIVE_SCRIPT,
   verifyRemoteRunnerReattachment,
   stopStaleRemoteRunnerBeforeBootstrap,
+  verifyRemoteRunnerOrSettledExit,
   readRemoteProviderPackManifest,
   prepareRemoteProviderPackOnTarget,
   resolveRemoteProviderPackSource,
@@ -881,13 +882,154 @@ describe("remote controller restart adoption", () => {
     await vi.advanceTimersByTimeAsync(30_000);
     const error = await outcome;
     const diagnostic = readRemoteRunnerRecoveryError(error);
-    expect(diagnostic).toEqual({ stage: "process_verification", reason: failure === "liveness" ? "process_not_running" : failure === "timeout" ? "probe_timeout" : "probe_transport_failed" });
+    expect(diagnostic).toEqual(failure === "liveness"
+      ? { stage: "process_verification", reason: "process_not_running", runnerLifecycle: "running", runnerSettled: false }
+      : { stage: "process_verification", reason: failure === "timeout" ? "probe_timeout" : "probe_transport_failed" });
     expect(JSON.stringify(diagnostic)).not.toContain("private");
     if (failure === "transport") {
       expect(error).toBe(transportError);
       expect(nativeSessionFailureSourceCode(error)).toBe("provider_transport_failed");
     }
   });
+  it.each([
+    [{ lifecycle: "suspended", outbox: [], pendingTerminalDelivery: null, pendingProviderCleanup: null }, "suspended", true],
+    [{ lifecycle: "turn_active", outbox: [{ id: 1 }], pendingTerminalDelivery: null }, "turn_active", false],
+    [{ lifecycle: "private free text", outbox: [] }, "other", true],
+    [{ lifecycle: undefined }, "unavailable", false],
+  ] as const)("records how an exited runner last left its durable state (%j)", async (state, runnerLifecycle, runnerSettled) => {
+    const { target, execute } = fixture(state);
+    const original = execute.getMockImplementation()!;
+    execute.mockImplementation(async request => request.args[2] === "paperclip-runner-recovery-check"
+      ? { stdout: "", stderr: "private probe output", exitCode: 45, timedOut: false }
+      : original(request));
+    const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" })
+      .catch(caught => caught);
+    const diagnostic = readRemoteRunnerRecoveryError(error);
+    expect(diagnostic).toEqual({ stage: "process_verification", reason: "process_not_running", runnerLifecycle, runnerSettled });
+    expect(JSON.stringify(diagnostic)).not.toContain("private");
+  });
+  const settledSuspension = { lifecycle: "suspended", outbox: [], pendingTerminalDelivery: null, pendingProviderCleanup: null };
+  const exitedRunner = (state: Record<string, unknown>, afterExit = state) => {
+    const { target, execute } = fixture(state);
+    const original = execute.getMockImplementation()!;
+    let exited = false;
+    execute.mockImplementation(async request => {
+      if (request.args[2] === "paperclip-runner-recovery-check") {
+        exited = true;
+        return { stdout: "", stderr: "", exitCode: 45, timedOut: false };
+      }
+      if (exited && request.args[1].includes("base64")) {
+        return { stdout: Buffer.from(JSON.stringify({ ...identity, ...afterExit })).toString("base64"),
+          stderr: "", exitCode: 0, timedOut: false };
+      }
+      return original(request);
+    });
+    return target;
+  };
+  it.each([2, 3])("resumes scheduled attempt %i whose runner exited after its previous attempt suspended it", async providerAttempt => {
+    await expect(verifyRemoteRunnerOrSettledExit({ claim, target: exitedRunner(settledSuspension) as never, identity,
+      runId: "run", normalizedSessionId: "session", providerAttempt })).resolves.toBeUndefined();
+  });
+  it.each([
+    ["an unsettled suspension", { ...settledSuspension, outbox: [{ id: 1 }] }, 2],
+    ["a pending provider cleanup", { ...settledSuspension, pendingProviderCleanup: { pid: 7 } }, 2],
+    ["a pending terminal delivery", { ...settledSuspension, pendingTerminalDelivery: { eventId: "event" } }, 2],
+    ["a pending warm transition", { ...settledSuspension, warmTransition: { transitionId: "t" } }, 2],
+    ["a runner that stopped mid-turn", { ...settledSuspension, lifecycle: "turn_active" }, 2],
+    ["an exhausted attempt budget", settledSuspension, 4],
+  ] as const)("keeps reattachment fail-closed after %s", async (_case, state, providerAttempt) => {
+    const error = await verifyRemoteRunnerOrSettledExit({ claim, target: exitedRunner(state) as never, identity,
+      runId: "run", normalizedSessionId: "session", providerAttempt }).catch(caught => caught);
+    expect(readRemoteRunnerRecoveryError(error)).toMatchObject({ stage: "process_verification", reason: "process_not_running" });
+  });
+  it.each([
+    ["an unsettled outbox", { ...settledSuspension, outbox: [{ id: 1 }] }],
+    ["a different runner", { ...settledSuspension, runnerInstanceId: "different" }],
+    ["an active turn", { ...settledSuspension, lifecycle: "turn_active" }],
+    ["a pending terminal delivery", { ...settledSuspension, pendingTerminalDelivery: { eventId: "event" } }],
+    ["a pending provider cleanup", { ...settledSuspension, pendingProviderCleanup: { pid: 7 } }],
+    ["a pending warm transition", { ...settledSuspension, warmTransition: { transitionId: "t" } }],
+  ] as const)("keeps reattachment fail-closed when the state read after the exit shows %s", async (_case, afterExit) => {
+    const error = await verifyRemoteRunnerOrSettledExit({ claim, target: exitedRunner(settledSuspension, afterExit) as never,
+      identity, runId: "run", normalizedSessionId: "session", providerAttempt: 2 }).catch(caught => caught);
+    expect(readRemoteRunnerRecoveryError(error)).toMatchObject({ stage: "process_verification", reason: "process_not_running" });
+  });
+  it.each(["timeout", "command", "json", "transport", "shared-deadline"] as const)("labels a post-exit state read %s failure within the recovery deadline", async failure => {
+    vi.useFakeTimers();
+    try {
+      const { target, execute } = fixture(settledSuspension);
+      const original = execute.getMockImplementation()!;
+      const postExitTimeouts: number[] = [];
+      let exited = false;
+      execute.mockImplementation(async request => {
+        if (request.args[2] === "paperclip-runner-recovery-check") {
+          exited = true;
+          return { stdout: "", stderr: "", exitCode: 45, timedOut: false };
+        }
+        if (request.args[1].includes("base64")) {
+          if (!exited && failure === "shared-deadline") await new Promise(resolveWait => setTimeout(resolveWait, 25_000));
+          if (exited) {
+            postExitTimeouts.push(Number(Reflect.get(request, "timeoutMs")));
+            if (failure === "transport") throw new Error("private sandbox transport detail");
+            if (failure === "json") return { stdout: Buffer.from("not json").toString("base64"), stderr: "", exitCode: 0, timedOut: false };
+            return { stdout: "", stderr: "private command output", exitCode: failure === "command" ? 1 : 0,
+              timedOut: failure !== "command" };
+          }
+        }
+        return original(request);
+      });
+      const startedAt = Date.now();
+      const outcome = verifyRemoteRunnerOrSettledExit({ claim, target: target as never, identity,
+        runId: "run", normalizedSessionId: "session", providerAttempt: 2 }).catch(error => error);
+      await vi.advanceTimersByTimeAsync(30_000);
+      const diagnostic = readRemoteRunnerRecoveryError(await outcome);
+      expect(diagnostic).toEqual({ stage: "runner_state", reason: failure === "command" ? "state_read_failed"
+        : failure === "json" ? "state_invalid" : failure === "transport" ? "probe_transport_failed" : "probe_timeout" });
+      expect(JSON.stringify(diagnostic)).not.toContain("private");
+      expect(Date.now() - startedAt).toBeLessThanOrEqual(30_000);
+      expect(postExitTimeouts.length).toBeGreaterThan(0);
+      if (failure === "shared-deadline") expect(postExitTimeouts.every(timeout => timeout > 0 && timeout <= 5_000)).toBe(true);
+    } finally { vi.useRealTimers(); }
+  });
+  it.each([
+    ["launches from", settledSuspension],
+    ["stays fail-closed for", { ...settledSuspension, lifecycle: "turn_active" }],
+  ] as const)("%s an exited runner when a scheduled retry builds its backend", async (outcome, runnerState) => {
+    const stateBase = await mkdtemp(join(tmpdir(), "paperclip-remote-settled-exit-"));
+    const previous = process.env.PAPERCLIP_RUNNER_STATE_DIR;
+    process.env.PAPERCLIP_RUNNER_STATE_DIR = stateBase;
+    const current = {
+      ...execution,
+      binding: { ...execution.binding, runId: "run" },
+      session: { ...execution.session, normalizedSessionId: "session" },
+    };
+    try {
+      state.createBackend.mockClear();
+      state.createTransport.mockClear();
+      await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: "runner",
+        runnerExecutionTarget: fixture().target as never });
+      state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
+      const root = state.createTransport.mock.calls.at(-1)![0].stateDirectory!;
+      await mkdir(join(root, "control-plane"), { recursive: true });
+      await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity)));
+      state.createTransport.mockClear();
+      const retry = createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: "runner",
+        runnerExecutionTarget: exitedRunner(runnerState) as never, restartRecovery: { ...claim, providerAttempt: 2 } });
+      if (outcome === "stays fail-closed for") {
+        expect(readRemoteRunnerRecoveryError(await retry.catch(caught => caught)))
+          .toMatchObject({ stage: "process_verification", reason: "process_not_running", runnerLifecycle: "turn_active" });
+        expect(state.createTransport).not.toHaveBeenCalled();
+        return;
+      }
+      await retry;
+      state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!();
+      expect(state.createTransport.mock.calls.at(-1)![0].adoptExistingRunner).toBeUndefined();
+    } finally {
+      if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR;
+      else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous;
+      await rm(stateBase, { recursive: true, force: true });
+    }
+  });
   it("rejects a stale process marker", async () => {
     const { execute, target } = fixture();
     execute

server/src/services/native-runtime/native-session-executor.ts

@@ -255,7 +255,8 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js";
 import { reportRunFailure } from "../run-failure-report.js";
 import {
   REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT,
-  RemoteRunnerRecoveryError, attachRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError, remoteRecoveryProbeReason,
+  RemoteRunnerRecoveryError, attachRemoteRunnerRecoveryDiagnostic, describeExitedRemoteRunner, readRemoteRunnerRecoveryError,
+  remoteRecoveryProbeReason,
 } from "./remote-runner-recovery-diagnostics.js";
 import { readRemoteRunnerRecoveryProbe, REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS } from "./remote-runner-recovery-probe.js";
 import { describeOwnFailedAttachment, inspectRemoteWarmSessionRecovery, isSettledRemoteWarmControlState, summarizePostTerminalEvents, type RemoteWarmSessionRecovery } from "./remote-warm-session-recovery.js";
@@ -308,6 +309,7 @@ import {
   type NativeLocalRunnerStopProof,
   isCurrentNativeControllerProcess,
   nextNativeProviderAttempt,
+  NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT,
   type NativeControllerIdentity,
   type NativeRestartRecoveryClaim,
 } from "./native-restart-recovery.js";
@@ -5721,9 +5723,10 @@ async function migrateRunnerdStateRootForExecution(input: {
         await verifyPluginRunnerTaskReattachment(target.runnerTask, identity);
         return;
       }
-      await verifyRemoteRunnerReattachment({
+      await verifyRemoteRunnerOrSettledExit({
         claim: input.restartRecovery, target: input.runnerExecutionTarget, identity,
         runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution),
+        providerAttempt: input.restartRecovery.providerAttempt,
       });
       return;
     }
@@ -13291,6 +13294,20 @@ export function parseRemoteRunnerProcessIdentity(
   return { pid, startedAt, ...(processStartFingerprint ? { processStartFingerprint } : {}) };
 }
 
+const REMOTE_RUNNER_IDENTITY_FIELDS = [
+  "runId",
+  "normalizedSessionId",
+  "runnerInstanceId",
+  "environmentLeaseId",
+  "turnId",
+  "itemId",
+] as const;
+
+function remoteRunnerStateDirectory(remoteCwd: string, normalizedSessionId: string) {
+  return posix.join(remoteCwd, ".paperclip-runtime", "paperclip-runner", "sessions",
+    createHash("sha256").update(normalizedSessionId).digest("hex"), "runner");
+}
+
 /** Verify remote ownership before exposing the transport's authenticated adoption path. */
 export async function verifyRemoteRunnerReattachment(input: {
   claim: Pick<Extract<
@@ -13302,6 +13319,7 @@ export async function verifyRemoteRunnerReattachment(input: {
   runId: string;
   normalizedSessionId: string;
   requireIdle?: boolean;
+  recoveryDeadline?: number;
 }) {
   const { claim, target, identity } = input;
   if (
@@ -13320,28 +13338,15 @@ export async function verifyRemoteRunnerReattachment(input: {
     throw new Error("native_remote_recovery_lease_mismatch");
   }
   const runner = target.runner;
-  const probeDeadline = Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS;
-  const stateDirectory = posix.join(
-    target.remoteCwd,
-    ".paperclip-runtime",
-    "paperclip-runner",
-    "sessions",
-    createHash("sha256").update(input.normalizedSessionId).digest("hex"),
-    "runner",
-  );
+  const probeDeadline = Math.min(input.recoveryDeadline ?? Number.POSITIVE_INFINITY,
+    Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS);
+  const stateDirectory = remoteRunnerStateDirectory(target.remoteCwd, input.normalizedSessionId);
   const runnerState = await readRemoteRunnerState({ runner, stateDirectory, recoveryDeadline: probeDeadline });
   if (input.requireIdle && (runnerState.lifecycle !== "ready" ||
       !Array.isArray(runnerState.outbox) || runnerState.outbox.length !== 0 || runnerState.pendingTerminalDelivery !== null)) {
     throw new Error("runner_state_identity_mismatch: remote_warm_session_not_idle");
   }
-  for (const field of [
-    "runId",
-    "normalizedSessionId",
-    "runnerInstanceId",
-    "environmentLeaseId",
-    "turnId",
-    "itemId",
-  ]) {
+  for (const field of REMOTE_RUNNER_IDENTITY_FIELDS) {
     if (
       typeof identity[field] !== "string" ||
       !identity[field] ||
@@ -13408,7 +13413,10 @@ export async function verifyRemoteRunnerReattachment(input: {
     return { alive, reason };
   };
   const verified = await check(undefined, probeDeadline);
-  if (!verified.alive) throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: verified.reason });
+  if (!verified.alive) {
+    throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: verified.reason,
+      ...(verified.reason === "process_not_running" ? describeExitedRemoteRunner(runnerState) : {}) });
+  }
   // These checks authorize an attempt to reconnect, not the task itself. The
   // transport still requires authentication against the exact durable PRP key.
   return {
@@ -13419,6 +13427,28 @@ export async function verifyRemoteRunnerReattachment(input: {
   };
 }
 
+export async function verifyRemoteRunnerOrSettledExit(
+  input: Parameters<typeof verifyRemoteRunnerReattachment>[0] & { providerAttempt: number },
+) {
+  const recoveryDeadline = Math.min(input.recoveryDeadline ?? Number.POSITIVE_INFINITY,
+    Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS);
+  try {
+    return await verifyRemoteRunnerReattachment({ ...input, recoveryDeadline });
+  } catch (error) {
+    const diagnostic = readRemoteRunnerRecoveryError(error);
+    const target = input.target;
+    if (input.providerAttempt > NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT || diagnostic?.stage !== "process_verification" ||
+        diagnostic.reason !== "process_not_running" ||
+        diagnostic.runnerLifecycle !== "suspended" || diagnostic.runnerSettled !== true ||
+        target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner) throw error;
+    const state = await readRemoteRunnerState({ runner: target.runner,
+      stateDirectory: remoteRunnerStateDirectory(target.remoteCwd, input.normalizedSessionId), recoveryDeadline });
+    if (REMOTE_RUNNER_IDENTITY_FIELDS.some(field => state[field] !== input.identity[field]) ||
+        state.lifecycle !== "suspended" || !describeExitedRemoteRunner(state).runnerSettled) throw error;
+    return undefined;
+  }
+}
+
 /** A crash before source bootstrap leaves a suspended Runner; a crash during
  * its drain leaves the same live source. Neither permits a replacement until
  * the physical process and all six persisted identity fields are proven. */
@@ -13433,8 +13463,7 @@ async function verifyStagedRemoteRunner(input: {
     throw new Error("native_runner_staged_attachment_profile_unsupported");
   }
   const sourceIdentity = input.recovery.sourceIdentity;
-  const stateDirectory = posix.join(target.remoteCwd, ".paperclip-runtime", "paperclip-runner", "sessions",
-    createHash("sha256").update(sourceIdentity.normalizedSessionId).digest("hex"), "runner");
+  const stateDirectory = remoteRunnerStateDirectory(target.remoteCwd, sourceIdentity.normalizedSessionId);
   const read = () => readRemoteRunnerState({ runner: target.runner!, stateDirectory });
   const state = await read();
   const control = record(JSON.parse(readBoundedNativeFile(resolve(scopedRunnerdStateRoot(input.execution),
@@ -14057,8 +14086,9 @@ async function prepareRemoteExternalRunner({ input, target, runner, root, identi
   const processIdentityReady = new Promise<void>((resolve, reject) => { identityReady = resolve; identityFailed = reject; });
   void processIdentityReady.catch(() => {});
   const adoptExistingRunner = input.restartRecovery?.kind === "reattach_remote_runner"
-    ? await verifyRemoteRunnerReattachment({ claim: input.restartRecovery, target, identity: { ...identity },
-        runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution) }) : undefined;
+    ? await verifyRemoteRunnerOrSettledExit({ claim: input.restartRecovery, target, identity: { ...identity },
+        runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution),
+        providerAttempt: input.restartRecovery.providerAttempt }) : undefined;
   if (adoptExistingRunner) { spawned(); identityReady(); }
   const launch = createRemoteRunnerProcessLauncher({ target, runner, remoteBinary,
     processIdentityPath: posix.join(runnerStateDirectory, "runner-process.identity"), stateDirectory: runnerStateDirectory,
@@ -15828,12 +15858,13 @@ async function createRunnerdBackendWithinSessionClaim(
           });
         })()
       : input.restartRecovery?.kind === "reattach_remote_runner"
-      ? await verifyRemoteRunnerReattachment({
+      ? await verifyRemoteRunnerOrSettledExit({
           claim: input.restartRecovery,
           target,
           identity: durableIdentity ?? {},
           runId: input.execution.binding.runId,
           normalizedSessionId: nativeSessionKey(input.execution),
+          providerAttempt: input.restartRecovery.providerAttempt,
         })
       : target.kind === "local" && localRecoveryProcess
         ? {

server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts

@@ -9,9 +9,29 @@ const reasons = [
   "state_read_failed", "state_invalid",
 ] as const;
 
+const runnerLifecycles = [
+  "connecting", "ready", "backpressure", "reconnecting", "session_opening", "session_open",
+  "turn_starting", "turn_active", "running", "suspended", "recoverable_failure", "reconciliation_required",
+  "provider_exited", "unrecoverable", "fenced", "failed", "completed", "closed",
+  "other", "unavailable",
+] as const;
+
 export interface RemoteRunnerRecoveryDiagnostic {
   stage: typeof stages[number];
   reason: typeof reasons[number];
+  runnerLifecycle?: typeof runnerLifecycles[number];
+  runnerSettled?: boolean;
+}
+
+export function describeExitedRemoteRunner(state: Record<string, unknown>): Pick<RemoteRunnerRecoveryDiagnostic,
+  "runnerLifecycle" | "runnerSettled"> {
+  const lifecycle = state.lifecycle;
+  return {
+    runnerLifecycle: typeof lifecycle !== "string" ? "unavailable"
+      : runnerLifecycles.find(candidate => candidate === lifecycle && candidate !== "other" && candidate !== "unavailable") ?? "other",
+    runnerSettled: Array.isArray(state.outbox) && state.outbox.length === 0 &&
+      state.pendingTerminalDelivery == null && state.pendingProviderCleanup == null && state.warmTransition == null,
+  };
 }
 
 const transportDiagnostics = new WeakMap<object, RemoteRunnerRecoveryDiagnostic>();
@@ -35,9 +55,13 @@ export class RemoteRunnerRecoveryError extends Error {
 export function readRemoteRunnerRecoveryDiagnostic(value: unknown): RemoteRunnerRecoveryDiagnostic | null {
   if (!value || typeof value !== "object") return null;
   try {
-    const { stage, reason } = value as Record<string, unknown>;
+    const { stage, reason, runnerLifecycle, runnerSettled } = value as Record<string, unknown>;
     if (!stages.some(candidate => candidate === stage) || !reasons.some(candidate => candidate === reason)) return null;
-    return { stage, reason } as RemoteRunnerRecoveryDiagnostic;
+    return {
+      stage, reason,
+      ...(runnerLifecycles.some(candidate => candidate === runnerLifecycle) ? { runnerLifecycle } : {}),
+      ...(typeof runnerSettled === "boolean" ? { runnerSettled } : {}),
+    } as RemoteRunnerRecoveryDiagnostic;
   } catch {
     return null;
   }

server/src/services/run-failure-diagnostics.ts

@@ -220,6 +220,8 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
   if (remoteRunnerRecovery) {
     execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage;
     execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;
+    if (remoteRunnerRecovery.runnerLifecycle) execution.remoteRunnerLifecycle = remoteRunnerRecovery.runnerLifecycle;
+    if (remoteRunnerRecovery.runnerSettled !== undefined) execution.remoteRunnerSettled = remoteRunnerRecovery.runnerSettled;
   }
   const configuration = read(result, "configurationIncomplete");
   if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") {