fix(native-runtime): retain remote recovery failure reasons (#16341)

Preserve closed marker-read and process-verification reasons through native recovery failures and operator reports. Keep existing transport errors and reattachment ownership checks.

Add shell, process-probe, persistence and privacy regression coverage. All current-head CI checks pass and Greptile is 5/5.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/observability.md

@@ -792,6 +792,16 @@ only for these labels; its text and stack are not copied, because they bypass th
 executor's identity redaction. These observations do not change close, quarantine
 or recovery behavior.
 
+Remote runner reattachment failures also record `remoteRunnerRecoveryStage`
+and `remoteRunnerRecoveryReason` from the saved run. These fields contain
+closed labels for marker reads and process verification. They distinguish
+missing or unsafe markers, invalid or missing fingerprints, changed or
+unavailable process identity, and failed or timed-out probes. Paths, PIDs,
+nonces, command lines and transport output are excluded. The existing
+`runner_remote_process_identity_unavailable` error message and native failure
+classification remain unchanged. The coordinator also saves these labels in
+its failure detail and recovery evidence. A later failure clears stale labels.
+
 ACP turns record `acpLastEventAgeMs`, `acpObservedEventCount`,
 `acpPendingToolCount`, and `acpToolInventoryComplete` at finalization, before
 usage reads, error logging, and cleanup. The age measures time since the last

server/src/services/__tests__/run-failure-diagnostics.test.ts

@@ -190,6 +190,25 @@ describe("run failure diagnostics", () => {
     }
   });
 
+  it("exports only the saved remote recovery stage and reason", () => {
+    const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({
+      errorCode: "runner_remote_process_identity_unavailable", resultJson: {
+        remoteRunnerRecovery: { stage: "identity_marker", reason: "fingerprint_missing",
+          path: "private-path", pid: 123, nonce: "private-nonce", stderr: "private-stderr" },
+      },
+    }), {}));
+    expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "fingerprint_missing" });
+    expect(result.exceptions).toEqual([]);
+    expect(JSON.stringify(result)).not.toContain("private");
+    for (const value of [null, { stage: "private-stage", reason: "marker_missing" },
+      { stage: "identity_marker", reason: "private-reason" },
+      Object.defineProperty({}, "stage", { get() { throw new Error("private"); } })]) {
+      expect(collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable",
+        resultJson: { remoteRunnerRecovery: value } }), {}).execution).toEqual({});
+    }
+    expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({});
+  });
+
   it("exports which native close barrier failed without tool identities or provider text", () => {
     const closeFailure = (settlement: unknown) => Object.assign(
       new Error("provider_transport_failed: runner did not durably suspend before checkpoint"),

server/src/services/native-runtime/native-session-executor.test.ts

@@ -52,6 +52,7 @@ import {
 import { createHash, generateKeyPairSync, randomUUID } from "node:crypto";
 import { DatabaseSync } from "node:sqlite";
 import { nativeSha256 } from "./canonical.js";
+import { RemoteRunnerRecoveryError, readRemoteRunnerRecoveryError } from "./remote-runner-recovery-diagnostics.js";
 import { agentDirectoryWorkingCopyService } from "../agent-directory-working-copies.js";
 import { probeAgentDirectory } from "../agent-directory-probe.js";
 import { AGENT_FILES_CONTRACT } from "../agent-file-store.js";
@@ -592,9 +593,40 @@ describe("remote controller restart adoption", () => {
       ? { stdout: "nonce\n123\n2026-09-19T10:00:00.000Z\nrunner\n", stderr: "", exitCode: 0, timedOut: false }
       : original(request));
     await expect(verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }))
-      .rejects.toThrow("runner_remote_process_identity_unavailable");
+      .rejects.toMatchObject({
+        message: "runner_remote_process_identity_unavailable",
+        remoteRunnerRecovery: { stage: "identity_marker", reason: "fingerprint_missing" },
+      });
   });
-  it.each(["start ticks", "boot identity"])("rejects PID reuse after the live %s changes", async (change) => {
+  it.each([
+    [31, "marker_missing"], [32, "marker_unsafe"], [33, "marker_read_failed"],
+  ] as const)("records the closed reason for marker read exit %s", async (exitCode, reason) => {
+    const { target, execute } = fixture();
+    const original = execute.getMockImplementation()!;
+    execute.mockImplementation(async request => request.args[2] === "paperclip-runner-recovery-identity"
+      ? { stdout: "", stderr: "private transport text", exitCode, timedOut: false }
+      : original(request));
+    await expect(verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }))
+      .rejects.toMatchObject({ message: "runner_remote_process_identity_unavailable",
+        remoteRunnerRecovery: { stage: "identity_marker", reason } });
+  });
+  it.each(["timeout", "transport"] as const)("records a marker %s without retaining transport text", async failure => {
+    const { target, execute } = fixture();
+    const transportError = new Error("private bridge response");
+    const original = execute.getMockImplementation()!;
+    execute.mockImplementation(async request => {
+      if (request.args[2] !== "paperclip-runner-recovery-identity") return original(request);
+      if (failure === "transport") throw transportError;
+      return { stdout: "", stderr: "private bridge response", exitCode: -1, timedOut: true };
+    });
+    const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error);
+    const diagnostic = readRemoteRunnerRecoveryError(error);
+    expect(diagnostic).toEqual({ stage: "identity_marker", reason: failure === "timeout" ? "probe_timeout" : "probe_transport_failed" });
+    expect(JSON.stringify(diagnostic)).not.toContain("private");
+    if (failure === "transport") expect(error).toBe(transportError);
+    else expect(error).toMatchObject({ message: "runner_remote_process_identity_unavailable" });
+  });
+  it.each(["start ticks", "boot identity", "fingerprint probe"])("rejects a changed or unavailable live %s", async (change) => {
     const root = await mkdtemp(join(tmpdir(), "remote-generation-"));
     const proc = join(root, "proc");
     const markerPath = join(root, "runner-process.identity");
@@ -621,18 +653,39 @@ describe("remote controller restart adoption", () => {
         args[1] = args[1].replaceAll("/proc/", `${proc}/`);
         args[3] = markerPath;
         let exitCode = 0;
-        try { execFileSync("sh", args, { stdio: "pipe" }); } catch { exitCode = 4; }
+        try { execFileSync("sh", args, { stdio: "pipe" }); } catch (error) { exitCode = (error as { status?: number }).status ?? 4; }
         return { stdout: "", stderr: "", exitCode, timedOut: false };
       });
       const adopted = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" });
       expect(await adopted.isAlive()).toBe(true);
       if (change === "start ticks") await writeFile(join(proc, String(pid), "stat"), stat(101));
-      else await writeFile(join(proc, "sys/kernel/random/boot_id"), "abcd-5678\n");
+      else if (change === "boot identity") await writeFile(join(proc, "sys/kernel/random/boot_id"), "abcd-5678\n");
+      else await rm(join(proc, String(pid), "stat"));
       expect(await adopted.isAlive()).toBe(false);
       await expect(verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }))
-        .rejects.toThrow("runner_remote_process_identity_unavailable");
+        .rejects.toMatchObject({ message: "runner_remote_process_identity_unavailable", remoteRunnerRecovery: {
+          stage: "process_verification", reason: change === "fingerprint probe" ? "fingerprint_unavailable" : "process_generation_changed",
+        } });
     } finally { await rm(root, { recursive: true, force: true }); }
   });
+  it.each(["liveness", "timeout", "transport"] as const)("retains the reason for a failed process %s probe", async failure => {
+    const { target, execute } = fixture();
+    const original = execute.getMockImplementation()!;
+    const transportError = new Error("provider_transport_failed: private bridge response");
+    execute.mockImplementation(async request => {
+      if (request.args[2] !== "paperclip-runner-recovery-check") return original(request);
+      if (failure === "transport") throw transportError;
+      return { stdout: "", stderr: "private probe output", exitCode: failure === "liveness" ? 45 : -1, timedOut: failure === "timeout" };
+    });
+    const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error);
+    const diagnostic = readRemoteRunnerRecoveryError(error);
+    expect(diagnostic).toEqual({ stage: "process_verification", reason: failure === "liveness" ? "process_not_running" : failure === "timeout" ? "probe_timeout" : "probe_transport_failed" });
+    expect(JSON.stringify(diagnostic)).not.toContain("private");
+    if (failure === "transport") {
+      expect(error).toBe(transportError);
+      expect(nativeSessionFailureSourceCode(error)).toBe("provider_transport_failed");
+    }
+  });
   it("rejects a stale process marker", async () => {
     const { execute, target } = fixture();
     execute
@@ -8685,6 +8738,26 @@ describe("native warm session supervision", () => {
 });
 
 describe("native session bounded recovery", () => {
+  it("persists closed remote recovery evidence without replacing saved run results", async () => {
+    const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
+    const remoteRunnerRecovery = { stage: "identity_marker", reason: "fingerprint_missing" } as const;
+    const failure = new RemoteRunnerRecoveryError(remoteRunnerRecovery);
+    state.execute.mockReset().mockRejectedValueOnce(failure);
+    state.upsertRecoveryAction.mockReset().mockResolvedValue({});
+    const service = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ update: vi.fn(async () => null) } as unknown as ReturnType<typeof issueServiceModule.issueService>);
+    try {
+    await expect(executePaperclipNativeSession({ db: leaseDb(execution, {}, {}, updates), execution, runnerInstanceId: "runner" })).rejects.toBe(failure);
+    expect(updates.find(update => update.table === nativeRunFinalizations && update.values.failureDetail)?.values.failureDetail)
+      .toMatchObject({ remoteRunnerRecovery });
+    const resultJson = updates.find(update => update.table === heartbeatRuns && update.values.error)?.values.resultJson as SQL;
+    const query = new PgDialect().sqlToQuery(resultJson);
+    expect(query.sql).toContain('coalesce("heartbeat_runs"."result_json",');
+    expect(query.sql).toContain("- 'remoteRunnerRecovery'");
+    expect(query.params).toContain(JSON.stringify({ remoteRunnerRecovery }));
+    expect(state.upsertRecoveryAction).toHaveBeenCalledWith(expect.objectContaining({ evidence: expect.objectContaining({ remoteRunnerRecovery }) }));
+    expect(state.execute).toHaveBeenCalledOnce();
+    } finally { service.mockRestore(); }
+  });
   it("blocks a denied Cursor task and gives recovery to the operator without another provider attempt", async () => {
     const updates: Array<{ table: unknown; values: Record<string, unknown> }> = [];
     const failure = new NativePermissionDeclinedError();

server/src/services/native-runtime/native-session-executor.ts

@@ -155,6 +155,10 @@ import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarge
 import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js";
 import { issueRecoveryActionService } from "../issue-recovery-actions.js";
 import { reportRunFailure } from "../run-failure-report.js";
+import {
+  REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT,
+  RemoteRunnerRecoveryError, attachRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError, remoteRecoveryProbeReason,
+} from "./remote-runner-recovery-diagnostics.js";
 import { persistActivity, publishActivity } from "../activity-log.js";
 import { commitNativeStatusDecision } from "./status-decision-committer.js";
 import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
@@ -9158,6 +9162,7 @@ async function executePaperclipNativeSessionWithinScope(
         error instanceof Error ? error.message : String(error),
       ).slice(0, 2_000);
       const sanitizedStderrTail = redactIdentityText(message).slice(-4_096);
+      const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error);
       // Set inside the transaction only when the write below genuinely
       // transitions the run into "failed". Read after the transaction
       // commits, so a rolled-back write never reports a false failure.
@@ -9190,6 +9195,7 @@ async function executePaperclipNativeSessionWithinScope(
             failureCode,
             failureDetail: identityRedactor.redact({
               message,
+              ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}),
               originalFailureCode:
                 error instanceof NativeProviderTerminalFailure
                   ? error.providerCode
@@ -9314,6 +9320,7 @@ async function executePaperclipNativeSessionWithinScope(
             nativePhase: phase,
             nativePhaseUpdatedAt: now,
             error: message,
+            resultJson: sql`(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) - 'remoteRunnerRecovery') || ${JSON.stringify(remoteRunnerRecovery ? { remoteRunnerRecovery } : {})}::jsonb`,
             errorCode: ownershipUnverified
               ? NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE
               : sourceFailureCode,
@@ -9376,6 +9383,7 @@ async function executePaperclipNativeSessionWithinScope(
             } } : {}),
             coordinatorAttempt: attempt,
             sourceFailureCode,
+            ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}),
             recoveryDisposition: failureCode,
             recoveryMode: recoveryEvidence.recoveryMode,
             providerSessionEstablished:
@@ -10726,13 +10734,13 @@ export async function verifyRemoteRunnerReattachment(input: {
     command: "sh",
     args: [
       "-c",
-      'test -f "$1" && test ! -L "$1" && cat -- "$1"',
+      REMOTE_RECOVERY_MARKER_READ_SCRIPT,
       "paperclip-runner-recovery-identity",
       identityPath,
     ],
     bypassSession: true,
     timeoutMs: 10_000,
-  });
+  }).catch(error => { throw attachRemoteRunnerRecoveryDiagnostic(error, { stage: "identity_marker", reason: "probe_transport_failed" }); });
   const nonce = marker.stdout.split("\n")[0] ?? "";
   const runnerInstanceId = identity.runnerInstanceId as string;
   const processIdentity =
@@ -10742,15 +10750,19 @@ export async function verifyRemoteRunnerReattachment(input: {
           runnerInstanceId,
         })
       : null;
-  if (!processIdentity?.processStartFingerprint)
-    throw new Error("runner_remote_process_identity_unavailable");
+  if (marker.exitCode !== 0 || marker.timedOut) {
+    throw new RemoteRunnerRecoveryError({ stage: "identity_marker",
+      reason: remoteRecoveryProbeReason("identity_marker", marker.exitCode, marker.timedOut) });
+  }
+  if (!processIdentity) throw new RemoteRunnerRecoveryError({ stage: "identity_marker", reason: "marker_invalid" });
+  if (!processIdentity.processStartFingerprint) throw new RemoteRunnerRecoveryError({ stage: "identity_marker", reason: "fingerprint_missing" });
   const expectedStartFingerprint = processIdentity.processStartFingerprint;
   const check = async (signal?: NodeJS.Signals) => {
     const result = await runner.execute({
       command: "sh",
       args: [
         "-c",
-        `${REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT}; ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; test "$process_fingerprint" = "$5" || exit 4` +
+        `${REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT}; if ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; then :; else exit 46; fi; test "$process_fingerprint" = "$5" || exit 47` +
           (signal ? '; kill -"$6" "$pid"' : ""),
         "paperclip-runner-recovery-check",
         identityPath,
@@ -10762,18 +10774,19 @@ export async function verifyRemoteRunnerReattachment(input: {
       ],
       bypassSession: true,
       timeoutMs: 10_000,
-    });
-    return result.exitCode === 0 && !result.timedOut;
+    }).catch(error => { throw attachRemoteRunnerRecoveryDiagnostic(error, { stage: "process_verification", reason: "probe_transport_failed" }); });
+    return { alive: result.exitCode === 0 && !result.timedOut,
+      reason: remoteRecoveryProbeReason("process_verification", result.exitCode, result.timedOut) };
   };
-  if (!(await check()))
-    throw new Error("runner_remote_process_identity_unavailable");
+  const verified = await check();
+  if (!verified.alive) throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: verified.reason });
   // These checks authorize an attempt to reconnect, not the task itself. The
   // transport still requires authentication against the exact durable PRP key.
   return {
     ...processIdentity,
     processGroupId: null,
-    isAlive: () => check(),
-    signal: check,
+    isAlive: async () => (await check()).alive,
+    signal: async (signal: NodeJS.Signals) => (await check(signal)).alive,
   };
 }
 

server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts

@@ -0,0 +1,70 @@
+import { execFileSync } from "node:child_process";
+import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+  REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT,
+  RemoteRunnerRecoveryError, readRemoteRunnerRecoveryError,
+} from "./remote-runner-recovery-diagnostics.js";
+
+function shellExit(script: string, args: string[]) {
+  try {
+    execFileSync("sh", ["-c", script, "recovery-test", ...args], { stdio: "pipe", timeout: 2_000 });
+    return 0;
+  } catch (error) {
+    return (error as { status: number }).status;
+  }
+}
+
+describe("remote recovery diagnostic shell receipts", () => {
+  it("distinguishes missing, unsafe and valid marker reads without exposing their contents", async () => {
+    const root = await mkdtemp(join(tmpdir(), "remote-recovery-read-"));
+    const marker = join(root, "marker");
+    try {
+      expect(shellExit(REMOTE_RECOVERY_MARKER_READ_SCRIPT, [marker])).toBe(31);
+      await writeFile(join(root, "target"), "private marker");
+      await symlink(join(root, "target"), marker);
+      expect(shellExit(REMOTE_RECOVERY_MARKER_READ_SCRIPT, [marker])).toBe(32);
+      await rm(marker);
+      await writeFile(marker, "private marker");
+      expect(shellExit(REMOTE_RECOVERY_MARKER_READ_SCRIPT, [marker])).toBe(0);
+    } finally { await rm(root, { recursive: true, force: true }); }
+  });
+
+  it("distinguishes a torn marker from a changed identity before signalling any process", async () => {
+    const root = await mkdtemp(join(tmpdir(), "remote-recovery-check-"));
+    const marker = join(root, "marker");
+    const args = [marker, "nonce", "runner", String(process.pid)];
+    try {
+      expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, args)).toBe(41);
+      await writeFile(marker, "nonce\n");
+      expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, args)).toBe(43);
+      await writeFile(marker, `other-nonce\n${process.pid}\n2026-10-14T00:00:00.000Z\nrunner\n`);
+      expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, args)).toBe(44);
+    } finally { await rm(root, { recursive: true, force: true }); }
+  });
+
+  it("reads only closed labels through bounded causes and contains hostile getters", () => {
+    const diagnostic = { stage: "process_verification", reason: "process_generation_changed" } as const;
+    const error = new RemoteRunnerRecoveryError(diagnostic);
+    expect(readRemoteRunnerRecoveryError(new Error("private outer text", { cause: error }))).toEqual(diagnostic);
+    expect(readRemoteRunnerRecoveryError(Object.assign(new Error("private"), {
+      code: error.code, remoteRunnerRecovery: { stage: "private", reason: "process_generation_changed" },
+    }))).toBeNull();
+    expect(readRemoteRunnerRecoveryError(Object.defineProperty({}, "code", { get() { throw new Error("private"); } }))).toBeNull();
+    const cyclic: { cause?: unknown } = {};
+    cyclic.cause = cyclic;
+    expect(readRemoteRunnerRecoveryError(cyclic)).toBeNull();
+  });
+
+  it("reports a retired owned child as unavailable before checking its command line", async () => {
+    const root = await mkdtemp(join(tmpdir(), "remote-recovery-retired-"));
+    const marker = join(root, "marker");
+    const pid = execFileSync("sh", ["-c", "printf '%s' $$"], { encoding: "utf8" }).trim();
+    try {
+      await writeFile(marker, `nonce\n${pid}\n2026-10-14T00:00:00.000Z\nrunner\n`);
+      expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, [marker, "nonce", "runner", pid])).toBe(45);
+    } finally { await rm(root, { recursive: true, force: true }); }
+  });
+});

server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts

@@ -0,0 +1,91 @@
+export const REMOTE_RUNNER_RECOVERY_ERROR_CODE = "runner_remote_process_identity_unavailable";
+
+const stages = ["identity_marker", "process_verification"] as const;
+const reasons = [
+  "marker_missing", "marker_unsafe", "marker_read_failed", "marker_invalid",
+  "fingerprint_missing", "process_not_running", "process_identity_changed",
+  "fingerprint_unavailable", "process_generation_changed", "probe_timeout",
+  "probe_transport_failed", "probe_failed",
+] as const;
+
+export interface RemoteRunnerRecoveryDiagnostic {
+  stage: typeof stages[number];
+  reason: typeof reasons[number];
+}
+
+const transportDiagnostics = new WeakMap<object, RemoteRunnerRecoveryDiagnostic>();
+
+export function attachRemoteRunnerRecoveryDiagnostic(error: unknown, diagnostic: RemoteRunnerRecoveryDiagnostic): unknown {
+  if (error && typeof error === "object") {
+    transportDiagnostics.set(error, diagnostic);
+    return error;
+  }
+  return new RemoteRunnerRecoveryError(diagnostic);
+}
+
+export class RemoteRunnerRecoveryError extends Error {
+  readonly code = REMOTE_RUNNER_RECOVERY_ERROR_CODE;
+
+  constructor(readonly remoteRunnerRecovery: RemoteRunnerRecoveryDiagnostic) {
+    super(REMOTE_RUNNER_RECOVERY_ERROR_CODE);
+  }
+}
+
+export function readRemoteRunnerRecoveryDiagnostic(value: unknown): RemoteRunnerRecoveryDiagnostic | null {
+  if (!value || typeof value !== "object") return null;
+  try {
+    const { stage, reason } = value as Record<string, unknown>;
+    if (!stages.some(candidate => candidate === stage) || !reasons.some(candidate => candidate === reason)) return null;
+    return { stage, reason } as RemoteRunnerRecoveryDiagnostic;
+  } catch {
+    return null;
+  }
+}
+
+export function readRemoteRunnerRecoveryError(error: unknown): RemoteRunnerRecoveryDiagnostic | null {
+  let candidate = error;
+  for (let depth = 0; depth < 4 && candidate && typeof candidate === "object"; depth++) {
+    const transportDiagnostic = transportDiagnostics.get(candidate);
+    if (transportDiagnostic) return transportDiagnostic;
+    try {
+      const record = candidate as Record<string, unknown>;
+      if (record.code === REMOTE_RUNNER_RECOVERY_ERROR_CODE) return readRemoteRunnerRecoveryDiagnostic(record.remoteRunnerRecovery);
+      candidate = record.cause;
+    } catch {
+      return null;
+    }
+  }
+  return null;
+}
+
+export const REMOTE_RECOVERY_MARKER_READ_SCRIPT =
+  'test ! -L "$1" || exit 32; test -f "$1" || exit 31; cat -- "$1" || exit 33';
+
+export const REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT = [
+  "set -eu",
+  "identity_path=$1; expected_nonce=$2; expected_runner_id=$3; expected_pid=$4",
+  'test ! -L "$identity_path" || exit 42',
+  'test -f "$identity_path" || exit 41',
+  '{ IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path" || exit 43',
+  'test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test "$pid" = "$expected_pid" && test -n "$started_at" || exit 44',
+  'kill -0 "$pid" 2>/dev/null || exit 45',
+  'if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 44; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 44; fi',
+].join("; ");
+
+export function remoteRecoveryProbeReason(stage: RemoteRunnerRecoveryDiagnostic["stage"], exitCode: number | null, timedOut: boolean): RemoteRunnerRecoveryDiagnostic["reason"] {
+  if (timedOut) return "probe_timeout";
+  if (stage === "identity_marker") {
+    if (exitCode === 31) return "marker_missing";
+    if (exitCode === 32) return "marker_unsafe";
+    if (exitCode === 33) return "marker_read_failed";
+  } else {
+    if (exitCode === 41) return "marker_missing";
+    if (exitCode === 42) return "marker_unsafe";
+    if (exitCode === 43) return "marker_read_failed";
+    if (exitCode === 44) return "process_identity_changed";
+    if (exitCode === 45) return "process_not_running";
+    if (exitCode === 46) return "fingerprint_unavailable";
+    if (exitCode === 47) return "process_generation_changed";
+  }
+  return "probe_failed";
+}

server/src/services/run-failure-diagnostics.ts

@@ -10,6 +10,7 @@ import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants
 import { redactCurrentUserText } from "../log-redaction.js";
 import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
 import { readNativeModelRejectionDiagnostic } from "./native-runtime/native-provider-failure.js";
+import { readRemoteRunnerRecoveryDiagnostic } from "./native-runtime/remote-runner-recovery-diagnostics.js";
 import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js";
 
 type Run = typeof heartbeatRuns.$inferSelect;
@@ -239,6 +240,11 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
     if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
   }
   const result = run.resultJson;
+  const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery"));
+  if (remoteRunnerRecovery) {
+    execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage;
+    execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;
+  }
   const configuration = read(result, "configurationIncomplete");
   if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") {
     const diagnostic = readWorkspaceBaseRefDiagnostic(read(configuration, "baseRefDiagnostic"));