fix(native-runtime): report remote recovery labels thrown before coordinator save (#16406)

Restart reattachment can fail in the executor before the native coordinator saves its result. The run failure report now reads the closed remote recovery stage and reason from the caught error when the saved result lacks them. The saved result keeps priority and the same closed-vocabulary validation applies.

Add a regression for thrown, wrapped-transport, saved-priority, and out-of-vocabulary cases. Full current-head CI and Greptile 5/5 passed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

doc/observability.md

@@ -793,7 +793,9 @@ executor's identity redaction. These observations do not change close, quarantin
 or recovery behavior.
 
 Remote runner reattachment failures also record `remoteRunnerRecoveryStage`
-and `remoteRunnerRecoveryReason` from the saved run. These fields contain
+and `remoteRunnerRecoveryReason` from the saved run. When reattachment fails
+before the coordinator saves its result, the report reads the same labels from
+the caught recovery error instead. These fields contain
 closed labels for marker reads and process verification. They distinguish
 missing or unsafe markers, invalid or missing fingerprints, changed or
 unavailable process identity, and failed or timed-out probes. Paths, PIDs,

server/src/services/__tests__/run-failure-diagnostics.test.ts

@@ -1,6 +1,7 @@
 import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js";
 import { describe, expect, it } from "vitest";
 import type { heartbeatRuns } from "@paperclipai/db";
+import { attachRemoteRunnerRecoveryDiagnostic, RemoteRunnerRecoveryError } from "../native-runtime/remote-runner-recovery-diagnostics.js";
 import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js";
 
 type Run = typeof heartbeatRuns.$inferSelect;
@@ -222,6 +223,28 @@ describe("run failure diagnostics", () => {
     expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({});
   });
 
+  it("exports the remote recovery stage and reason from an error thrown before the result was saved", () => {
+    const thrown = Object.assign(new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }),
+      { path: "private-path", pid: 123 });
+    const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown }));
+    expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" });
+    expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]);
+    expect(JSON.stringify(result)).not.toContain("private");
+
+    const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"),
+      { stage: "identity_marker", reason: "probe_transport_failed" });
+    expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: new Error("wrapped", { cause: transport }) }).execution)
+      .toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "probe_transport_failed" });
+
+    const saved = collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable",
+      resultJson: { remoteRunnerRecovery: { stage: "identity_marker", reason: "marker_missing" } } }), { error: thrown });
+    expect(saved.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "marker_missing" });
+    expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {
+      error: Object.assign(new Error("runner_remote_process_identity_unavailable"), { code: "runner_remote_process_identity_unavailable",
+        remoteRunnerRecovery: { stage: "private-stage", reason: "marker_missing" } }),
+    }).execution).toEqual({});
+  });
+
   it("exports which native close barrier failed without tool identities or provider text", () => {
     const closeFailure = (settlement: unknown) => Object.assign(
       new Error("provider_transport_failed: runner did not durably suspend before checkpoint"),

server/src/services/run-failure-diagnostics.ts

@@ -10,7 +10,7 @@ import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants
 import { redactCurrentUserText } from "../log-redaction.js";
 import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js";
 import { readNativeModelRejectionDiagnostic, readNativeTurnFailureDiagnostic } from "./native-runtime/native-provider-failure.js";
-import { readRemoteRunnerRecoveryDiagnostic } from "./native-runtime/remote-runner-recovery-diagnostics.js";
+import { readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError } from "./native-runtime/remote-runner-recovery-diagnostics.js";
 import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js";
 
 type Run = typeof heartbeatRuns.$inferSelect;
@@ -240,7 +240,11 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport
     if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs;
   }
   const result = run.resultJson;
-  const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery"));
+  // Restart reattachment can fail before the native coordinator saves its
+  // result. The thrown error then reaches the generic run failure path, so
+  // read the same closed labels from the error.
+  const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")) ??
+    readRemoteRunnerRecoveryError(options.error);
   if (remoteRunnerRecovery) {
     execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage;
     execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;