From 11d69d04c7531cf7eb37f7994b7112797e145837 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 15 Oct 2026 02:30:58 -0700 Subject: [PATCH] fix(native-runtime): report remote recovery labels thrown before coordinator save (#16406) Restart reattachment can fail in the executor before the native coordinator saves its result. The run failure report now reads the closed remote recovery stage and reason from the caught error when the saved result lacks them. The saved result keeps priority and the same closed-vocabulary validation applies. Add a regression for thrown, wrapped-transport, saved-priority, and out-of-vocabulary cases. Full current-head CI and Greptile 5/5 passed. Co-Authored-By: Claude Opus 5.5 --- doc/observability.md | 4 +++- .../__tests__/run-failure-diagnostics.test.ts | 23 +++++++++++++++++++ .../src/services/run-failure-diagnostics.ts | 8 +++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/doc/observability.md b/doc/observability.md index 67fd9b4a4a0..78ae54df3a6 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -793,7 +793,9 @@ executor's identity redaction. These observations do not change close, quarantin or recovery behavior. Remote runner reattachment failures also record `remoteRunnerRecoveryStage` -and `remoteRunnerRecoveryReason` from the saved run. These fields contain +and `remoteRunnerRecoveryReason` from the saved run. When reattachment fails +before the coordinator saves its result, the report reads the same labels from +the caught recovery error instead. These fields contain closed labels for marker reads and process verification. They distinguish missing or unsafe markers, invalid or missing fingerprints, changed or unavailable process identity, and failed or timed-out probes. Paths, PIDs, diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index f2f269039c2..da5f8228b1a 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -1,6 +1,7 @@ import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js"; import { describe, expect, it } from "vitest"; import type { heartbeatRuns } from "@paperclipai/db"; +import { attachRemoteRunnerRecoveryDiagnostic, RemoteRunnerRecoveryError } from "../native-runtime/remote-runner-recovery-diagnostics.js"; import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -222,6 +223,28 @@ describe("run failure diagnostics", () => { expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({}); }); + it("exports the remote recovery stage and reason from an error thrown before the result was saved", () => { + const thrown = Object.assign(new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }), + { path: "private-path", pid: 123 }); + const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown })); + expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" }); + expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]); + expect(JSON.stringify(result)).not.toContain("private"); + + const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"), + { stage: "identity_marker", reason: "probe_transport_failed" }); + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: new Error("wrapped", { cause: transport }) }).execution) + .toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "probe_transport_failed" }); + + const saved = collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", + resultJson: { remoteRunnerRecovery: { stage: "identity_marker", reason: "marker_missing" } } }), { error: thrown }); + expect(saved.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "marker_missing" }); + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { + error: Object.assign(new Error("runner_remote_process_identity_unavailable"), { code: "runner_remote_process_identity_unavailable", + remoteRunnerRecovery: { stage: "private-stage", reason: "marker_missing" } }), + }).execution).toEqual({}); + }); + it("exports which native close barrier failed without tool identities or provider text", () => { const closeFailure = (settlement: unknown) => Object.assign( new Error("provider_transport_failed: runner did not durably suspend before checkpoint"), diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index 47422a63c7f..3b331168f76 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -10,7 +10,7 @@ import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants import { redactCurrentUserText } from "../log-redaction.js"; import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js"; import { readNativeModelRejectionDiagnostic, readNativeTurnFailureDiagnostic } from "./native-runtime/native-provider-failure.js"; -import { readRemoteRunnerRecoveryDiagnostic } from "./native-runtime/remote-runner-recovery-diagnostics.js"; +import { readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError } from "./native-runtime/remote-runner-recovery-diagnostics.js"; import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -240,7 +240,11 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs; } const result = run.resultJson; - const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")); + // Restart reattachment can fail before the native coordinator saves its + // result. The thrown error then reaches the generic run failure path, so + // read the same closed labels from the error. + const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")) ?? + readRemoteRunnerRecoveryError(options.error); if (remoteRunnerRecovery) { execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage; execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;