diff --git a/doc/observability.md b/doc/observability.md index 67fd9b4a4a0..78ae54df3a6 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -793,7 +793,9 @@ executor's identity redaction. These observations do not change close, quarantin or recovery behavior. Remote runner reattachment failures also record `remoteRunnerRecoveryStage` -and `remoteRunnerRecoveryReason` from the saved run. These fields contain +and `remoteRunnerRecoveryReason` from the saved run. When reattachment fails +before the coordinator saves its result, the report reads the same labels from +the caught recovery error instead. These fields contain closed labels for marker reads and process verification. They distinguish missing or unsafe markers, invalid or missing fingerprints, changed or unavailable process identity, and failed or timed-out probes. Paths, PIDs, diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index f2f269039c2..da5f8228b1a 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -1,6 +1,7 @@ import { captureEnvironmentAcquisitionDiagnostic } from "../environment-acquisition-diagnostics.js"; import { describe, expect, it } from "vitest"; import type { heartbeatRuns } from "@paperclipai/db"; +import { attachRemoteRunnerRecoveryDiagnostic, RemoteRunnerRecoveryError } from "../native-runtime/remote-runner-recovery-diagnostics.js"; import { collectRunFailureDiagnostics, collectRunFailureSecretValues, redactRunFailureSecretValues, sanitizeRunFailureDiagnostics, sanitizeRunFailureText } from "../run-failure-diagnostics.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -222,6 +223,28 @@ describe("run failure diagnostics", () => { expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({}); }); + it("exports the remote recovery stage and reason from an error thrown before the result was saved", () => { + const thrown = Object.assign(new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }), + { path: "private-path", pid: 123 }); + const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: thrown })); + expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "process_verification", remoteRunnerRecoveryReason: "process_not_running" }); + expect(result.exceptions).toMatchObject([{ code: "runner_remote_process_identity_unavailable" }]); + expect(JSON.stringify(result)).not.toContain("private"); + + const transport = attachRemoteRunnerRecoveryDiagnostic(new Error("socket closed"), + { stage: "identity_marker", reason: "probe_transport_failed" }); + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { error: new Error("wrapped", { cause: transport }) }).execution) + .toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "probe_transport_failed" }); + + const saved = collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", + resultJson: { remoteRunnerRecovery: { stage: "identity_marker", reason: "marker_missing" } } }), { error: thrown }); + expect(saved.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "marker_missing" }); + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), { + error: Object.assign(new Error("runner_remote_process_identity_unavailable"), { code: "runner_remote_process_identity_unavailable", + remoteRunnerRecovery: { stage: "private-stage", reason: "marker_missing" } }), + }).execution).toEqual({}); + }); + it("exports which native close barrier failed without tool identities or provider text", () => { const closeFailure = (settlement: unknown) => Object.assign( new Error("provider_transport_failed: runner did not durably suspend before checkpoint"), diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index 47422a63c7f..3b331168f76 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -10,7 +10,7 @@ import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants import { redactCurrentUserText } from "../log-redaction.js"; import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js"; import { readNativeModelRejectionDiagnostic, readNativeTurnFailureDiagnostic } from "./native-runtime/native-provider-failure.js"; -import { readRemoteRunnerRecoveryDiagnostic } from "./native-runtime/remote-runner-recovery-diagnostics.js"; +import { readRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError } from "./native-runtime/remote-runner-recovery-diagnostics.js"; import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -240,7 +240,11 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs; } const result = run.resultJson; - const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")); + // Restart reattachment can fail before the native coordinator saves its + // result. The thrown error then reaches the generic run failure path, so + // read the same closed labels from the error. + const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")) ?? + readRemoteRunnerRecoveryError(options.error); if (remoteRunnerRecovery) { execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage; execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason;