From ff0f976de1036555fb9deda8c1f31a5f748723f6 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:24:02 -0500 Subject: [PATCH] fix(native-runtime): retain remote recovery failure reasons (#16341) Preserve closed marker-read and process-verification reasons through native recovery failures and operator reports. Keep existing transport errors and reattachment ownership checks. Add shell, process-probe, persistence and privacy regression coverage. All current-head CI checks pass and Greptile is 5/5. Co-Authored-By: Paperclip --- doc/observability.md | 10 ++ .../__tests__/run-failure-diagnostics.test.ts | 19 ++++ .../native-session-executor.test.ts | 83 ++++++++++++++++- .../native-runtime/native-session-executor.ts | 35 ++++--- ...remote-runner-recovery-diagnostics.test.ts | 70 ++++++++++++++ .../remote-runner-recovery-diagnostics.ts | 91 +++++++++++++++++++ .../src/services/run-failure-diagnostics.ts | 6 ++ 7 files changed, 298 insertions(+), 16 deletions(-) create mode 100644 server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts create mode 100644 server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts diff --git a/doc/observability.md b/doc/observability.md index 439384fc4e9..67fd9b4a4a0 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -792,6 +792,16 @@ only for these labels; its text and stack are not copied, because they bypass th executor's identity redaction. These observations do not change close, quarantine or recovery behavior. +Remote runner reattachment failures also record `remoteRunnerRecoveryStage` +and `remoteRunnerRecoveryReason` from the saved run. These fields contain +closed labels for marker reads and process verification. They distinguish +missing or unsafe markers, invalid or missing fingerprints, changed or +unavailable process identity, and failed or timed-out probes. Paths, PIDs, +nonces, command lines and transport output are excluded. The existing +`runner_remote_process_identity_unavailable` error message and native failure +classification remain unchanged. The coordinator also saves these labels in +its failure detail and recovery evidence. A later failure clears stale labels. + ACP turns record `acpLastEventAgeMs`, `acpObservedEventCount`, `acpPendingToolCount`, and `acpToolInventoryComplete` at finalization, before usage reads, error logging, and cleanup. The age measures time since the last diff --git a/server/src/services/__tests__/run-failure-diagnostics.test.ts b/server/src/services/__tests__/run-failure-diagnostics.test.ts index d21bddde1c1..7f1f790fb4c 100644 --- a/server/src/services/__tests__/run-failure-diagnostics.test.ts +++ b/server/src/services/__tests__/run-failure-diagnostics.test.ts @@ -190,6 +190,25 @@ describe("run failure diagnostics", () => { } }); + it("exports only the saved remote recovery stage and reason", () => { + const result = sanitizeRunFailureDiagnostics(collectRunFailureDiagnostics(run({ + errorCode: "runner_remote_process_identity_unavailable", resultJson: { + remoteRunnerRecovery: { stage: "identity_marker", reason: "fingerprint_missing", + path: "private-path", pid: 123, nonce: "private-nonce", stderr: "private-stderr" }, + }, + }), {})); + expect(result.execution).toEqual({ remoteRunnerRecoveryStage: "identity_marker", remoteRunnerRecoveryReason: "fingerprint_missing" }); + expect(result.exceptions).toEqual([]); + expect(JSON.stringify(result)).not.toContain("private"); + for (const value of [null, { stage: "private-stage", reason: "marker_missing" }, + { stage: "identity_marker", reason: "private-reason" }, + Object.defineProperty({}, "stage", { get() { throw new Error("private"); } })]) { + expect(collectRunFailureDiagnostics(run({ errorCode: "runner_remote_process_identity_unavailable", + resultJson: { remoteRunnerRecovery: value } }), {}).execution).toEqual({}); + } + expect(collectRunFailureDiagnostics(run({ errorCode: "adapter_failed" }), {}).execution).toEqual({}); + }); + it("exports which native close barrier failed without tool identities or provider text", () => { const closeFailure = (settlement: unknown) => Object.assign( new Error("provider_transport_failed: runner did not durably suspend before checkpoint"), diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 741bda8685e..b0555926fe4 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -52,6 +52,7 @@ import { import { createHash, generateKeyPairSync, randomUUID } from "node:crypto"; import { DatabaseSync } from "node:sqlite"; import { nativeSha256 } from "./canonical.js"; +import { RemoteRunnerRecoveryError, readRemoteRunnerRecoveryError } from "./remote-runner-recovery-diagnostics.js"; import { agentDirectoryWorkingCopyService } from "../agent-directory-working-copies.js"; import { probeAgentDirectory } from "../agent-directory-probe.js"; import { AGENT_FILES_CONTRACT } from "../agent-file-store.js"; @@ -592,9 +593,40 @@ describe("remote controller restart adoption", () => { ? { stdout: "nonce\n123\n2026-09-19T10:00:00.000Z\nrunner\n", stderr: "", exitCode: 0, timedOut: false } : original(request)); await expect(verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" })) - .rejects.toThrow("runner_remote_process_identity_unavailable"); + .rejects.toMatchObject({ + message: "runner_remote_process_identity_unavailable", + remoteRunnerRecovery: { stage: "identity_marker", reason: "fingerprint_missing" }, + }); }); - it.each(["start ticks", "boot identity"])("rejects PID reuse after the live %s changes", async (change) => { + it.each([ + [31, "marker_missing"], [32, "marker_unsafe"], [33, "marker_read_failed"], + ] as const)("records the closed reason for marker read exit %s", async (exitCode, reason) => { + const { target, execute } = fixture(); + const original = execute.getMockImplementation()!; + execute.mockImplementation(async request => request.args[2] === "paperclip-runner-recovery-identity" + ? { stdout: "", stderr: "private transport text", exitCode, timedOut: false } + : original(request)); + await expect(verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" })) + .rejects.toMatchObject({ message: "runner_remote_process_identity_unavailable", + remoteRunnerRecovery: { stage: "identity_marker", reason } }); + }); + it.each(["timeout", "transport"] as const)("records a marker %s without retaining transport text", async failure => { + const { target, execute } = fixture(); + const transportError = new Error("private bridge response"); + const original = execute.getMockImplementation()!; + execute.mockImplementation(async request => { + if (request.args[2] !== "paperclip-runner-recovery-identity") return original(request); + if (failure === "transport") throw transportError; + return { stdout: "", stderr: "private bridge response", exitCode: -1, timedOut: true }; + }); + const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error); + const diagnostic = readRemoteRunnerRecoveryError(error); + expect(diagnostic).toEqual({ stage: "identity_marker", reason: failure === "timeout" ? "probe_timeout" : "probe_transport_failed" }); + expect(JSON.stringify(diagnostic)).not.toContain("private"); + if (failure === "transport") expect(error).toBe(transportError); + else expect(error).toMatchObject({ message: "runner_remote_process_identity_unavailable" }); + }); + it.each(["start ticks", "boot identity", "fingerprint probe"])("rejects a changed or unavailable live %s", async (change) => { const root = await mkdtemp(join(tmpdir(), "remote-generation-")); const proc = join(root, "proc"); const markerPath = join(root, "runner-process.identity"); @@ -621,18 +653,39 @@ describe("remote controller restart adoption", () => { args[1] = args[1].replaceAll("/proc/", `${proc}/`); args[3] = markerPath; let exitCode = 0; - try { execFileSync("sh", args, { stdio: "pipe" }); } catch { exitCode = 4; } + try { execFileSync("sh", args, { stdio: "pipe" }); } catch (error) { exitCode = (error as { status?: number }).status ?? 4; } return { stdout: "", stderr: "", exitCode, timedOut: false }; }); const adopted = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }); expect(await adopted.isAlive()).toBe(true); if (change === "start ticks") await writeFile(join(proc, String(pid), "stat"), stat(101)); - else await writeFile(join(proc, "sys/kernel/random/boot_id"), "abcd-5678\n"); + else if (change === "boot identity") await writeFile(join(proc, "sys/kernel/random/boot_id"), "abcd-5678\n"); + else await rm(join(proc, String(pid), "stat")); expect(await adopted.isAlive()).toBe(false); await expect(verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" })) - .rejects.toThrow("runner_remote_process_identity_unavailable"); + .rejects.toMatchObject({ message: "runner_remote_process_identity_unavailable", remoteRunnerRecovery: { + stage: "process_verification", reason: change === "fingerprint probe" ? "fingerprint_unavailable" : "process_generation_changed", + } }); } finally { await rm(root, { recursive: true, force: true }); } }); + it.each(["liveness", "timeout", "transport"] as const)("retains the reason for a failed process %s probe", async failure => { + const { target, execute } = fixture(); + const original = execute.getMockImplementation()!; + const transportError = new Error("provider_transport_failed: private bridge response"); + execute.mockImplementation(async request => { + if (request.args[2] !== "paperclip-runner-recovery-check") return original(request); + if (failure === "transport") throw transportError; + return { stdout: "", stderr: "private probe output", exitCode: failure === "liveness" ? 45 : -1, timedOut: failure === "timeout" }; + }); + const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error); + const diagnostic = readRemoteRunnerRecoveryError(error); + expect(diagnostic).toEqual({ stage: "process_verification", reason: failure === "liveness" ? "process_not_running" : failure === "timeout" ? "probe_timeout" : "probe_transport_failed" }); + expect(JSON.stringify(diagnostic)).not.toContain("private"); + if (failure === "transport") { + expect(error).toBe(transportError); + expect(nativeSessionFailureSourceCode(error)).toBe("provider_transport_failed"); + } + }); it("rejects a stale process marker", async () => { const { execute, target } = fixture(); execute @@ -8685,6 +8738,26 @@ describe("native warm session supervision", () => { }); describe("native session bounded recovery", () => { + it("persists closed remote recovery evidence without replacing saved run results", async () => { + const updates: Array<{ table: unknown; values: Record }> = []; + const remoteRunnerRecovery = { stage: "identity_marker", reason: "fingerprint_missing" } as const; + const failure = new RemoteRunnerRecoveryError(remoteRunnerRecovery); + state.execute.mockReset().mockRejectedValueOnce(failure); + state.upsertRecoveryAction.mockReset().mockResolvedValue({}); + const service = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ update: vi.fn(async () => null) } as unknown as ReturnType); + try { + await expect(executePaperclipNativeSession({ db: leaseDb(execution, {}, {}, updates), execution, runnerInstanceId: "runner" })).rejects.toBe(failure); + expect(updates.find(update => update.table === nativeRunFinalizations && update.values.failureDetail)?.values.failureDetail) + .toMatchObject({ remoteRunnerRecovery }); + const resultJson = updates.find(update => update.table === heartbeatRuns && update.values.error)?.values.resultJson as SQL; + const query = new PgDialect().sqlToQuery(resultJson); + expect(query.sql).toContain('coalesce("heartbeat_runs"."result_json",'); + expect(query.sql).toContain("- 'remoteRunnerRecovery'"); + expect(query.params).toContain(JSON.stringify({ remoteRunnerRecovery })); + expect(state.upsertRecoveryAction).toHaveBeenCalledWith(expect.objectContaining({ evidence: expect.objectContaining({ remoteRunnerRecovery }) })); + expect(state.execute).toHaveBeenCalledOnce(); + } finally { service.mockRestore(); } + }); it("blocks a denied Cursor task and gives recovery to the operator without another provider attempt", async () => { const updates: Array<{ table: unknown; values: Record }> = []; const failure = new NativePermissionDeclinedError(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 80e0f40addb..daa869dd6de 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -155,6 +155,10 @@ import { nativeRuntimeContractForProvider, nativeToolContractFingerprintForTarge import { verifyRetainedMaintenanceNoLaunch } from "./native-maintenance-no-launch.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { reportRunFailure } from "../run-failure-report.js"; +import { + REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, + RemoteRunnerRecoveryError, attachRemoteRunnerRecoveryDiagnostic, readRemoteRunnerRecoveryError, remoteRecoveryProbeReason, +} from "./remote-runner-recovery-diagnostics.js"; import { persistActivity, publishActivity } from "../activity-log.js"; import { commitNativeStatusDecision } from "./status-decision-committer.js"; import { resolvePaperclipInstanceRoot } from "../../home-paths.js"; @@ -9158,6 +9162,7 @@ async function executePaperclipNativeSessionWithinScope( error instanceof Error ? error.message : String(error), ).slice(0, 2_000); const sanitizedStderrTail = redactIdentityText(message).slice(-4_096); + const remoteRunnerRecovery = readRemoteRunnerRecoveryError(error); // Set inside the transaction only when the write below genuinely // transitions the run into "failed". Read after the transaction // commits, so a rolled-back write never reports a false failure. @@ -9190,6 +9195,7 @@ async function executePaperclipNativeSessionWithinScope( failureCode, failureDetail: identityRedactor.redact({ message, + ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}), originalFailureCode: error instanceof NativeProviderTerminalFailure ? error.providerCode @@ -9314,6 +9320,7 @@ async function executePaperclipNativeSessionWithinScope( nativePhase: phase, nativePhaseUpdatedAt: now, error: message, + resultJson: sql`(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) - 'remoteRunnerRecovery') || ${JSON.stringify(remoteRunnerRecovery ? { remoteRunnerRecovery } : {})}::jsonb`, errorCode: ownershipUnverified ? NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE : sourceFailureCode, @@ -9376,6 +9383,7 @@ async function executePaperclipNativeSessionWithinScope( } } : {}), coordinatorAttempt: attempt, sourceFailureCode, + ...(remoteRunnerRecovery ? { remoteRunnerRecovery } : {}), recoveryDisposition: failureCode, recoveryMode: recoveryEvidence.recoveryMode, providerSessionEstablished: @@ -10726,13 +10734,13 @@ export async function verifyRemoteRunnerReattachment(input: { command: "sh", args: [ "-c", - 'test -f "$1" && test ! -L "$1" && cat -- "$1"', + REMOTE_RECOVERY_MARKER_READ_SCRIPT, "paperclip-runner-recovery-identity", identityPath, ], bypassSession: true, timeoutMs: 10_000, - }); + }).catch(error => { throw attachRemoteRunnerRecoveryDiagnostic(error, { stage: "identity_marker", reason: "probe_transport_failed" }); }); const nonce = marker.stdout.split("\n")[0] ?? ""; const runnerInstanceId = identity.runnerInstanceId as string; const processIdentity = @@ -10742,15 +10750,19 @@ export async function verifyRemoteRunnerReattachment(input: { runnerInstanceId, }) : null; - if (!processIdentity?.processStartFingerprint) - throw new Error("runner_remote_process_identity_unavailable"); + if (marker.exitCode !== 0 || marker.timedOut) { + throw new RemoteRunnerRecoveryError({ stage: "identity_marker", + reason: remoteRecoveryProbeReason("identity_marker", marker.exitCode, marker.timedOut) }); + } + if (!processIdentity) throw new RemoteRunnerRecoveryError({ stage: "identity_marker", reason: "marker_invalid" }); + if (!processIdentity.processStartFingerprint) throw new RemoteRunnerRecoveryError({ stage: "identity_marker", reason: "fingerprint_missing" }); const expectedStartFingerprint = processIdentity.processStartFingerprint; const check = async (signal?: NodeJS.Signals) => { const result = await runner.execute({ command: "sh", args: [ "-c", - `${REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT}; ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; test "$process_fingerprint" = "$5" || exit 4` + + `${REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT}; if ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; then :; else exit 46; fi; test "$process_fingerprint" = "$5" || exit 47` + (signal ? '; kill -"$6" "$pid"' : ""), "paperclip-runner-recovery-check", identityPath, @@ -10762,18 +10774,19 @@ export async function verifyRemoteRunnerReattachment(input: { ], bypassSession: true, timeoutMs: 10_000, - }); - return result.exitCode === 0 && !result.timedOut; + }).catch(error => { throw attachRemoteRunnerRecoveryDiagnostic(error, { stage: "process_verification", reason: "probe_transport_failed" }); }); + return { alive: result.exitCode === 0 && !result.timedOut, + reason: remoteRecoveryProbeReason("process_verification", result.exitCode, result.timedOut) }; }; - if (!(await check())) - throw new Error("runner_remote_process_identity_unavailable"); + const verified = await check(); + if (!verified.alive) throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: verified.reason }); // These checks authorize an attempt to reconnect, not the task itself. The // transport still requires authentication against the exact durable PRP key. return { ...processIdentity, processGroupId: null, - isAlive: () => check(), - signal: check, + isAlive: async () => (await check()).alive, + signal: async (signal: NodeJS.Signals) => (await check(signal)).alive, }; } diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts new file mode 100644 index 00000000000..b917c2b3d32 --- /dev/null +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts @@ -0,0 +1,70 @@ +import { execFileSync } from "node:child_process"; +import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, + RemoteRunnerRecoveryError, readRemoteRunnerRecoveryError, +} from "./remote-runner-recovery-diagnostics.js"; + +function shellExit(script: string, args: string[]) { + try { + execFileSync("sh", ["-c", script, "recovery-test", ...args], { stdio: "pipe", timeout: 2_000 }); + return 0; + } catch (error) { + return (error as { status: number }).status; + } +} + +describe("remote recovery diagnostic shell receipts", () => { + it("distinguishes missing, unsafe and valid marker reads without exposing their contents", async () => { + const root = await mkdtemp(join(tmpdir(), "remote-recovery-read-")); + const marker = join(root, "marker"); + try { + expect(shellExit(REMOTE_RECOVERY_MARKER_READ_SCRIPT, [marker])).toBe(31); + await writeFile(join(root, "target"), "private marker"); + await symlink(join(root, "target"), marker); + expect(shellExit(REMOTE_RECOVERY_MARKER_READ_SCRIPT, [marker])).toBe(32); + await rm(marker); + await writeFile(marker, "private marker"); + expect(shellExit(REMOTE_RECOVERY_MARKER_READ_SCRIPT, [marker])).toBe(0); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("distinguishes a torn marker from a changed identity before signalling any process", async () => { + const root = await mkdtemp(join(tmpdir(), "remote-recovery-check-")); + const marker = join(root, "marker"); + const args = [marker, "nonce", "runner", String(process.pid)]; + try { + expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, args)).toBe(41); + await writeFile(marker, "nonce\n"); + expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, args)).toBe(43); + await writeFile(marker, `other-nonce\n${process.pid}\n2026-10-14T00:00:00.000Z\nrunner\n`); + expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, args)).toBe(44); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("reads only closed labels through bounded causes and contains hostile getters", () => { + const diagnostic = { stage: "process_verification", reason: "process_generation_changed" } as const; + const error = new RemoteRunnerRecoveryError(diagnostic); + expect(readRemoteRunnerRecoveryError(new Error("private outer text", { cause: error }))).toEqual(diagnostic); + expect(readRemoteRunnerRecoveryError(Object.assign(new Error("private"), { + code: error.code, remoteRunnerRecovery: { stage: "private", reason: "process_generation_changed" }, + }))).toBeNull(); + expect(readRemoteRunnerRecoveryError(Object.defineProperty({}, "code", { get() { throw new Error("private"); } }))).toBeNull(); + const cyclic: { cause?: unknown } = {}; + cyclic.cause = cyclic; + expect(readRemoteRunnerRecoveryError(cyclic)).toBeNull(); + }); + + it("reports a retired owned child as unavailable before checking its command line", async () => { + const root = await mkdtemp(join(tmpdir(), "remote-recovery-retired-")); + const marker = join(root, "marker"); + const pid = execFileSync("sh", ["-c", "printf '%s' $$"], { encoding: "utf8" }).trim(); + try { + await writeFile(marker, `nonce\n${pid}\n2026-10-14T00:00:00.000Z\nrunner\n`); + expect(shellExit(REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, [marker, "nonce", "runner", pid])).toBe(45); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts new file mode 100644 index 00000000000..9eb00409025 --- /dev/null +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts @@ -0,0 +1,91 @@ +export const REMOTE_RUNNER_RECOVERY_ERROR_CODE = "runner_remote_process_identity_unavailable"; + +const stages = ["identity_marker", "process_verification"] as const; +const reasons = [ + "marker_missing", "marker_unsafe", "marker_read_failed", "marker_invalid", + "fingerprint_missing", "process_not_running", "process_identity_changed", + "fingerprint_unavailable", "process_generation_changed", "probe_timeout", + "probe_transport_failed", "probe_failed", +] as const; + +export interface RemoteRunnerRecoveryDiagnostic { + stage: typeof stages[number]; + reason: typeof reasons[number]; +} + +const transportDiagnostics = new WeakMap(); + +export function attachRemoteRunnerRecoveryDiagnostic(error: unknown, diagnostic: RemoteRunnerRecoveryDiagnostic): unknown { + if (error && typeof error === "object") { + transportDiagnostics.set(error, diagnostic); + return error; + } + return new RemoteRunnerRecoveryError(diagnostic); +} + +export class RemoteRunnerRecoveryError extends Error { + readonly code = REMOTE_RUNNER_RECOVERY_ERROR_CODE; + + constructor(readonly remoteRunnerRecovery: RemoteRunnerRecoveryDiagnostic) { + super(REMOTE_RUNNER_RECOVERY_ERROR_CODE); + } +} + +export function readRemoteRunnerRecoveryDiagnostic(value: unknown): RemoteRunnerRecoveryDiagnostic | null { + if (!value || typeof value !== "object") return null; + try { + const { stage, reason } = value as Record; + if (!stages.some(candidate => candidate === stage) || !reasons.some(candidate => candidate === reason)) return null; + return { stage, reason } as RemoteRunnerRecoveryDiagnostic; + } catch { + return null; + } +} + +export function readRemoteRunnerRecoveryError(error: unknown): RemoteRunnerRecoveryDiagnostic | null { + let candidate = error; + for (let depth = 0; depth < 4 && candidate && typeof candidate === "object"; depth++) { + const transportDiagnostic = transportDiagnostics.get(candidate); + if (transportDiagnostic) return transportDiagnostic; + try { + const record = candidate as Record; + if (record.code === REMOTE_RUNNER_RECOVERY_ERROR_CODE) return readRemoteRunnerRecoveryDiagnostic(record.remoteRunnerRecovery); + candidate = record.cause; + } catch { + return null; + } + } + return null; +} + +export const REMOTE_RECOVERY_MARKER_READ_SCRIPT = + 'test ! -L "$1" || exit 32; test -f "$1" || exit 31; cat -- "$1" || exit 33'; + +export const REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT = [ + "set -eu", + "identity_path=$1; expected_nonce=$2; expected_runner_id=$3; expected_pid=$4", + 'test ! -L "$identity_path" || exit 42', + 'test -f "$identity_path" || exit 41', + '{ IFS= read -r nonce; IFS= read -r pid; IFS= read -r started_at; IFS= read -r runner_id; } < "$identity_path" || exit 43', + 'test "$nonce" = "$expected_nonce" && test "$runner_id" = "$expected_runner_id" && test "$pid" = "$expected_pid" && test -n "$started_at" || exit 44', + 'kill -0 "$pid" 2>/dev/null || exit 45', + 'if test -r "/proc/$pid/cmdline"; then command_line=$(tr "\\000" "\\n" < "/proc/$pid/cmdline"); printf "%s\\n" "$command_line" | grep -Fqx -- "--runner-id" || exit 44; printf "%s\\n" "$command_line" | grep -Fqx -- "$expected_runner_id" || exit 44; fi', +].join("; "); + +export function remoteRecoveryProbeReason(stage: RemoteRunnerRecoveryDiagnostic["stage"], exitCode: number | null, timedOut: boolean): RemoteRunnerRecoveryDiagnostic["reason"] { + if (timedOut) return "probe_timeout"; + if (stage === "identity_marker") { + if (exitCode === 31) return "marker_missing"; + if (exitCode === 32) return "marker_unsafe"; + if (exitCode === 33) return "marker_read_failed"; + } else { + if (exitCode === 41) return "marker_missing"; + if (exitCode === 42) return "marker_unsafe"; + if (exitCode === 43) return "marker_read_failed"; + if (exitCode === 44) return "process_identity_changed"; + if (exitCode === 45) return "process_not_running"; + if (exitCode === 46) return "fingerprint_unavailable"; + if (exitCode === 47) return "process_generation_changed"; + } + return "probe_failed"; +} diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index 009d7264cea..291a9503b77 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -10,6 +10,7 @@ import { ACP_TOOL_KINDS } from "@paperclipai/adapter-utils/acpx-engine/constants import { redactCurrentUserText } from "../log-redaction.js"; import { redactSensitiveText, REDACTED_EVENT_VALUE } from "../redaction.js"; import { readNativeModelRejectionDiagnostic } from "./native-runtime/native-provider-failure.js"; +import { readRemoteRunnerRecoveryDiagnostic } from "./native-runtime/remote-runner-recovery-diagnostics.js"; import { MANAGED_GIT_WORKTREE_REASON_CODES, PERSISTED_WORKSPACE_SOURCE_REASON_CODES, readManagedGitInspectionDiagnostic } from "./workspace-validation-diagnostics.js"; type Run = typeof heartbeatRuns.$inferSelect; @@ -239,6 +240,11 @@ export function collectRunFailureDiagnostics(run: Run, options: RunFailureReport if (Number.isFinite(durationMs) && durationMs >= 0) execution.durationMs = durationMs; } const result = run.resultJson; + const remoteRunnerRecovery = readRemoteRunnerRecoveryDiagnostic(read(result, "remoteRunnerRecovery")); + if (remoteRunnerRecovery) { + execution.remoteRunnerRecoveryStage = remoteRunnerRecovery.stage; + execution.remoteRunnerRecoveryReason = remoteRunnerRecovery.reason; + } const configuration = read(result, "configurationIncomplete"); if (run.errorCode === "configuration_incomplete" && read(configuration, "reason") === "workspace_base_ref_unresolved") { const diagnostic = readWorkspaceBaseRefDiagnostic(read(configuration, "baseRefDiagnostic"));