diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index fccd60629d4..b7dcbe9e8e7 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1693,7 +1693,9 @@ Runner keeps its pending events and commits them under the prior authority befor rotates. Core accepts any authenticated event after the terminal, except one that starts work, and the next run's log names what was recorded. Identity, source and ownership checks are unchanged. A failed attachment reports its original cause; the -same run does not attach again. +same run does not attach again. A later run may start a new provider thread, and the +run's log and events link here when this happens. Whether a later run should keep +that thread is tracked in [#19047](https://github.com/paperclipai/paperclip/issues/19047). ### Provider continuity and bounded finalization diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 40595602922..41a6e101079 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -4,6 +4,7 @@ import { reconcileTaskChatRetryReceipts } from "./task-retry.js"; import { registerIssueRecoveryWork } from "./recovery/work.js"; import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js"; import { NATIVE_RETAINED_OWNER_WAIT } from "./native-runtime/native-controller-recovery.js"; +import { describeFailedAttachment } from "./native-runtime/remote-warm-session-recovery.js"; import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js"; import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js"; import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js"; @@ -3625,6 +3626,13 @@ export function heartbeatService( failureDetail: coordinator?.failureDetail ?? null, }, }).catch(() => undefined); + const failedAttachment = describeFailedAttachment( + (coordinator?.failureDetail as { message?: unknown } | null)?.message, + ); + if (failedAttachment) { + await appendRunEvent(run, { eventType: "lifecycle", stream: "system", level: "warn", message: failedAttachment, + payload: { code: "native_run_attach_failed" } }).catch(() => undefined); + } if (coordinator?.nextAttemptAt) { scheduleNativeSessionResumeDispatch( run.id, diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index cf0b510e647..f8e1e833b3e 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -286,6 +286,18 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expectCleanup(input); }); + it("logs a pointer to the docs when a same-run retry follows a failed attachment", async () => { + const { input, run } = await fixture(true); + await db.update(heartbeatRuns).set({ nativeIssueId: input.task.issueId }).where(eq(heartbeatRuns.id, run.id)); + await db.insert(nativeRunFinalizations).values({ companyId: run.companyId, issueId: input.task.issueId!, runId: run.id, phase: "retryable_failure" }); + const error = new Error('native_session_recovery_failed: Error: PRP command run.attach failed: {"result":{"message":"run.attach requires a settled Codex provider session: active_turn"}}'); + vi.mocked(executePaperclipNativeSession).mockRejectedValueOnce(error); + await expect(executeHeartbeatRuntime(db, input)).rejects.toMatchObject({ name: "NativeSessionResumeScheduledError", original: error }); + expect(input.output.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("doc/execution-semantics.md#session-settlement-after-a-run")); + expect(input.output.onLog).toHaveBeenCalledWith("stderr", expect.stringContaining("run.attach requires a settled Codex provider session: active_turn")); + expectCleanup(input); + }); + it("preserves the retained sandbox while native admission waits for its current owner", async () => { const { input } = await fixture(true); const error = new NativeControllerRecoveryWaitingError(new Date(Date.now() + 60_000)); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 6a0898f46f0..258a50b501f 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1,6 +1,7 @@ import { readClaimedNativeStopRecovery } from "../native-runtime/native-cancellation-request.js"; import { NativeStartupCleanupPendingError } from "../native-runtime/native-startup-cleanup.js"; import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js"; +import { describeFailedAttachment } from "../native-runtime/remote-warm-session-recovery.js"; import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; @@ -1143,6 +1144,8 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe : false; if (nativeResumeScheduled) { input.effects.onNativeSessionResumeScheduled(); + const failedAttachment = describeFailedAttachment(adapterErr instanceof Error ? adapterErr.message : null); + if (failedAttachment) await Promise.resolve(onLog("stderr", `[paperclip] ${failedAttachment}\n`)).catch(() => undefined); throw new NativeSessionResumeScheduledError(adapterErr); } // Adapter (or its restore finally) threw — or the finalize record diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.test.ts b/server/src/services/native-runtime/remote-warm-session-recovery.test.ts index 095ce924320..f364811e957 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.test.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { describeOwnFailedAttachment, isSettledRemoteWarmControlState, summarizePostTerminalEvents } from "./remote-warm-session-recovery.js"; +import { describeFailedAttachment, describeOwnFailedAttachment, isSettledRemoteWarmControlState, summarizePostTerminalEvents } from "./remote-warm-session-recovery.js"; const identity = { runId: "run", normalizedSessionId: "session", runnerInstanceId: "runner", environmentLeaseId: "lease", turnId: "turn", itemId: "item" }; const terminal = { eventType: "run.terminal", envelope: { ...identity } }; @@ -78,6 +78,18 @@ describe("settled remote warm control evidence", () => { .toBe("provider.notice.recorded x2, session.goal.snapshot x1"); }); + it("points a failed attachment at the reviewed docs with its real cause", () => { + const stored = 'native_session_recovery_failed: Error: PRP command run.attach failed: {"commandType":"run.attach","result":{"code":"command_execution_failed","message":"run.attach requires a settled Codex provider session: active_turn"},"status":"failed"}'; + const pointer = describeFailedAttachment(stored); + expect(pointer).toContain("run.attach requires a settled Codex provider session: active_turn"); + expect(pointer).toContain("https://github.com/paperclipai/paperclip/blob/master/doc/execution-semantics.md#session-settlement-after-a-run"); + expect(describeFailedAttachment("run.attach requires the restored Codex provider process")).toContain("restored Codex provider process"); + const quoted = `PRP command run.attach failed: ${JSON.stringify({ result: { message: 'run.attach provider is invalid: unknown "value" here' } })}`; + expect(describeFailedAttachment(quoted)).toContain('run.attach provider is invalid: unknown "value" here. A later run'); + expect(describeFailedAttachment("Plugin Runner operation failed; reconcile the saved task")).toBeNull(); + expect(describeFailedAttachment(null)).toBeNull(); + }); + it("names the original failure when a same-run retry finds its own failed attachment", () => { expect(describeOwnFailedAttachment([{ stderrTail: "run.attach requires a settled session: active_turn" }])) .toBe("runner_state_identity_mismatch: prior_attach_failed: run.attach requires a settled session: active_turn"); diff --git a/server/src/services/native-runtime/remote-warm-session-recovery.ts b/server/src/services/native-runtime/remote-warm-session-recovery.ts index 44a37c42ef8..bc0d7ed31d4 100644 --- a/server/src/services/native-runtime/remote-warm-session-recovery.ts +++ b/server/src/services/native-runtime/remote-warm-session-recovery.ts @@ -79,6 +79,18 @@ export function describeOwnFailedAttachment(recoveryHistory: unknown): string { return `runner_state_identity_mismatch: prior_attach_failed: ${typeof cause === "string" && cause ? cause : "unknown"}`; } +const SETTLEMENT_DOC = "https://github.com/paperclipai/paperclip/blob/master/doc/execution-semantics.md#session-settlement-after-a-run"; + +/** Points whoever diagnoses a failed attachment at reviewed docs. The stored + * failure is already redacted and bounded; other failures produce no pointer. */ +export function describeFailedAttachment(failureMessage: unknown): string | null { + if (typeof failureMessage !== "string" || !/\brun\.attach (failed|requires)\b/.test(failureMessage)) return null; + const encoded = /"message":("(?:[^"\\]|\\.)*")/.exec(failureMessage)?.[1]; + let cause = failureMessage; + try { if (encoded) cause = JSON.parse(encoded) as string; } catch { /* Keep the whole failure. */ } + return `This run could not attach to its provider session: ${cause}. A later run may start a new provider thread. See ${SETTLEMENT_DOC}`; +} + export async function inspectRemoteWarmSessionRecovery(input: { db: Db; execution: NativeExecutionInput;