diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index f6eca1ad66d..9b9d6c170ac 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -9587,7 +9587,7 @@ describe("executeNativeSession recovery", () => { expect(history.filter(event => event.payload.kind === "usage")).toEqual([usage]); }); - it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "different-turn", "foreign-settlement", "active-settled-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { + it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "failed-terminal", "lost-not-recoverable", "different-turn", "foreign-settlement", "active-settled-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId, governedWait: { sourceEvent: trigger, result: yieldedResult } }; @@ -9596,6 +9596,11 @@ describe("executeNativeSession recovery", () => { runId: fault === "foreign-settlement" ? "another-run" : identity.runId, normalizedSessionId: identity.sessionId, turnId: trigger.turnId, }; if (fault === "unproven-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "unproven", eventType: "turn.failed", payload: { providerTerminalObserved: false } }); + // Only a recoverable restart loss proves the old provider lifetime ended. + if (fault === "failed-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "failed", eventType: "turn.failed", + payload: { status: "failed", error: { code: "model_not_found", recoverable: false } } }); + if (fault === "lost-not-recoverable") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "lost", eventType: "turn.failed", + payload: { status: "failed", error: { code: "provider_turn_lost_on_restore", recoverable: false } } }); if (fault === "different-turn") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "new-turn", turnId: "different", eventType: "turn.started" }); const start = vi.fn(); const complete = vi.fn(); @@ -9615,15 +9620,21 @@ describe("executeNativeSession recovery", () => { // the parked turn's terminal and the controller died during the accounting // drain. The drain had already discarded numbered non-accounting events, so // the saved replay has gaps after the trigger. - it.each(["terminal-after-restart", "late-terminal-saved"])("settles a saved governed wait whose drain left sequence gaps before a restart (%s)", async variant => { - const lateTerminalSaved = variant === "late-terminal-saved"; + // Claude (ACPX) Q1 with PV1/PV2 + K1 (wave w-20261110T094930-s214): the + // restored runner proves the old provider lifetime ended and reports the + // parked turn as lost on restore instead of interrupted. + it.each(["terminal-after-restart", "late-terminal-saved", "lost-on-restore-after-restart", "lost-on-restore-saved"])("settles a saved governed wait whose drain left sequence gaps before a restart (%s)", async variant => { + const lateTerminalSaved = variant.endsWith("-saved"); + const lostOnRestore = variant.startsWith("lost-on-restore"); const turn = (event: PrpEvent) => ({ ...event, turnId: "turn-waiting" }); const trigger = turn(runnerEvent(1, "item.completed", { kind: "dynamicToolCall", item: { id: "ask-1", name: "request_human_input" } })); const acknowledgement = turn(runnerEvent(2, "item.completed", { kind: "interrupt_acknowledgement", status: "acknowledged" })); const discardedMessage = turn(runnerEvent(3, "item.completed", { kind: "agentMessage", text: "I asked the board." })); const discardedToolEnd = turn(runnerEvent(4, "tool.execution.completed", { name: "request_human_input", status: "completed" })); const usage = turn(runnerEvent(5, "item.completed", { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } })); - const stopped = turn(runnerEvent(6, "turn.interrupted", { providerTerminalObserved: true })); + const stopped = lostOnRestore + ? turn(runnerEvent(6, "turn.failed", { provider: "acpx", status: "failed", providerTerminalObserved: false, code: "provider_turn_lost_on_restore", error: { code: "provider_turn_lost_on_restore", recoverable: true } })) + : turn(runnerEvent(6, "turn.interrupted", { providerTerminalObserved: true })); const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; const start = vi.fn(async () => ({ turnId: trigger.turnId })); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index f06e80af74c..178c28ec56b 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -424,6 +424,20 @@ function isTurnTerminal(event: PrpEvent): boolean { * parked turn. The drain discards every other numbered event, so the saved * replay has sequence gaps exactly where those discarded events were. */ +/** + * A parked turn's terminal proves the provider stopped its work. An interrupted + * or cancelled terminal does unless the provider never confirmed it. A failed + * terminal does only when Runnerd reports the turn lost on restore: it emits + * that cause after proving the old provider lifetime ended and restoring the + * conversation (ACPX reports a restart-killed turn this way; Codex reports it + * interrupted). That event carries `providerTerminalObserved: false`, because + * the provider itself sent no terminal; the restart proof stands in for it. + */ +function governedWaitTerminalProven(event: PrpEvent): boolean { + if (event.eventType === "turn.failed") return isNativeRestartInterruption(event.payload.error); + return event.payload.providerTerminalObserved !== false; +} + function isGovernedDrainRetainedEvent(event: PrpEvent, turnId: string | undefined): boolean { if (event.turnId !== turnId) return false; return isTurnTerminal(event) @@ -1235,7 +1249,7 @@ async function consumeTurn( eventCount += receipt.disposition === "committed" ? 1 : 0; highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); if (terminal) { - if (trailing.eventType === "turn.failed" || trailing.payload.providerTerminalObserved === false) { + if (!governedWaitTerminalProven(trailing)) { throw new NativeProviderTerminalFailure("governed_wait_provider_terminal_unproven", false); } governedProviderTerminalObserved = true; @@ -2870,7 +2884,7 @@ export async function executeNativeSession( const governedTerminal = governedWait ? governedReplay.find(event => event.turnId === governedWait!.sourceEvent.turnId && isTurnTerminal(event)) : undefined; - if (governedTerminal?.eventType === "turn.failed" || governedTerminal?.payload.providerTerminalObserved === false) { + if (governedTerminal && !governedWaitTerminalProven(governedTerminal)) { throw new NativeProviderTerminalFailure("governed_wait_provider_terminal_unproven", false); } if (governedWait && recoveredActiveTurnId && recoveredActiveTurnId !== governedWait.sourceEvent.turnId) {