diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 3b49eadb3ba..770afe9ba6e 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -769,6 +769,86 @@ describe("executeNativeSession recovery", () => { expect(events.filter(event => event.eventType === "run.terminal")).toHaveLength(1); }); + // A chat follow-up resumes the warm conversation of its previous run, so its + // session is recovered, and no restart claim exists. The turn it starts is + // still its own when a mid-run runnerd restore reports that turn lost. + it.each([ + { label: "continues its own turn on a resumed warm session once when runnerd is restored mid-execution", lostTurnId: "turn-own", dispositionOnly: false, continues: true }, + { label: "does not continue a lost turn a resumed warm session never started", lostTurnId: "turn-elsewhere", dispositionOnly: false, continues: false }, + { label: "does not continue a lost disposition-only recovery turn", lostTurnId: "turn-own", dispositionOnly: true, continues: false }, + ])("$label", async ({ lostTurnId, dispositionOnly, continues }) => { + const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; + // A warm checkpoint rebound to a new run keeps only its provider identity + // (loadWarmNativeCheckpoint). A same-run recovery after a completed turn + // keeps that turn and asks only for the disposition. + const warm: PersistedNativeSession = { + backendKind: "runner", driverKind: "acpx_runtime", sessionId: "thread-recovery", identity, + providerSessionId: "provider-recovery", cursor: "0", activeTurnId: null, semanticResult: null, + pendingRuntimeRequests: [], lineage: [], + terminalTurns: dispositionOnly + ? [{ turnId: "turn-completed", fingerprint: JSON.stringify({ terminalState: "completed", error: null, result: null }) }] + : [], + }; + const interrupted: PersistedNativeSession = { + ...warm, + terminalTurns: [...warm.terminalTurns!, + { turnId: lostTurnId, fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], + }; + let startedTurns = 0; + const startTurn = vi.fn(async () => { + startedTurns += 1; + return { turnId: startedTurns === 1 ? "turn-own" : "turn-continuation" }; + }); + // The runner reports the loss only after this execution submitted its turn. + const turnSubmitted = vi.waitFor(() => { expect(startedTurns).toBeGreaterThan(0); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() { + await turnSubmitted; + yield { ...runnerEvent(1, "turn.failed", { status: "failed", error: failure }), turnId: lostTurnId }; + if (startTurn.mock.calls.length < 2) return; + yield { ...runnerEvent(2, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; + }, + startTurn, + async result() { return startTurn.mock.calls.length > 1 ? { result, terminal, turnId: "turn-continuation" } : null; }, + async snapshot() { return structuredClone(startTurn.mock.calls.length ? interrupted : warm); }, + async close() {}, + }; + const events: PrpEvent[] = []; + const backend: NativeSessionBackend = { + async descriptor() { return { kind: "runner", name: "warm-restore", version: "1", capabilities: await session.capabilities() }; }, + async openSession() { throw new Error("must resume the warm provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event: PrpEvent) { events.push(event); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + const execution = executeNativeSession({ + input, backend, controlPlane: port, persistedSession: warm, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }); + if (!continues) { + await expect(execution).rejects.toMatchObject({ code: "native_provider_terminal_failed" }); + expect(startTurn).toHaveBeenCalledOnce(); + return; + } + await expect(execution).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } }); + expect(startTurn).toHaveBeenCalledTimes(2); + // The lost turn carried this run's task, not a disposition-only request. + const first = startTurn.mock.calls[0]![0] as Parameters[0]; + expect(first.message.text).toContain(JSON.stringify(input.task.prompt).slice(1, -1)); + const submitted = startTurn.mock.calls[1]![0] as Parameters[0]; + expect(submitted.continuation).toBe(true); + expectRestartContinuationGuidance(submitted.message.text); + expect(submitted.message.text).not.toContain(input.task.prompt); + expect(events.filter(event => event.eventType === "run.terminal")).toHaveLength(1); + }); + it.each([ { label: "after the continuation starts", staleFirst: false, staleTurnId: "turn-unrecorded-continuation" }, { label: "before the continuation starts", staleFirst: true, staleTurnId: "turn-unrecorded-continuation" }, diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 795df4003b7..7fed5d55bf4 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -3103,7 +3103,9 @@ export async function executeNativeSession( if (await requestContinuationDisposition(event, signal)) return true; const restoredAtStartup = recovered && options.resumeInterruptedTurn === true && event.turnId === persistedSession?.activeTurnId; - const restoredInThisExecution = !recovered && ownTurnId !== null && event.turnId === ownTurnId; + // The task turn this execution started is its own, whether the session + // was opened fresh or resumed from a warm or recovered checkpoint. + const restoredInThisExecution = ownTurnId !== null && event.turnId === ownTurnId; if (restartContinuationStarted || !(restoredAtStartup || restoredInThisExecution) || // A compaction-only run never continues task work after a restart. // Runnerd records a restart-lost turn only for these providers, after @@ -3428,7 +3430,7 @@ export async function executeNativeSession( // the fenced checkpoint right before the submit. A new session's // normal path adds nothing before turn.start. if (recovered) await checkpoint(); - ({ turnId: ownTurnId } = await session.startTurn({ + const startedTaskTurn = await session.startTurn({ message: { role: "user", text: JSON.stringify(modelEnvelope) }, ...(recovered && modelEnvelope.schema === "paperclip.native-continuation.v1" ? { continuation: true as const } : {}), @@ -3438,7 +3440,10 @@ export async function executeNativeSession( input.provider.kind === "cursor_cloud" ? "default" : "executionMode" in input ? input.executionMode : "default", - })); + }); + // Only a turn carrying this run's task work can be continued after a + // mid-run runnerd restore. A disposition-only recovery turn cannot. + if (!dispositionOnlyRecovery || effectFreeInitialAcpxTurn) ownTurnId = startedTaskTurn.turnId; await checkpoint(); } settleStartupDecision();