diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index c5a8842fc5f..16e98e8830f 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -710,6 +710,64 @@ describe("executeNativeSession recovery", () => { expect(checkpoints.some(snapshot => snapshot.terminalTurns?.some(turn => turn.turnId === "turn-recovery"))).toBe(true); }); + it.each([ + { label: "continues its own turn once when runnerd is restored mid-execution", lostTurnId: "turn-recovery", continues: true }, + { label: "does not continue a lost turn this execution never started", lostTurnId: "turn-elsewhere", continues: false }, + ])("$label", async ({ lostTurnId, continues }) => { + const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; + const interrupted: PersistedNativeSession = { + backendKind: "runner", driverKind: "acpx_runtime", sessionId: "thread-recovery", identity, + providerSessionId: "provider-recovery", cursor: "0", activeTurnId: null, semanticResult: null, + pendingRuntimeRequests: [], lineage: [], + terminalTurns: [{ turnId: lostTurnId, fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], + }; + const startTurn = vi.fn(async () => ({ + turnId: startTurn.mock.calls.length === 1 ? "turn-recovery" : "turn-continuation", + })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() { + yield { ...runnerEvent(1, "turn.failed", { status: "failed", error: failure }), turnId: lostTurnId }; + if (startTurn.mock.calls.length < 2) return; + yield { ...runnerEvent(2, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; + }, + startTurn, + async result() { return startTurn.mock.calls.length > 1 ? { result, terminal, turnId: "turn-continuation" } : null; }, + async snapshot() { return startTurn.mock.calls.length ? structuredClone(interrupted) : { ...structuredClone(interrupted), terminalTurns: [] }; }, + async close() {}, + }; + const events: PrpEvent[] = []; + const backend: NativeSessionBackend = { + async descriptor() { return { kind: "runner", name: "in-execution-restore", version: "1", capabilities: await session.capabilities() }; }, + async openSession() { return session; }, + async recoverSession() { throw new Error("a fresh execution opens its own session"); }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event: PrpEvent) { events.push(event); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + const execution = executeNativeSession({ + input, backend, controlPlane: port, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", + }); + if (!continues) { + await expect(execution).rejects.toMatchObject({ code: "native_provider_terminal_failed" }); + expect(startTurn).toHaveBeenCalledOnce(); + return; + } + await expect(execution).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } }); + expect(startTurn).toHaveBeenCalledTimes(2); + const submitted = startTurn.mock.calls[1]![0] as Parameters[0]; + expect(submitted.continuation).toBe(true); + expectRestartContinuationGuidance(submitted.message.text); + expect(submitted.message.text).not.toContain(input.task.prompt); + expect(events.filter(event => event.eventType === "run.terminal")).toHaveLength(1); + }); + it.each([ { label: "after the continuation starts", staleFirst: false, staleTurnId: "turn-unrecorded-continuation" }, { label: "before the continuation starts", staleFirst: true, staleTurnId: "turn-unrecorded-continuation" }, diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index ce262c411af..795df4003b7 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -3040,6 +3040,7 @@ export async function executeNativeSession( }; }; let restartContinuationStarted = false; + let ownTurnId: string | null = null; let borrowedContinuationTurnId: string | null = null; let restartContinuationTurnId: string | null = null; let settleStartupDecision: () => void = () => undefined; @@ -3100,14 +3101,16 @@ export async function executeNativeSession( }; const resumeInterruptedTurn = async (event: PrpEvent, signal: AbortSignal) => { if (await requestContinuationDisposition(event, signal)) return true; - if (restartContinuationStarted || !recovered || !options.resumeInterruptedTurn || + const restoredAtStartup = recovered && options.resumeInterruptedTurn === true && + event.turnId === persistedSession?.activeTurnId; + const restoredInThisExecution = !recovered && ownTurnId !== null && event.turnId === ownTurnId; + if (restartContinuationStarted || !(restoredAtStartup || restoredInThisExecution) || // A compaction-only run never continues task work after a restart. // Runnerd records a restart-lost turn only for these providers, after // proving the old process ended and restoring the same conversation. !["codex", "acpx", "opencode"].includes(input.provider.kind) || options.sessionCompaction || options.sessionGoalControl || options.resumeSessionGoalHeartbeat || recoveredSnapshot.goal || persistedSession?.semanticResult || persistedSession?.pendingRuntimeRequests?.length || - event.eventType !== "turn.failed" || !isNativeRestartInterruption(event.payload.error) || - event.turnId !== persistedSession?.activeTurnId) return false; + event.eventType !== "turn.failed" || !isNativeRestartInterruption(event.payload.error)) return false; const snapshot = await session.snapshot(); signal.throwIfAborted(); if (nativeRestartInterruptedTurnId(snapshot) !== event.turnId || snapshot.pendingRuntimeRequests?.length) return false; @@ -3425,7 +3428,7 @@ export async function executeNativeSession( // the fenced checkpoint right before the submit. A new session's // normal path adds nothing before turn.start. if (recovered) await checkpoint(); - await session.startTurn({ + ({ turnId: ownTurnId } = await session.startTurn({ message: { role: "user", text: JSON.stringify(modelEnvelope) }, ...(recovered && modelEnvelope.schema === "paperclip.native-continuation.v1" ? { continuation: true as const } : {}), @@ -3435,7 +3438,7 @@ export async function executeNativeSession( input.provider.kind === "cursor_cloud" ? "default" : "executionMode" in input ? input.executionMode : "default", - }); + })); await checkpoint(); } settleStartupDecision();