diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 6811cdfd7bc..0ff940f210e 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -8896,6 +8896,105 @@ describe("executeNativeSession recovery", () => { ).toEqual(["run.result.accepted", "run.terminal"]); }); + it("proves it still owns a recovered session right before starting a turn in it", async () => { + // Staging run cd2bf273: a turn nobody can attribute appeared in the same + // provider thread across server swaps. The recovery checkpoint is fenced + // (controller boot, generation, lease), but preparing the turn's input can + // take a while and another generation may take the run over meanwhile. + // The fenced checkpoint is repeated right before the submit, so the + // replaced server submits nothing. + const checkpoint: PersistedNativeSession = { + backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", + activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], lineage: [], + }; + const startTurn = vi.fn(async () => ({ turnId: "late-turn" })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + events: () => (async function* () {})(), + startTurn, + async result() { return null; }, + async snapshot() { return structuredClone(checkpoint); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { kind: "mock", name: "recovery-backend", version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + let takenOver = false; + const checkpointSession = vi.fn(async () => { if (takenOver) throw new Error("native_runner_checkpoint_owner_lost"); }); + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + checkpointSession, + async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 0, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun: vi.fn(async () => undefined), + }; + await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + // Another server generation claims the run while this one prepares the turn. + prepareWakeAttachmentPrompt: async () => { takenOver = true; return ""; } }).catch(() => undefined); + expect(takenOver).toBe(true); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it.each([ + { label: "at startup", interruptedAtStartup: true }, + { label: "when the lost turn's terminal arrives", interruptedAtStartup: false }, + ])("proves it still owns a recovered session right before a restart continuation $label", async ({ interruptedAtStartup }) => { + // Same takeover window as the startup turn, on the restart-continuation + // path: steered messages and attachments are prepared before the submit. + const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; + const checkpoint: PersistedNativeSession = { + backendKind: "runner", driverKind: "acpx_runtime", sessionId: "record-recovery", identity, + providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", + terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], + }; + const interrupted: PersistedNativeSession = { + ...checkpoint, activeTurnId: null, + terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], + }; + const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, + async *events() { + if (!interruptedAtStartup) yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); + }, + startTurn, + async result() { return null; }, + async snapshot() { return structuredClone(interrupted); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, + async openSession() { throw new Error("must preserve the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + let takenOver = false; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() { if (takenOver) throw new Error("native_runner_checkpoint_owner_lost"); }, + async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + const prepareWakeAttachmentPrompt = vi.fn(async () => { takenOver = true; return ""; }); + await executeNativeSession({ + input, backend, controlPlane: port, persistedSession: interruptedAtStartup ? interrupted : checkpoint, + runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, + // Another server generation claims the run while this one prepares the continuation. + prepareWakeAttachmentPrompt, + }).catch(() => undefined); + expect(prepareWakeAttachmentPrompt).toHaveBeenCalledOnce(); + expect(startTurn).not.toHaveBeenCalled(); + }); + it("resolves a checkpointed result-less disposition without resubmitting when its terminal event is missing", async () => { const workProposal: PrpEvent = { ...runnerEvent(1, "run.result.proposed", result), diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index f3bded13bf3..3496497b61d 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -2867,6 +2867,9 @@ export async function executeNativeSession( signal?.throwIfAborted(); const attachmentPrompt = await options.prepareWakeAttachmentPrompt?.(); signal?.throwIfAborted(); + // The fenced checkpoint proves this controller still owns the run + // right before the submit; input preparation above may have taken a while. + await checkpoint(signal); const started = await session.startTurn(restartContinuation(steeredMessages, attachmentPrompt)); restartContinuationTurnId = started.turnId; if (started.borrowedDispositionMarker === true) borrowedContinuationTurnId = started.turnId; @@ -3208,6 +3211,10 @@ export async function executeNativeSession( modelEnvelope.events = `${modelEnvelope.events}\n\n${supplemental}`; } } + // A recovered session is shared provider state: prove ownership with + // the fenced checkpoint right before the submit. A new session's + // normal path adds nothing before turn.start. + if (recovered) await checkpoint(); await session.startTurn({ message: { role: "user", text: JSON.stringify(modelEnvelope) }, ...(recovered && modelEnvelope.schema === "paperclip.native-continuation.v1"