import { validatePrpStructuredRunResult } from "./protocol/replay-contract.js"; import { createProviderFailure } from "./drivers/provider-failure.js"; import { createHash } from "node:crypto"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { createCapabilityRunnerdCodexTransport } from "./live/runnerd-codex-transport.js"; import { defaultCapabilityRunnerdBinary, fakeCodex, fakeCodexArgs } from "../test/support/runnerd-transport.js"; import { join } from "node:path"; import { resolveQualifiedAcpxProfile } from "./drivers/acpx/qualified-profiles.js"; import { describe, expect, it, vi } from "vitest"; import { createCodexNativeSessionBackend, createRunnerdNativeSessionBackend } from "./backends/codex-native-backend.js"; import { FakeCodexTransport, WORKSPACE } from "./drivers/codex/codex-app-server-driver.test-support.js"; import type { ControlPlanePort } from "./contracts/control-plane-port.js"; import type { NativeExecutionInputV1, NativeExecutionInputV5 } from "./contracts/native-execution.js"; import type { NativeRunIdentity } from "./contracts/types.js"; import type { NativeSession, NativeSessionBackend, NativeSessionRecoveryResult, PersistedNativeSession, } from "./contracts/native-session-backend.js"; import { NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeSessionCloseUnrecoverableError, NativeSessionCleanupQuarantinedError, NativeSessionProtocolIntegrityError, nativeRestartContinuationInFlight, nativeRestartInterruptedTurnId, } from "./contracts/native-session-backend.js"; import { PROVIDER_ATTACH_REFUSED, PROVIDER_SIDE_EFFECT_PENDING, PROVIDER_STATE_INCOMPATIBLE } from "./contracts/harness-driver.js"; import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState, } from "./protocol/replay-contract.js"; import { NATIVE_RUNTIME_ASSET_SCHEMA, PAPERCLIP_EXECUTION_PROMPT, PAPERCLIP_EXECUTION_PROMPT_REVISION, canonicalNativeRuntimeContextDigest, nativeRuntimePromptDigest, type NativeRuntimeContextSnapshot, } from "./contracts/runtime-context.js"; import { executeNativeSession, completeTerminatedRemoteNativeSessionCleanup, completeTerminatedLocalNativeSessionCleanup, nativeSessionCleanupBlock, onNativeSessionCleanupQuarantined, type ExecuteNativeSessionOptions, } from "./native-session-runtime.js"; const identity = { runId: "run-recovery", sessionId: "session-recovery", companyId: "company-recovery", issueId: "issue-recovery", agentId: "agent-recovery", }; const result: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Recovered native work completed.", completionClaim: { contractRevision: "1", objectiveSatisfied: true, criteria: [ { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, ], remainingWork: [], }, evidence: [], verification: [{ commandOrCheck: "recovery", status: "passed" }], attentionRequests: [], artifacts: [], }; const terminal: PrpTerminalState = { schema: "paperclip.prp.terminal.v1", turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "done", }; const yieldedResult: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Waiting for the requested response.", completionClaim: { contractRevision: "1", objectiveSatisfied: false, criteria: [ { criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"], }, ], remainingWork: [ { description: "Resume after the response.", blocksCompletion: true }, ], }, evidence: [{ ref: "interaction:pending" }], verification: [], attentionRequests: [], artifacts: [{ kind: "issue_thread_interaction", ref: "interaction:pending" }], continuation: { kind: "response_wake", summary: "Resume from the answer.", idempotencyKey: "interaction-response:pending", }, }; const input: NativeExecutionInputV1 = { schema: "paperclip.native-execution-input.v1", binding: { companyId: identity.companyId, runId: identity.runId, issueId: identity.issueId, agentId: identity.agentId, executionWorkspaceId: "workspace-recovery", }, task: { identifier: "PAP-RECOVERY", title: "Recover native work", description: null, prompt: "# PAP-RECOVERY: Recover native work", workMode: "standard", }, workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null, }, session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, }, provider: { kind: "codex", model: null }, completionContract: { id: "contract-recovery", sha256: "contract-recovery-sha", schemaVersion: "paperclip.completion-contract.v1", contract: { revision: "1", objective: "Recover native work", criteria: [{ id: "objective", requirement: "Complete after recovery" }], }, }, interactionResponses: [], credentialBindings: [], }; type RuntimeContextFields = Omit; function preparedInput(): NativeExecutionInputV5 { const digest = "0".repeat(64); const context = { prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() }, instructions: { entryPath: "AGENTS.md", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/instructions", fileCount: 1, totalBytes: 1 } }, skills: [], mcp: { assignmentSetId: "none", digest, bindingId: null }, } satisfies RuntimeContextFields; const requirement = input.completionContract.contract.criteria[0]!.requirement; const prompt = `${input.task.prompt}\n\n${requirement}`; return { ...input, schema: "paperclip.native-execution-input.v5", executionMode: "default", planningContext: null, task: { ...input.task, description: requirement, prompt }, provider: { kind: "codex", model: null, approvalPolicy: "never" }, runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }, completionSources: { promptSha256: createHash("sha256").update(prompt).digest("hex"), contractRevision: input.completionContract.contract.revision, criteria: [{ id: "objective", source: { kind: "description", id: identity.issueId, revision: createHash("sha256").update(requirement).digest("hex") } }], }, }; } function controlEvent( sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record, ): PrpEvent { return { schema: "paperclip.prp.event.v1", sourceEventId: `control-recovery:${identity.runId}:${sourceSeq}`, sourceSeq, sourceInstanceId: "control-recovery", sourceKind: "control_plane", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-recovery", eventType, schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:00.000Z", payload, }; } function canonicalTestJson(value: unknown): string { if (Array.isArray(value)) { return `[${value.map(canonicalTestJson).join(",")}]`; } if (typeof value === "object" && value !== null) { const record = value as Record; return `{${Object.keys(record) .sort() .map((key) => `${JSON.stringify(key)}:${canonicalTestJson(record[key])}`) .join(",")}}`; } return JSON.stringify(value) ?? "undefined"; } // A restart can kill processes that earlier turns started. The continuation // must point the agent at workspace outputs instead of the lost process. const RESTART_LOST_PROCESS_GUIDANCE = "may no longer be running"; function expectRestartContinuationGuidance(text: string) { const events = String(JSON.parse(text).events); expect(events).toContain("This is a resumed session. The process running the previous turn was lost"); expect(events).not.toContain("control-plane restart"); expect(events).toContain("Shell commands and background processes started in earlier turns " + RESTART_LOST_PROCESS_GUIDANCE); expect(events).toContain("check the workspace for the outputs or artifacts"); expect(events).toContain("If the expected output is missing, re-run the command."); expect(events).toContain("Do not block only because such a process or its tool session cannot be found or inspected."); expect(events).not.toContain("blindly rerun"); expect(events).toContain("Reconcile any unfinished tool or command"); } function runnerEvent( sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record = {}, ): PrpEvent { return { schema: "paperclip.prp.event.v1", sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, sourceSeq, sourceInstanceId: "runner-recovery", sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-recovery", eventType, schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:00.000Z", payload, }; } function highestContiguous(events: PrpEvent[]): number { const sequences = new Set(events.map((event) => event.sourceSeq)); let cursor = 0; while (sequences.has(cursor + 1)) cursor += 1; return cursor; } describe("executeNativeSession recovery", () => { it("completes Cursor Cloud semantic planning through its default-only harness without losing the pinned revision", async () => { await writeFile(join(WORKSPACE, "cloud-plan-instructions.md"), "Make the plan only. Do not implement it.\n"); const prepared = preparedInput(); const context = { ...prepared.runtimeContext, instructions: { ...prepared.runtimeContext.instructions, entryPath: "cloud-plan-instructions.md", bundle: { ...prepared.runtimeContext.instructions.bundle, rootPath: WORKSPACE } } }; const planningContext = { documentId: "plan-document", baseRevisionId: "plan-revision-3", baseRevisionNumber: 3, markdown: "# Existing canonical plan", sha256: createHash("sha256").update("# Existing canonical plan").digest("hex"), reviewContext: { feedback: "Keep the requested scope" } }; const executionInput: NativeExecutionInputV5 = { ...prepared, executionMode: "plan", planningContext, task: { ...prepared.task, workMode: "planning" }, runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }, workspace: { ...input.workspace, cwd: WORKSPACE }, session: { ...input.session, driverKind: "cursor_cloud_api", lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, provider: { kind: "cursor_cloud", model: "explicit-cloud-model", cursorCloud: { schema: "paperclip.cursor-cloud.config.v1", repos: [{ url: "https://github.com/example/fixture" }] } }, }; const planResult: PrpStructuredRunResult = { ...result, summary: "The requested plan is ready.", artifacts: [{ kind: "native_provider_plan", ref: "Keep the requested scope." }] }; const transport = new FakeCodexTransport(); transport.confirmCollaborationMode = false; const backend = createRunnerdNativeSessionBackend(executionInput, { runnerInstanceId: "runner-recovery", transportFactory: () => transport }); const committed: PrpEvent[] = []; const completeRun = vi.fn(async () => undefined); const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { committed.push(structuredClone(event)); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents(query) { const events = committed.filter(event => event.sourceInstanceId === query.sourceInstanceId); return { events: events.filter(event => event.sourceSeq > query.afterSourceSeq).slice(0, query.limit), highestContiguousSourceSeq: highestContiguous(events) }; }, completeRun, }; const abort = new AbortController(); const completion = executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", signal: abort.signal }); let failure: unknown; void completion.catch(error => { failure = error; }); try { await vi.waitFor(() => expect(transport.calls.some(call => call.method === "turn/start") || failure).toBeTruthy()); if (failure) await completion; const started = transport.calls.find(call => call.method === "turn/start")!; const envelope = JSON.parse((started.params.input as Array<{ text: string }>)[0]!.text); expect(envelope).toMatchObject({ executionMode: "plan", task: { workMode: "planning" }, planningContext }); expect(transport.calls.find(call => call.method === "thread/start")?.params.baseInstructions).toContain("Make the plan only. Do not implement it."); expect(started.params).not.toHaveProperty("collaborationMode"); transport.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); transport.push("paperclip/runResult", { threadId: "thread-1", turnId: "turn-1", result: planResult }); transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] } }); await expect(completion).resolves.toMatchObject({ result: planResult, terminal: { runTerminalState: "succeeded" } }); expect(completeRun).toHaveBeenCalledOnce(); expect(committed.some(event => event.eventType === "run.result.accepted")).toBe(true); expect(executionInput.planningContext).toEqual(planningContext); } finally { abort.abort(); await completion.catch(() => undefined); } }); /** A batching port whose queued deltas commit only when flushed: it has no timer. */ const batchingDeltaFixture = (events: () => AsyncGenerator, commitDelta: (sourceSeq: number) => Promise = async () => {}) => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, events, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, semanticResult: null, terminal: null, terminalTurns: [], pendingRuntimeRequests: [], lineage: [] }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "delta-fixture", version: "1", capabilities, runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, async openSession() { return session; }, }; const log: string[] = []; let queued: Array<() => void> = []; const port: ControlPlanePort = { pipelinesStreamingEvents: true, flushStreamingEvents() { log.push("flush"); const flushed = queued; queued = []; for (const commit of flushed) commit(); }, async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent, options) { if (event.sourceKind !== "runner") return { cursor: 0, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; log.push(`append:${event.sourceSeq}`); if (event.eventType === "item.delta") { expect(options?.streaming).toBe(true); await new Promise(resolve => queued.push(resolve)); await commitDelta(event.sourceSeq); log.push(`committed:${event.sourceSeq}`); } return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; return { backend, port, log }; }; const streamedDelta = (sourceSeq: number) => ({ ...runnerEvent(sourceSeq, "item.delta", { kind: "agentMessage", text: `chunk ${sourceSeq}` }), itemId: "message-1", priority: 1 as const }); it("starts text delta appends without waiting and flushes them before any other event", async () => { const f = batchingDeltaFixture(async function* () { for (const sourceSeq of [1, 2, 3]) yield streamedDelta(sourceSeq); yield runnerEvent(4, "run.result.proposed", result); yield runnerEvent(5, "turn.completed"); }); // The port has no timer: the turn finishes only if the next event flushes the queued deltas. const execution = await executeNativeSession({ input: preparedInput(), backend: f.backend, controlPlane: f.port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); expect(f.log.slice(0, 5)).toEqual(["append:1", "append:2", "append:3", "flush", "committed:1"]); expect(f.log.indexOf("append:4")).toBeGreaterThan(Math.max(...[1, 2, 3].map(seq => f.log.indexOf(`committed:${seq}`)))); expect(execution).toMatchObject({ nativeEventCount: 7, highestContiguousSourceSeq: 5 }); }); it("joins pending delta appends before a failed turn settles", async () => { let releaseCommit = () => {}; const commitReleased = new Promise(resolve => { releaseCommit = resolve; }); const f = batchingDeltaFixture(async function* () { yield streamedDelta(1); throw new Error("provider stream broke"); }, () => commitReleased); let settled = false; const execution = executeNativeSession({ input: preparedInput(), backend: f.backend, controlPlane: f.port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }).finally(() => { settled = true; f.log.push("settled"); }); void execution.catch(() => undefined); await vi.waitFor(() => expect(f.log).toContain("flush")); // The delta's commit still runs, so the turn has not settled. expect(settled).toBe(false); releaseCommit(); await expect(execution).rejects.toThrow("provider stream broke"); expect(f.log.indexOf("committed:1")).toBeLessThan(f.log.indexOf("settled")); }); it("fails the turn within the cleanup bound when a pending delta append never settles", async () => { const f = batchingDeltaFixture(async function* () { yield streamedDelta(1); throw new Error("provider stream broke"); }, () => new Promise(() => {})); // No turn timeout: only the bounded cleanup can end this turn. await expect(executeNativeSession({ input: preparedInput(), backend: f.backend, controlPlane: f.port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow("provider stream broke"); expect(f.log).not.toContain("committed:1"); }); it.each([false, true].flatMap(recovered => [ { recovered, cwd: "/home/agent" }, { recovered, cwd: "/projects/repository/worktrees/task" }, { recovered, cwd: undefined }, ]))("uses the admitted session cwd in the task envelope (recovered=$recovered, cwd=$cwd)", async ({ recovered, cwd }) => { const executionInput = preparedInput(); const originalWorkspace = structuredClone(executionInput.workspace); const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const snapshot: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", workingDirectory: cwd, cursor: null, activeTurnId: null, semanticResult: null, terminal: null, terminalTurns: [], pendingRuntimeRequests: [], lineage: [], }; const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "run.result.proposed", result); yield runnerEvent(2, "turn.completed"); }, startTurn, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return structuredClone(snapshot); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "workspace-fixture", version: "1", capabilities, runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, async openSession() { return session; }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async loadSessionCheckpoint() { return recovered ? structuredClone(snapshot) : null; }, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); expect(JSON.parse(startTurn.mock.calls[0]![0].message.text).workspace).toEqual({ cwd: cwd ?? originalWorkspace.cwd }); // Fresh and ordinary resumed turns do not carry restart-continuation guidance. expect(startTurn.mock.calls[0]![0].message.text).not.toContain(RESTART_LOST_PROCESS_GUIDANCE); expect(executionInput.workspace).toEqual(originalWorkspace); }); it("settles a restart continuation that ended without a result through one disposition turn (Codex driver)", async () => { const lost = { code: "provider_turn_lost_on_restore", recoverable: true }; const originalTransport = new FakeCodexTransport(); const recoveredTransport = new FakeCodexTransport(); const transports = [originalTransport, recoveredTransport]; const executionInput = { ...input, workspace: { ...input.workspace, cwd: WORKSPACE } }; const backend = createCodexNativeSessionBackend(executionInput, { runnerInstanceId: "runner-recovery", transportFactory: () => transports.shift()!, }); const original = await backend.openSession({ identity, workingDirectory: WORKSPACE }); await original.startTurn({ message: { role: "user", text: "Finish the assigned review." } }); const checkpoint = await original.snapshot(); expect(checkpoint).toMatchObject({ activeTurnId: "turn-1", semanticResult: null }); await original.close({ reason: "controller lost during the turn" }); recoveredTransport.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, turns: [{ id: "turn-1", status: "failed", error: lost, items: [] }] } }; const turnStarts = () => recoveredTransport.calls.filter(call => call.method === "turn/start"); Object.defineProperty(recoveredTransport, "turnStartResponse", { get: () => Promise.resolve({ turn: { id: `turn-${turnStarts().length + 1}`, status: "inProgress", items: [] } }), }); const checkpoints: PersistedNativeSession[] = []; const completeRun = vi.fn(async () => undefined); const port: ControlPlanePort = { async openRun() {}, async checkpointSession(value) { checkpoints.push(structuredClone(value)); }, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; const prepareWakeAttachmentPrompt = vi.fn(async () => "Read the fresh attachment at /run/attachments/restarted.png."); const execution = executeNativeSession({ input: executionInput, backend, controlPlane: port, persistedSession: checkpoint, prepareWakeAttachmentPrompt, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, }); void execution.catch(() => undefined); await vi.waitFor(() => expect(turnStarts()).toHaveLength(1)); expect(prepareWakeAttachmentPrompt).toHaveBeenCalledOnce(); expect(JSON.stringify(turnStarts()[0]!.params)).toContain("/run/attachments/restarted.png"); // Like the RC-60 Claude turns: the continuation ends saying it will wait // for a background command, with no paperclip_finish. recoveredTransport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-2", status: "completed", items: [] } }); await vi.waitFor(() => expect(turnStarts()).toHaveLength(2)); const disposition = JSON.stringify(turnStarts()[1]!.params); expect(disposition).toContain("Paperclip semantic-result recovery after a restart continuation"); recoveredTransport.push("paperclip/runResult", { threadId: "thread-1", turnId: "turn-3", result }); recoveredTransport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-3", status: "completed", items: [] } }); await expect(execution).resolves.toMatchObject({ result, turnId: "turn-3" }); expect(completeRun).toHaveBeenCalledOnce(); expect(turnStarts()).toHaveLength(2); expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-3" }); }); it.each([ "completed", "pending", "duplicate", "conflicting", "other turn", "newer turn", "wrong session", "invalid", "restore-lost live", "restore-lost read", ])("recovers a journaled completion proposal after restart: %s", async (scenario) => { const originalTransport = new FakeCodexTransport(); const recoveredTransport = new FakeCodexTransport(); const transports = [originalTransport, recoveredTransport]; const executionInput = { ...input, workspace: { ...input.workspace, cwd: WORKSPACE } }; const backend = createCodexNativeSessionBackend(executionInput, { runnerInstanceId: "runner-recovery", transportFactory: () => transports.shift()!, }); const original = await backend.openSession({ identity, workingDirectory: WORKSPACE }); const journal: PrpEvent[] = []; const iterator = original.events()[Symbol.asyncIterator](); const drainThrough = async (eventType: string) => { while (true) { const next = await iterator.next(); if (next.done) throw new Error(`stream closed before ${eventType}`); journal.push(structuredClone(next.value)); if (next.value.eventType === eventType) return; } }; await original.startTurn({ message: { role: "user", text: "Finish the assigned review." } }); originalTransport.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" }, }); await drainThrough("turn.started"); const checkpoint = await original.snapshot(); expect(checkpoint.semanticResult).toBeNull(); originalTransport.push("paperclip/runResult", { threadId: "thread-1", turnId: "turn-1", result, }); await drainThrough("run.result.proposed"); expect((await original.snapshot()).semanticResult).toEqual(result); const proposal = journal.at(-1)!; if (scenario === "other turn") proposal.turnId = "prior-turn"; if (scenario === "wrong session") proposal.normalizedSessionId = "other-session"; if (scenario === "invalid") proposal.payload = { reportedWorkDisposition: "done" }; if (["duplicate", "conflicting", "newer turn"].includes(scenario)) { journal.push({ ...proposal, sourceSeq: proposal.sourceSeq + 1, sourceEventId: `${proposal.sourceEventId}:next`, ...(scenario === "newer turn" ? { eventType: "turn.started" as const, turnId: "newer-turn", payload: {} } : { payload: scenario === "conflicting" ? { ...result, summary: "A different result." } : result }), }); } const persistedProposalCount = journal.filter(event => event.eventType === "run.result.proposed").length; await original.close({ reason: "controller disconnected after persisting the proposal" }); recoveredTransport.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, turns: [{ id: "turn-1", ...(scenario === "restore-lost read" ? { status: "failed", error: { code: "provider_turn_lost_on_restore", recoverable: true } } : { status: ["pending", "restore-lost live"].includes(scenario) ? "inProgress" : "completed" }), items: [ { id: "answer", type: "agentMessage", text: "The review was published." }, ] }], } }; const completeRun = vi.fn(async () => undefined); const checkpoints: PersistedNativeSession[] = []; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession(value) { checkpoints.push(structuredClone(value)); }, async appendEvent(event: PrpEvent) { const existing = journal.find(value => value.sourceEventId === event.sourceEventId); if (existing) expect(event).toEqual(existing); else journal.push(structuredClone(event as PrpEvent)); return { cursor: journal.length, highestContiguousSourceSeq: event.sourceSeq, disposition: existing ? "duplicate" : "committed" }; }, async replayEvents(query) { const events = journal.filter(event => event.sourceInstanceId === query.sourceInstanceId); return { events: structuredClone(events.filter(event => event.sourceSeq > query.afterSourceSeq).slice(0, query.limit)), highestContiguousSourceSeq: highestContiguous(events), }; }, completeRun, }; const execution = executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); if (["conflicting", "wrong session", "invalid"].includes(scenario)) { await expect(execution).rejects.toBeInstanceOf(NativeSessionProtocolIntegrityError); expect(completeRun).not.toHaveBeenCalled(); expect(recoveredTransport.calls).toEqual([]); return; } if (["other turn", "newer turn"].includes(scenario)) { await expect(execution).rejects.toThrow("native_finalization_missing"); expect(completeRun).not.toHaveBeenCalled(); expect(recoveredTransport.calls.filter(call => call.method === "turn/start")).toEqual([]); return; } if (scenario === "pending" || scenario === "restore-lost live") { await vi.waitFor(() => expect(checkpoints[0]?.semanticResult).toEqual(result)); expect(completeRun).not.toHaveBeenCalled(); // ACPX reports a turn lost on restore as a live terminal once the // restored runner reopens the conversation. recoveredTransport.push("turn/completed", { threadId: "thread-1", turn: scenario === "pending" ? { id: "turn-1", status: "completed", items: [] } : { id: "turn-1", status: "failed", error: { code: "provider_turn_lost_on_restore", recoverable: true }, items: [] }, }); } await expect(execution).resolves.toMatchObject({ result, turnId: "turn-1", terminal }); expect(completeRun).toHaveBeenCalledExactlyOnceWith( expect.objectContaining({ result, turnId: "turn-1" }), expect.anything(), ); expect(recoveredTransport.calls.filter(call => call.method === "turn/start")).toEqual([]); expect(journal.filter(event => event.eventType === "run.result.proposed")).toHaveLength(persistedProposalCount); expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, activeTurnId: null }); }); it.each([ { label: "continues a restart-lost Codex turn", allowed: true, error: { code: "provider_turn_lost_on_restore", recoverable: true } }, { label: "requires the control plane restart claim", allowed: false, error: { code: "provider_turn_lost_on_restore", recoverable: true } }, { label: "preserves real provider errors", allowed: true, error: { code: "model_not_found", recoverable: false } }, { label: "does not reopen historical error-null failures", allowed: true, error: null }, ])("$label without replaying the task", async ({ label, allowed, error }) => { const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "codex_app_server", sessionId: "thread-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const failure = error; const interrupted: PersistedNativeSession = { ...checkpoint, activeTurnId: null, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], }; const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); if (!startTurn.mock.calls.length) return; yield { ...runnerEvent(2, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; }, startTurn, async result() { return startTurn.mock.calls.length ? { result, terminal, turnId: "turn-continuation" } : null; }, async snapshot() { return structuredClone(interrupted); }, async close() {}, }; const events: PrpEvent[] = []; const checkpoints: PersistedNativeSession[] = []; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, async appendEvent(event: PrpEvent) { events.push(event); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: allowed, }); if (label !== "continues a restart-lost Codex turn") { await expect(execution).rejects.toMatchObject({ code: "native_provider_terminal_failed" }); expect(startTurn).not.toHaveBeenCalled(); return; } await expect(execution).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } }); expect(startTurn).toHaveBeenCalledOnce(); const submitted = startTurn.mock.calls[0]![0] as Parameters[0]; expect(submitted.continuation).toBe(true); expectRestartContinuationGuidance(submitted.message.text); expect(submitted.message.text).not.toContain(input.task.prompt); expect(events.filter(event => event.eventType === "turn.failed")).toHaveLength(1); expect(events.filter(event => event.eventType === "run.terminal")).toHaveLength(1); expect(checkpoints.some(snapshot => snapshot.terminalTurns?.some(turn => turn.turnId === "turn-recovery"))).toBe(true); }); it.each([ { label: "after the continuation starts", staleFirst: false, staleTurnId: "turn-unrecorded-continuation" }, { label: "before the continuation starts", staleFirst: true, staleTurnId: "turn-unrecorded-continuation" }, { label: "as the checkpointed interrupted turn before the continuation starts", staleFirst: true, staleTurnId: "turn-recovery" }, ])("keeps the restart continuation when the restored runner reports an earlier unrecorded continuation lost $label", async ({ staleFirst, staleTurnId }) => { let staleAppended!: () => void; const staleTerminal = new Promise(resolve => { staleAppended = resolve; }); const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const interrupted: PersistedNativeSession = { backendKind: "runner", driverKind: "codex_app_server", sessionId: "thread-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: null, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const startTurn = vi.fn(async () => { if (staleFirst) await staleTerminal; return { turnId: "turn-continuation" }; }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield { ...runnerEvent(1, "turn.failed", { status: "failed", error: failure }), turnId: staleTurnId }; yield { ...runnerEvent(2, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; }, startTurn, async result() { return startTurn.mock.calls.length ? { result, terminal, turnId: "turn-continuation" } : null; }, async snapshot() { return structuredClone(interrupted); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(event); if (event.eventType === "turn.failed") staleAppended(); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: interrupted, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, })).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } }); expect(startTurn).toHaveBeenCalledOnce(); expect((startTurn.mock.calls[0]![0] as Parameters[0]).continuation).toBe(true); expect(events.filter(event => event.eventType === "turn.failed").map(event => event.turnId)).toEqual([staleTurnId]); }); it("recovers a restart continuation whose checkpoint still carries the lost turn's failed terminal", async () => { const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "codex_app_server", sessionId: "thread-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-continuation", terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "failed", turnTerminalState: "failed", reportedWorkDisposition: "yielded" }, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], semanticResult: null, pendingRuntimeRequests: [], lineage: [], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-continuation", }; const startTurn = vi.fn(async () => ({ turnId: "turn-unexpected" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield { ...runnerEvent(1, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; }, startTurn, async result() { return { result, terminal, turnId: "turn-continuation" }; }, async snapshot() { return structuredClone(checkpoint); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, })).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } }); expect(startTurn).not.toHaveBeenCalled(); expect(nativeRestartContinuationInFlight(checkpoint)).toBe(true); expect(nativeRestartContinuationInFlight({ ...checkpoint, activeTurnId: "turn-recovery" })).toBe(false); expect(nativeRestartContinuationInFlight({ ...checkpoint, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: { code: "model_not_found", recoverable: false }, result: null }) }] })) .toBe(false); }); it.each([ { label: "after it starts", proposedFirst: false }, { label: "after the continuation proposed its result", proposedFirst: true }, ])("keeps the restart continuation when the lost turn's terminal is delivered again $label", async ({ proposedFirst }) => { const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "acpx_runtime", sessionId: "record-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const interrupted: PersistedNativeSession = { ...checkpoint, activeTurnId: null, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], }; const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); if (!startTurn.mock.calls.length) return; if (proposedFirst) yield { ...runnerEvent(2, "run.result.proposed", result), turnId: "turn-continuation" }; yield runnerEvent(3, "turn.failed", { status: "failed", error: failure }); yield { ...runnerEvent(4, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; }, startTurn, async result() { return startTurn.mock.calls.length ? { result, terminal, turnId: "turn-continuation" } : null; }, async snapshot() { return structuredClone(interrupted); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, })).resolves.toMatchObject({ result, turnId: "turn-continuation", terminal: { runTerminalState: "succeeded" } }); expect(startTurn).toHaveBeenCalledOnce(); }); it.each(["acpx_runtime", "opencode_server"] as const)("continues a restart-lost %s turn once in the same conversation", async (driverKind) => { const qualified = resolveQualifiedAcpxProfile("claude", "claude-sonnet-5"); const acpxInput: NativeExecutionInputV1 = (driverKind === "acpx_runtime" ? { ...input, session: { ...input.session, driverKind }, provider: { kind: "acpx", agent: "claude", model: "claude-sonnet-5", permissionPolicy: "interactive", profile: { driverKind: qualified.driverKind, protocolVersion: qualified.protocolVersion, acpxVersion: qualified.acpxVersion, agent: qualified.agent, agentProfileVersion: qualified.agentProfileVersion, agentServerPackage: qualified.agentServerPackage, agentServerVersion: qualified.agentServerVersion, agentRuntimePackage: qualified.agentRuntimePackage, agentRuntimeVersion: qualified.agentRuntimeVersion, commandDigest: qualified.commandDigest, } }, } : { ...input, session: { ...input.session, driverKind }, provider: { kind: "opencode", model: "openrouter/deepseek/deepseek-v4-flash-0731" }, }) as NativeExecutionInputV1; const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind, sessionId: "record-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const interrupted: PersistedNativeSession = { ...checkpoint, activeTurnId: null, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], }; const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); if (!startTurn.mock.calls.length) return; yield { ...runnerEvent(2, "turn.completed", { status: "completed" }), turnId: "turn-continuation" }; }, startTurn, async result() { return startTurn.mock.calls.length ? { result, terminal, turnId: "turn-continuation" } : null; }, async snapshot() { return structuredClone(interrupted); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input: acpxInput, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, })).resolves.toMatchObject({ result, turnId: "turn-continuation" }); expect(startTurn).toHaveBeenCalledOnce(); const submitted = startTurn.mock.calls[0]![0] as Parameters[0]; expect(submitted.continuation).toBe(true); expectRestartContinuationGuidance(submitted.message.text); expect(nativeRestartInterruptedTurnId(interrupted)).toBe("turn-recovery"); }); it.each([ { label: "asks once for the disposition when a restart continuation ends without a result", driverReArms: true }, { label: "fails closed when the driver keeps the continuation's disposition marker", driverReArms: false }, ])("$label", async ({ driverReArms }) => { const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "acpx_runtime", sessionId: "record-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const lostTerminal = { turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }; const completedTerminal = (turnId: string) => ({ turnId, fingerprint: JSON.stringify({ terminalState: "completed", error: null, result: null }) }); let starts = 0; const startTurn = vi.fn(async () => ++starts === 1 ? { turnId: "turn-continuation", borrowedDispositionMarker: true as const } : { turnId: "turn-disposition" }); let continuationDone = false; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); if (!startTurn.mock.calls.length) return; // The agent ends the continuation saying it will wait for a // background command, without paperclip_finish. continuationDone = true; yield { ...runnerEvent(2, "turn.completed", { status: "completed", error: null }), turnId: "turn-continuation" }; if (startTurn.mock.calls.length < 2) return; yield { ...runnerEvent(3, "turn.completed", { status: "completed", error: null }), turnId: "turn-disposition" }; }, startTurn, async result() { return startTurn.mock.calls.length >= 2 ? { result, terminal, turnId: "turn-disposition" } : null; }, async snapshot() { const calls = startTurn.mock.calls.length; if (calls === 0) return { ...checkpoint, activeTurnId: null, terminalTurns: [lostTerminal] }; // The continuation borrows the driver's one-shot marker while it runs. if (calls === 1 && !continuationDone) return { ...checkpoint, activeTurnId: "turn-continuation", terminalTurns: [lostTerminal], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-continuation" }; if (calls === 1) return { ...checkpoint, activeTurnId: null, terminalTurns: [lostTerminal, completedTerminal("turn-continuation")], dispositionOnlyRecoveryConsumed: !driverReArms, dispositionOnlyRecoveryTurnId: driverReArms ? null : "turn-continuation" }; return { ...checkpoint, activeTurnId: null, terminalTurns: [lostTerminal, completedTerminal("turn-continuation"), completedTerminal("turn-disposition")], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-disposition" }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, }); if (!driverReArms) { await expect(execution).rejects.toThrow("native_finalization_missing: session returned no semantic result"); expect(startTurn).toHaveBeenCalledOnce(); return; } await expect(execution).resolves.toMatchObject({ result, turnId: "turn-disposition" }); expect(startTurn).toHaveBeenCalledTimes(2); const continuation = startTurn.mock.calls[0]![0] as Parameters[0]; expect(continuation.message.text).toContain("paperclip_finish"); expect(continuation.message.text).toContain("background"); const disposition = startTurn.mock.calls[1]![0] as Parameters[0]; expect(disposition.continuation).toBeUndefined(); expect(disposition.message.text).toContain("Paperclip semantic-result recovery"); expect(disposition.message.text).toContain("paperclip_finish or paperclip_block"); expect(disposition.message.text).not.toContain(input.task.prompt); }); it("asks for the disposition when the continuation completes before its start checkpoint settles", async () => { // The checkpoint already records the lost turn, so the continuation starts // while the event consumer is running. The provider completes it at once: // the driver has returned the marker before any post-start snapshot, and // the completion reaches the consumer while the start checkpoint is held. const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const lostTerminal = { turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }; const completedTerminal = (turnId: string) => ({ turnId, fingerprint: JSON.stringify({ terminalState: "completed", error: null, result: null }) }); const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "acpx_runtime", sessionId: "record-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: null, terminalTurns: [lostTerminal], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; let releaseStartCheckpoint!: () => void; const startCheckpointHeld = new Promise(resolve => { releaseStartCheckpoint = resolve; }); let continuationCompletionSeen = false; let starts = 0; const startTurn = vi.fn(async () => ++starts === 1 ? { turnId: "turn-continuation", borrowedDispositionMarker: true as const } : { turnId: "turn-disposition" }); let wakeEvents!: () => void; let eventsWoken = new Promise(resolve => { wakeEvents = resolve; }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { while (startTurn.mock.calls.length < 1) { await eventsWoken; eventsWoken = new Promise(resolve => { wakeEvents = resolve; }); } continuationCompletionSeen = true; yield { ...runnerEvent(2, "turn.completed", { status: "completed", error: null }), turnId: "turn-continuation" }; while (startTurn.mock.calls.length < 2) { await eventsWoken; eventsWoken = new Promise(resolve => { wakeEvents = resolve; }); } yield { ...runnerEvent(3, "turn.completed", { status: "completed", error: null }), turnId: "turn-disposition" }; }, startTurn: async (value) => { const started = await startTurn(value); wakeEvents(); return started; }, async result() { return startTurn.mock.calls.length >= 2 ? { result, terminal, turnId: "turn-disposition" } : null; }, async snapshot() { const calls = startTurn.mock.calls.length; if (calls === 0) return structuredClone(checkpoint); if (calls === 1) return { ...checkpoint, terminalTurns: [lostTerminal, completedTerminal("turn-continuation")], dispositionOnlyRecoveryConsumed: false, dispositionOnlyRecoveryTurnId: null }; return { ...checkpoint, terminalTurns: [lostTerminal, completedTerminal("turn-continuation"), completedTerminal("turn-disposition")], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-disposition" }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const persisted: PersistedNativeSession[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession(snapshot) { // Hold the continuation's start checkpoint until the consumer has // already received the continuation's completion. if (startTurn.mock.calls.length === 1 && persisted.every(value => value.terminalTurns?.length !== 2)) { await vi.waitFor(() => expect(continuationCompletionSeen).toBe(true)); await new Promise(resolve => setTimeout(resolve, 20)); releaseStartCheckpoint(); } persisted.push(structuredClone(snapshot)); }, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, })).resolves.toMatchObject({ result, turnId: "turn-disposition" }); await startCheckpointHeld; expect(startTurn).toHaveBeenCalledTimes(2); expect((startTurn.mock.calls[0]![0] as { continuation?: true }).continuation).toBe(true); expect(startTurn.mock.calls[1]![0].message.text).toContain("Paperclip semantic-result recovery after a restart continuation"); }); it("never continues task work when a restart loses a compaction turn", async () => { const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "codex_app_server", sessionId: "thread-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const interrupted: PersistedNativeSession = { ...checkpoint, activeTurnId: null, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], }; const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); }, startTurn, async result() { return null; }, async snapshot() { return structuredClone(interrupted); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, sessionCompaction: { requestId: "compact-1" }, })).rejects.toMatchObject({ code: "native_provider_terminal_failed" }); expect(startTurn).not.toHaveBeenCalled(); }); it.each([undefined, 0, 7 * 24 * 60 * 60 * 1000, 30 * 24 * 60 * 60 * 1000])( "honors long-lived turn duration independently of operation bounds (%s)", async (turnTimeoutMs) => { vi.useFakeTimers(); let finish = () => {}; const waiting = new Promise((resolve) => { finish = resolve; }); const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; const cancel = vi.fn(() => { finish(); return { cleanup: Promise.resolve() }; }); const close = vi.fn(async () => { finish(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "tool.execution.started", { name: "wait for CI", status: "running" }); await waiting; yield runnerEvent(2, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, cancel, close, }; const appendEvent = vi.fn(async (event: PrpEvent) => ({ cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", })); const completeRun = vi.fn(async () => {}); try { const execution = executeNativeSession({ input, turnTimeoutMs, backend: { async descriptor() { return { kind: "mock", name: "long-lived", version: "1", capabilities }; }, async openSession() { return session; }, }, controlPlane: { async openRun() {}, async checkpointSession() {}, appendEvent, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); // Observe rejection before advancing the clock, including on the old implementation. let failure: unknown; const observed = execution.catch(error => { failure = error; return null; }); await vi.advanceTimersByTimeAsync(0); expect(appendEvent).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(6 * 24 * 60 * 60 * 1000); expect(failure).toBeUndefined(); expect(cancel).not.toHaveBeenCalled(); expect(close).not.toHaveBeenCalled(); expect(completeRun).not.toHaveBeenCalled(); if (turnTimeoutMs) { // Includes a 30-day bound beyond Node's single-timer maximum. await vi.advanceTimersByTimeAsync(turnTimeoutMs - 6 * 24 * 60 * 60 * 1000 - 1); expect(failure).toBeUndefined(); await vi.advanceTimersByTimeAsync(1); await observed; expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain(`native session timed out after ${turnTimeoutMs}ms`); expect(cancel).toHaveBeenCalledOnce(); expect(completeRun).not.toHaveBeenCalled(); } else { await vi.advanceTimersByTimeAsync(2 * 24 * 60 * 60 * 1000); finish(); expect(await observed).toMatchObject({ result }); expect(completeRun).toHaveBeenCalledOnce(); expect(cancel).not.toHaveBeenCalled(); } } finally { finish(); vi.useRealTimers(); } }, ); it.each([{ proposal: false, replayed: false }, { proposal: true, replayed: false }, { proposal: false, replayed: true }])("settles committed or duplicate provider exits while their stream stays open ($proposal, $replayed)", async ({ proposal, replayed }) => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; let release!: () => void; const held = new Promise(resolve => { release = resolve; }); const close = vi.fn(async () => { release(); }); const events: PrpEvent[] = []; const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { if (proposal) yield runnerEvent(1, "run.result.proposed", result); yield runnerEvent(proposal ? 2 : 1, "session.failed", { code: "provider_exited", provider: "codex", recoverable: false, expected: false, completedByExitedProcess: false, previousTurnCompleted: false, exitCode: 1, processGeneration: 1 }); await held; }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close, }; const completeRun = vi.fn(); const execution = executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "open-stream-exit", version: "1", capabilities }; }, async openSession() { return session; }, }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { if (!("sourceSeq" in event)) throw new Error("Expected provider PRP event"); events.push(event); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: replayed ? "duplicate" : "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); let settled = false, succeeded = false, failure: unknown; const observed = execution.then(() => { succeeded = true; settled = true; }, error => { failure = error; settled = true; }); try { await vi.waitFor(() => expect(settled).toBe(true), { timeout: 100, interval: 5 }); expect(succeeded).toBe(false); expect(failure).toMatchObject({ code: "native_provider_terminal_failed", providerCode: "provider_exited", recoverable: false }); expect(completeRun).not.toHaveBeenCalled(); expect(events.filter(event => event.eventType === "run.result.proposed").map(event => event.payload)).toEqual(proposal ? [result] : []); expect(events.some(event => event.eventType === "run.result.accepted")).toBe(false); expect(close).toHaveBeenCalledOnce(); } finally { release(); await observed; } }); it.each(["expected", "completedByExitedProcess", "previousTurnCompleted", "provider", "run", "session"] as const)("retains EOF settlement when provider exit fails the %s safety predicate", async mismatch => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; let release!: () => void; const held = new Promise(resolve => { release = resolve; }); let streamBlocked = false; const close = vi.fn(async () => { release(); }); const event = { ...runnerEvent(1, "session.failed", { code: "provider_exited", provider: mismatch === "provider" ? "acpx" : "codex", recoverable: false, expected: mismatch === "expected", completedByExitedProcess: mismatch === "completedByExitedProcess", previousTurnCompleted: mismatch === "previousTurnCompleted", exitCode: 1, processGeneration: 1 }), ...(mismatch === "run" ? { runId: "foreign-run" } : {}), ...(mismatch === "session" ? { normalizedSessionId: "foreign-session" } : {}) }; const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield event; streamBlocked = true; await held; }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close, }; const completeRun = vi.fn(); const execution = executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "guarded-provider-exit", version: "1", capabilities }; }, async openSession() { return session; }, }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(value) { if (!("sourceSeq" in value)) throw new Error("Expected provider PRP event"); return { cursor: value.sourceSeq, highestContiguousSourceSeq: value.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); let settled = false, failure: unknown; const observed = execution.then(() => { settled = true; }, error => { failure = error; settled = true; }); try { await vi.waitFor(() => expect(streamBlocked).toBe(true), { timeout: 100, interval: 5 }); expect(settled).toBe(false); expect(close).not.toHaveBeenCalled(); expect(completeRun).not.toHaveBeenCalled(); release(); await observed; expect(failure).toMatchObject({ code: "native_provider_terminal_failed", providerCode: "provider_exited", recoverable: false }); expect(close).toHaveBeenCalledOnce(); expect(completeRun).not.toHaveBeenCalled(); } finally { release(); await observed; } }); it("preserves a recoverable provider-exit goal until its later authoritative completion", async () => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const activeGoal = { threadId: "provider-recovery", objective: "Finish work", status: "active" as const, tokenBudget: null, tokensUsed: 100, timeUsedSeconds: 1, createdAt: 1, updatedAt: 2 }; const completeGoal = { ...activeGoal, status: "complete" as const }; let release!: () => void, reached!: () => void; const held = new Promise(resolve => { release = resolve; }); const failureObserved = new Promise(resolve => { reached = resolve; }); const close = vi.fn(async () => { release(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "session.goal.updated", { goal: activeGoal, workingNow: true }); yield runnerEvent(2, "session.failed", { code: "provider_exited", provider: "codex", recoverable: true, expected: false, completedByExitedProcess: false, previousTurnCompleted: false, exitCode: 1, processGeneration: 1 }); reached(); await held; yield runnerEvent(3, "session.goal.updated", { goal: activeGoal, workingNow: false }); yield runnerEvent(4, "turn.completed"); yield runnerEvent(5, "session.goal.snapshot", { goal: completeGoal, workingNow: false }); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async goal() { return completeGoal; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], goal: activeGoal, lineage: [] }; }, close, }; const completeRun = vi.fn(); const execution = executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "recoverable-open-exit", version: "1", capabilities }; }, async openSession() { return session; }, }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { if (!("sourceSeq" in event)) throw new Error("Expected provider PRP event"); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }); let settled = false; const observed = execution.then(value => { settled = true; return value; }, error => { settled = true; throw error; }); try { await failureObserved; expect(settled).toBe(false); expect(close).not.toHaveBeenCalled(); expect(completeRun).not.toHaveBeenCalled(); release(); expect(await observed).toMatchObject({ result: { reportedWorkDisposition: "done" } }); expect(close).toHaveBeenCalledOnce(); expect(completeRun).toHaveBeenCalledOnce(); } finally { release(); await observed.catch(() => undefined); } }); it.each([false, true])("preserves a durable session failure when its stream closes (throws=%s)", async throws => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "session.failed", { error: { code: "notification_transport_failed" }, recoverable: false }); if (throws) throw new Error("provider stdout closed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, async close() {}, }; const completeRun = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "failure-fixture", version: "1", capabilities }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })) .rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "notification_transport_failed", recoverable: false }); expect(completeRun).not.toHaveBeenCalled(); }); it.each((["complete", "paused", "blocked", "limited", "usageLimited", "budgetLimited"] as const) .flatMap((status) => [false, true].map((snapshotBeforeUpdate) => ({ status, snapshotBeforeUpdate }))))( "handles a new chat turn instead of completing it from an existing $status goal (snapshot: $snapshotBeforeUpdate)", async ({ status, snapshotBeforeUpdate }) => { const oldGoal = { threadId: "provider-recovery", objective: "Say hello", status, tokenBudget: null, tokensUsed: 500, timeUsedSeconds: 2, createdAt: Date.parse("2026-08-09T00:00:00.000Z"), updatedAt: Date.parse("2026-08-09T00:00:02.000Z"), }; const reply = { ...result, summary: "Said bye in response to the new message." }; const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: oldGoal.threadId, activeTurnId: null, semanticResult: null, terminal: null, terminalTurns: [], pendingRuntimeRequests: [], goal: { ...oldGoal, createdAt: oldGoal.createdAt / 1000, updatedAt: oldGoal.updatedAt / 1000 }, }; const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const goal = vi.fn(async () => oldGoal); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { let seq = 0; // The resume snapshot is durable UI state, not work for this prompt. if (snapshotBeforeUpdate) yield runnerEvent(++seq, "session.goal.snapshot", { goal: { ...oldGoal, createdAt: new Date(oldGoal.createdAt).toISOString(), elapsedSeconds: oldGoal.timeUsedSeconds, }, workingNow: false, }); // Codex replays the unchanged goal as an update during resume too. // Usage-only changes do not make an inactive goal own a new prompt. yield runnerEvent(++seq, "session.goal.updated", { goal: { ...oldGoal, createdAt: new Date(oldGoal.createdAt).toISOString(), tokensUsed: 600, updatedAt: new Date(oldGoal.updatedAt + 1000).toISOString(), }, workingNow: false, }); yield runnerEvent(++seq, "turn.started"); yield runnerEvent(++seq, "run.result.proposed", reply); yield runnerEvent(++seq, "turn.completed"); }, startTurn, goal, async result() { return { result: reply, terminal, turnId: "turn-recovery" }; }, async snapshot() { return checkpoint; }, async close() {}, }; const appended: PrpEvent[] = []; const digest = "0".repeat(64); const context = { prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() }, instructions: { entryPath: "AGENTS.md", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/instructions", fileCount: 1, totalBytes: 1 } }, skills: [], mcp: { assignmentSetId: "none", digest, bindingId: null }, } satisfies RuntimeContextFields; const completed = await executeNativeSession({ input: snapshotBeforeUpdate ? { ...input, schema: "paperclip.native-execution-input.v4", executionMode: "default", planningContext: null, provider: { kind: "codex", model: null, approvalPolicy: "never" }, runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context) }, continuationPrompt: "Say bye", } : { ...input, task: { ...input.task, prompt: "Say bye" } }, backend: { async descriptor() { return { kind: "mock", name: "chat-after-goal", version: "1", capabilities, runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" } }; }, async openSession() { throw new Error("must resume the same provider session"); }, async recoverSession() { return { recovered: true, session }; }, }, persistedSession: checkpoint, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { appended.push(event); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1000, }); expect(startTurn).toHaveBeenCalledOnce(); const submitted = startTurn.mock.calls[0]![0]!; if (snapshotBeforeUpdate) { expect(submitted.continuation).toBe(true); expect(JSON.parse(submitted.message.text)).toMatchObject({ schema: "paperclip.native-continuation.v1", events: "Say bye", completion: { instruction: expect.stringContaining("Earlier reports belong to earlier turns") } }); } else { expect(submitted).not.toHaveProperty("continuation"); expect(JSON.parse(submitted.message.text).task.prompt).toBe("Say bye"); } expect(goal).not.toHaveBeenCalled(); expect(completed.providerSessionId).toBe(oldGoal.threadId); expect(completed.result).toEqual(reply); expect(appended.map((event) => event.eventType)).toEqual([ ...(snapshotBeforeUpdate ? ["session.goal.snapshot"] : []), "session.goal.updated", "turn.started", "run.result.proposed", "turn.completed", "run.result.accepted", "run.terminal", ]); }); it("applies a session goal control without starting an ordinary turn", async () => { const activeGoal = { threadId: "provider-recovery", objective: "Verify goal mode", status: "active" as const, tokenBudget: 12_000, tokensUsed: 100, timeUsedSeconds: 1, createdAt: Date.parse("2026-08-09T00:00:00.000Z"), updatedAt: Date.parse("2026-08-09T00:00:01.000Z"), }; const completeGoal = { ...activeGoal, status: "complete" as const, tokensUsed: 500, timeUsedSeconds: 2, updatedAt: Date.parse("2026-08-09T00:00:02.000Z"), }; const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const goal = vi.fn(async (operation: Parameters>[0]) => operation.action === "get" ? completeGoal : activeGoal, ); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "session.goal.snapshot", { goal: null, workingNow: false, }); yield runnerEvent(2, "session.goal.updated", { requestId: "goal-create", goal: { objective: activeGoal.objective, status: activeGoal.status, tokenBudget: activeGoal.tokenBudget, tokensUsed: activeGoal.tokensUsed, elapsedSeconds: activeGoal.timeUsedSeconds, }, workingNow: false, }); yield runnerEvent(3, "turn.started"); yield runnerEvent(4, "run.result.proposed", result); yield runnerEvent(5, "turn.completed"); yield runnerEvent(6, "session.goal.snapshot", { goal: { objective: completeGoal.objective, status: completeGoal.status, tokenBudget: completeGoal.tokenBudget, tokensUsed: completeGoal.tokensUsed, elapsedSeconds: completeGoal.timeUsedSeconds, }, workingNow: false, }); }, startTurn, goal, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], goal: completeGoal, lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "goal-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const appended: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { appended.push(event); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", sessionGoalControl: { requestId: "goal-create", action: "create", objective: activeGoal.objective, tokenBudget: activeGoal.tokenBudget, }, }); expect(startTurn).not.toHaveBeenCalled(); expect(goal).toHaveBeenNthCalledWith(1, { action: "set", objective: activeGoal.objective, status: "active", requestId: "goal-create", tokenBudget: activeGoal.tokenBudget, }); expect(goal).toHaveBeenNthCalledWith(2, { action: "get" }); expect(appended.map((event) => event.eventType)).toContain("session.goal.snapshot"); expect(completed.result).toMatchObject({ reportedWorkDisposition: "done", summary: result.summary, completionClaim: { objectiveSatisfied: true }, }); }); it.each((["complete", "paused", "blocked", "limited", "usageLimited", "budgetLimited"] as const) .flatMap((status) => [false, true].map((lateGoal) => ({ status, lateGoal }))))( "reconciles an out-of-band $status goal (after semantic result: $lateGoal)", async ({ status, lateGoal }) => { const activeGoal = { threadId: "provider-agent-goal", objective: "Finish autonomous work", status: "active" as const, tokenBudget: null, tokensUsed: 100, timeUsedSeconds: 1, createdAt: Date.parse("2026-08-09T00:00:00.000Z"), updatedAt: Date.parse("2026-08-09T00:00:01.000Z"), }; const completeGoal = { ...activeGoal, status, tokensUsed: 500, timeUsedSeconds: 2, updatedAt: Date.parse("2026-08-09T00:00:02.000Z"), }; const startTurn = vi.fn(async () => ({ turnId: "turn-agent-goal" })); const goal = vi.fn(async () => completeGoal); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { if (lateGoal) yield runnerEvent(1, "run.result.proposed", result); yield runnerEvent(lateGoal ? 2 : 1, "session.goal.updated", { goal: { objective: activeGoal.objective, status: activeGoal.status, tokenBudget: activeGoal.tokenBudget, tokensUsed: activeGoal.tokensUsed, elapsedSeconds: activeGoal.timeUsedSeconds, }, workingNow: true, }); if (!lateGoal) yield runnerEvent(2, "run.result.proposed", result); // A newly observed goal owns the lifetime even when the completion // proposal arrived first. await new Promise((resolve) => setTimeout(resolve, 25)); yield runnerEvent(3, "session.goal.updated", { goal: { objective: completeGoal.objective, status: completeGoal.status, tokenBudget: completeGoal.tokenBudget, tokensUsed: completeGoal.tokensUsed, elapsedSeconds: completeGoal.timeUsedSeconds, }, workingNow: true, }); yield runnerEvent(4, "turn.completed"); }, startTurn, goal, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-agent-goal", identity, providerSessionId: activeGoal.threadId, cursor: null, activeTurnId: null, pendingRuntimeRequests: [], goal: activeGoal, lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "agent-goal-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const appended: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { appended.push(event); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-agent-goal", controlPlaneInstanceId: "control-agent-goal", }); expect(startTurn).toHaveBeenCalledOnce(); expect(goal).not.toHaveBeenCalled(); expect(appended.map((event) => event.eventType)).toEqual([ ...(lateGoal ? ["run.result.proposed", "session.goal.updated"] : ["session.goal.updated", "run.result.proposed"]), "session.goal.updated", "turn.completed", "run.result.accepted", "run.terminal", ]); expect(completed.result).toMatchObject({ reportedWorkDisposition: status === "complete" ? "done" : status === "blocked" ? "blocked" : "yielded", ...(status === "complete" ? { summary: result.summary } : {}), completionClaim: { objectiveSatisfied: status === "complete" }, }); }); it.each([ { keepSessionOpen: false, recoveryOnly: false }, { keepSessionOpen: true, recoveryOnly: false }, { keepSessionOpen: false, recoveryOnly: true }, { keepSessionOpen: true, recoveryOnly: true }, ].flatMap((options) => [false, true].map((cleared) => ({ ...options, cleared }))))("reconciles goal recovery without replaying completed controls (warm=$keepSessionOpen, recovery=$recoveryOnly, cleared=$cleared)", async ({ keepSessionOpen, recoveryOnly, cleared }) => { const pausedGoal = { threadId: "provider-recovery", objective: "Verify recovered goal control", status: recoveryOnly ? "complete" as const : "paused" as const, tokenBudget: null, tokensUsed: 250, timeUsedSeconds: 2, createdAt: Date.parse("2026-08-09T00:00:00.000Z"), updatedAt: Date.parse("2026-08-09T00:00:02.000Z"), }; const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const goal = vi.fn(async () => cleared ? null : pausedGoal); const requestId = recoveryOnly ? `recovery_${input.binding.runId}` : cleared ? "goal-clear" : "goal-pause"; const close = vi.fn(async () => {}); let snapshotCount = 0; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, cleared ? "session.goal.cleared" : "session.goal.updated", { requestId, goal: cleared ? null : { objective: pausedGoal.objective, status: pausedGoal.status, tokenBudget: pausedGoal.tokenBudget, tokensUsed: pausedGoal.tokensUsed, elapsedSeconds: pausedGoal.timeUsedSeconds, }, workingNow: !recoveryOnly, }); yield runnerEvent(2, "turn.completed"); yield runnerEvent(3, "session.goal.snapshot", { goal: cleared ? null : { objective: pausedGoal.objective, status: pausedGoal.status, tokenBudget: pausedGoal.tokenBudget, tokensUsed: pausedGoal.tokensUsed, elapsedSeconds: pausedGoal.timeUsedSeconds, }, workingNow: false, }); }, startTurn, goal, async result() { return null; }, async snapshot() { snapshotCount += 1; return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: !recoveryOnly && snapshotCount === 1 ? "turn-recovery" : null, pendingRuntimeRequests: [], goal: cleared ? null : pausedGoal, lineage: [], }; }, close, }; const persistedSession: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: "turn-recovery", pendingRuntimeRequests: [], goal: { ...pausedGoal, status: "active" }, lineage: [], }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "goal-recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("fresh session must not be opened"); }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession, keepSessionOpen, requireSessionCloseBeforeReturn: true, resumeSessionGoalHeartbeat: recoveryOnly, sessionGoalControl: recoveryOnly ? null : { requestId, action: cleared ? "clear" : "pause", }, }); expect(startTurn).not.toHaveBeenCalled(); expect(goal).toHaveBeenNthCalledWith(1, { action: recoveryOnly ? "get" : cleared ? "clear" : "pause", requestId, }); expect(goal).toHaveBeenCalledTimes(cleared && !recoveryOnly ? 2 : 1); if (cleared && !recoveryOnly) { expect(goal).toHaveBeenNthCalledWith(2, { action: "get" }); } expect(close).toHaveBeenCalledTimes(1); expect(completed.result).toMatchObject({ reportedWorkDisposition: recoveryOnly && !cleared ? "done" : "yielded", completionClaim: { objectiveSatisfied: recoveryOnly && !cleared }, }); }); describe("context compaction", () => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; function compactingSession(startTurn: NativeSession["startTurn"], activeTurnId: string | null = null): NativeSession { return { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "turn.started"); yield runnerEvent(2, "context.compacted", { schema: "paperclip.context.compacted.v1", compactionId: "compaction-1", reason: "provider", preTokens: null, postTokens: null, sameSession: true, }); yield runnerEvent(3, "turn.completed"); }, startTurn, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId, pendingRuntimeRequests: [], lineage: [], }; }, close: vi.fn(async () => {}), }; } const persistedSession: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; it("compacts a resumed session and settles without task work", async () => { const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const session = compactingSession(startTurn); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "compaction", version: "1", capabilities }; }, async openSession() { throw new Error("fresh session must not be opened"); }, async recoverSession() { return { recovered: true, session }; }, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession, requireSessionCloseBeforeReturn: true, sessionCompaction: { requestId: "compact-1" }, }); expect(startTurn).toHaveBeenCalledTimes(1); expect(startTurn).toHaveBeenCalledWith(expect.objectContaining({ compact: true, continuation: true })); expect(completed.result).toMatchObject({ reportedWorkDisposition: "yielded", summary: "Context compacted on request.", completionClaim: { objectiveSatisfied: false, remainingWork: [] }, }); expect(completed.result.continuation).toMatchObject({ kind: "same_agent" }); expect(validatePrpStructuredRunResult(completed.result)).toMatchObject({ ok: true }); }); it("keeps waiting on its own compaction turn after a restart", async () => { const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const session = compactingSession(startTurn, "turn-recovery"); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "compaction", version: "1", capabilities }; }, async openSession() { throw new Error("fresh session must not be opened"); }, async recoverSession() { return { recovered: true, session }; }, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession: { ...persistedSession, activeTurnId: "turn-recovery" }, requireSessionCloseBeforeReturn: true, sessionCompaction: { requestId: "compact-1" }, }); expect(startTurn).not.toHaveBeenCalled(); expect(completed.result).toMatchObject({ summary: "Context compacted on request." }); }); it("never compacts over another run's checkpoint", async () => { const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const session = compactingSession(startTurn, "turn-recovery"); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "compaction", version: "1", capabilities }; }, async openSession() { throw new Error("fresh session must not be opened"); }, async recoverSession() { return { recovered: true, session }; }, }; await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession: { ...persistedSession, identity: { ...identity, runId: "run-earlier" }, activeTurnId: "turn-recovery", }, sessionCompaction: { requestId: "compact-1" }, })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); expect(startTurn).not.toHaveBeenCalled(); }); it("refuses to compact a fresh session that has no prior context", async () => { const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const session = compactingSession(startTurn); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "compaction", version: "1", capabilities }; }, async openSession() { return session; }, }; await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", sessionCompaction: { requestId: "compact-1" }, })).rejects.toThrow("native_context_compaction_unavailable"); expect(startTurn).not.toHaveBeenCalled(); }); }); it.each([ { recoverable: false, message: "There's an issue with the selected model (custom-model). It may not exist or you may not have access to it.", modelRejected: true, }, { recoverable: true, message: "There's an issue with the selected model (custom-model). It may not exist or you may not have access to it.", modelRejected: false, }, { recoverable: false, message: "The model service failed while processing output.", modelRejected: false, }, { // A driver's closed category marks the rejection; no provider text is needed. recoverable: false, message: "Provider turn failed", modelRejected: true, category: "model_unavailable" as const, }, { recoverable: true, message: "Provider turn failed", modelRejected: false, category: "model_unavailable" as const, }, ] as Array<{ recoverable: boolean; message: string; modelRejected: boolean; category?: "model_unavailable" }>)( "preserves structured provider failure and model retry classification ($recoverable, $modelRejected, $category)", async ({ recoverable, message, modelRejected, category }) => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; const close = vi.fn(async () => {}); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "turn.failed", { error: { code: "RUNTIME", recoverable, message, ...(category ? { providerFailure: createProviderFailure({ provider: "openrouter", phase: "turn", message: "openrouter/unavailable-model is not a valid model ID", httpStatus: 400, category, }) } : {}) }, }); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "model-rejection", version: "1", capabilities, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const result = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await expect(result).rejects.toThrow(message); await expect(result).rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "RUNTIME", recoverable, }); if (modelRejected) { await expect(result).rejects.toThrow("native_provider_model_rejected:"); } else { await expect(result).rejects.not.toThrow( "native_provider_model_rejected", ); } expect(close).toHaveBeenCalled(); }, ); it.each([ { status: "active", code: "semantic_completion_rejected", recoverable: false }, { status: "complete", code: "semantic_completion_rejected", recoverable: false }, { status: "complete", code: "provider_output_failed", recoverable: false }, { status: "active", code: "provider_turn_lost_on_restore", recoverable: true }, ] as const)( "preserves session-goal failure priority ($status, $code, recoverable=$recoverable)", async ({ status, code, recoverable }) => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const activeGoal = { threadId: "provider-recovery", objective: "Finish autonomous work", status: "active" as const, tokenBudget: null, tokensUsed: 100, timeUsedSeconds: 1, createdAt: Date.parse("2026-08-09T00:00:00.000Z"), updatedAt: Date.parse("2026-08-09T00:00:01.000Z"), }; const completeGoal = { ...activeGoal, status: "complete" as const }; const goal = vi.fn(async () => completeGoal); const startTurn = vi.fn(async () => ({ turnId: "turn-recovery" })); const close = vi.fn(async () => {}); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "session.goal.updated", { goal: activeGoal, workingNow: true }); yield runnerEvent(2, "session.failed", { code, recoverable, message: "The turn ended without correcting its rejected completion report.", }); yield runnerEvent(3, "session.goal.updated", { goal: { ...activeGoal, status }, workingNow: false }); yield runnerEvent(4, "turn.completed"); yield runnerEvent(5, "session.goal.snapshot", { goal: completeGoal, workingNow: false }); }, startTurn, goal, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], goal: activeGoal, lineage: [] }; }, close, }; const appended: PrpEvent[] = []; const completeRun = vi.fn(async () => {}); const execution = executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "failed-goal", version: "1", capabilities }; }, async openSession() { return session; }, }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { appended.push(event); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); if (recoverable) { await expect(execution).resolves.toMatchObject({ result: { reportedWorkDisposition: "done" } }); expect(goal).toHaveBeenCalledExactlyOnceWith({ action: "get" }); expect(completeRun).toHaveBeenCalledOnce(); } else { await expect(execution).rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: code, recoverable: false, }); expect(appended.map(event => event.eventType)).toEqual([ "session.goal.updated", "session.failed", "session.goal.updated", "turn.completed", ]); expect(goal).not.toHaveBeenCalled(); expect(completeRun).not.toHaveBeenCalled(); } expect(startTurn).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); }, ); it("keeps governed-wait discovery synchronous", () => { type GovernedWaitResolver = NonNullable< ExecuteNativeSessionOptions["resolveGovernedWait"] >; const resolver: GovernedWaitResolver = () => null; // An async resolver could retain control-plane mutation authority after // execution settles, so the public boundary rejects it at compile time. // @ts-expect-error governed-wait discovery must not return a promise const asynchronousResolver: GovernedWaitResolver = async () => null; expect(resolver).toBeTypeOf("function"); void asynchronousResolver; }); it("preserves durable success while quarantined cleanup stays bounded", async () => { vi.useFakeTimers(); try { let executionCloseCount = 0; let quarantineAttempt = 0; const close = vi.fn(({ reason }: { reason: string }) => { if (reason === "native session quarantined cleanup recovery") { quarantineAttempt += 1; const attempt = quarantineAttempt; return new Promise((resolve, reject) => { setTimeout(() => { if (attempt < 3) reject(new Error("transient quarantine failure")); else resolve(); }, 6_500); }); } if (reason === "native session execution complete") { executionCloseCount += 1; if (executionCloseCount > 1) return Promise.resolve(); } return Promise.reject(new Error("persistent close failure")); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await expect(execute()).resolves.toMatchObject({ result }); expect(close).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(3_000); expect(close).toHaveBeenCalledTimes(5); // Admission inherits the already-running three-attempt recovery and // waits through its bounded attempts instead of timing out after one. const recoveredExecution = execute(); let admissionSettled = false; void recoveredExecution.then( () => { admissionSettled = true; }, () => { admissionSettled = true; }, ); await vi.advanceTimersByTimeAsync(20_000); expect(admissionSettled).toBe(false); expect(openSession).toHaveBeenCalledTimes(1); expect(close).toHaveBeenCalledTimes(7); await vi.advanceTimersByTimeAsync(2_000); await expect(recoveredExecution).resolves.toMatchObject({ result }); expect(quarantineAttempt).toBe(3); expect(close).toHaveBeenCalledTimes(8); expect(openSession).toHaveBeenCalledTimes(2); } finally { vi.useRealTimers(); } }); it("awaits the quarantine owner that replaces an exhausted close recovery", async () => { vi.useFakeTimers(); try { let executionCloseCount = 0; const close = vi.fn(({ reason }: { reason: string }) => { if (reason === "native session execution complete") { executionCloseCount += 1; return executionCloseCount === 1 ? Promise.reject(new Error("initial close failed")) : Promise.resolve(); } if ( reason.startsWith( "native session cleanup recovery after close failure", ) ) { return Promise.reject(new Error("bounded close recovery failed")); } if (reason === "native session quarantined cleanup recovery") { return new Promise((resolve) => setTimeout(resolve, 50)); } return Promise.resolve(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await expect(execute()).resolves.toMatchObject({ result }); const admitted = execute(); await vi.advanceTimersByTimeAsync(3_100); await expect(admitted).resolves.toMatchObject({ result }); expect(openSession).toHaveBeenCalledTimes(2); expect(close).toHaveBeenCalledWith({ reason: "native session quarantined cleanup recovery", }); } finally { vi.useRealTimers(); } }); it("covers maximum-duration retained and replacement recovery phases", async () => { vi.useFakeTimers(); try { let executionCloseCount = 0; let retainedRecoveryAttempt = 0; let quarantineRecoveryAttempt = 0; const close = vi.fn(({ reason }: { reason: string }) => { if (reason === "native session execution complete") { executionCloseCount += 1; if (executionCloseCount > 1) return Promise.resolve(); return new Promise((_resolve, reject) => { setTimeout(() => reject(new Error("initial close failed")), 6_900); }); } if ( reason.startsWith( "native session cleanup recovery after close failure", ) ) { retainedRecoveryAttempt += 1; return new Promise((_resolve, reject) => { setTimeout( () => reject(new Error("bounded retained recovery failed")), 6_900, ); }); } if (reason === "native session quarantined cleanup recovery") { quarantineRecoveryAttempt += 1; const attempt = quarantineRecoveryAttempt; return new Promise((resolve, reject) => { setTimeout(() => { if (attempt < 3) reject(new Error("transient quarantine recovery failure")); else resolve(); }, 6_500); }); } return Promise.resolve(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); const firstExecution = execute(); await vi.advanceTimersByTimeAsync(100); await expect(firstExecution).resolves.toMatchObject({ result }); const admitted = execute(); let admissionSettled = false; void admitted.then( () => { admissionSettled = true; }, () => { admissionSettled = true; }, ); // The initial close plus three near-bound retries exceed the old 23s // admission grace but remain within the configured 31s owner phase. await vi.advanceTimersByTimeAsync(23_100); expect(retainedRecoveryAttempt).toBe(2); expect(quarantineRecoveryAttempt).toBe(0); expect(admissionSettled).toBe(false); expect(openSession).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(7_600); expect(retainedRecoveryAttempt).toBe(3); expect(quarantineRecoveryAttempt).toBe(1); expect(admissionSettled).toBe(false); expect(openSession).toHaveBeenCalledOnce(); // The replacement then receives its complete three-attempt bound rather // than inheriting only the remainder of the retained owner's deadline. await vi.advanceTimersByTimeAsync(21_600); await expect(admitted).resolves.toMatchObject({ result }); expect(quarantineRecoveryAttempt).toBe(3); expect(openSession).toHaveBeenCalledTimes(2); } finally { vi.useRealTimers(); } }); it("runs a full admission batch after an inherited scheduled attempt fails", async () => { vi.useFakeTimers(); try { let executionCloseCount = 0; let scheduledRecoveryAttempt = 0; let admissionRecoveryAttempt = 0; const close = vi.fn(({ reason }: { reason: string }) => { if (reason === "native session execution complete") { executionCloseCount += 1; return executionCloseCount === 1 ? Promise.reject(new Error("initial close failed")) : Promise.resolve(); } if ( reason.startsWith( "native session cleanup recovery after close failure", ) ) { return Promise.reject(new Error("bounded retained recovery failed")); } if (reason === "native session quarantined cleanup recovery") { return Promise.reject( new Error("bounded quarantine recovery failed"), ); } if ( reason === "native session scheduled quarantined cleanup recovery" ) { scheduledRecoveryAttempt += 1; return new Promise((_resolve, reject) => { setTimeout( () => reject(new Error("scheduled cleanup failed")), 6_500, ); }); } if (reason === "native session quarantined admission recovery") { admissionRecoveryAttempt += 1; const attempt = admissionRecoveryAttempt; return new Promise((resolve, reject) => { setTimeout(() => { if (attempt === 1) reject(new Error("transient admission cleanup failure")); else resolve(); }, 6_500); }); } return Promise.resolve(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await expect(execute()).resolves.toMatchObject({ result }); // Exhaust retained and initial quarantine batches, then enter the slow // autonomous one-attempt recovery scheduled sixty seconds later. await vi.advanceTimersByTimeAsync(65_001); expect(scheduledRecoveryAttempt).toBe(1); const admitted = execute(); let admissionSettled = false; void admitted.then( () => { admissionSettled = true; }, () => { admissionSettled = true; }, ); await vi.advanceTimersByTimeAsync(6_600); expect(admissionRecoveryAttempt).toBe(1); expect(admissionSettled).toBe(false); expect(openSession).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(14_100); await expect(admitted).resolves.toMatchObject({ result }); expect(admissionRecoveryAttempt).toBe(2); expect(openSession).toHaveBeenCalledTimes(2); } finally { vi.useRealTimers(); } }); it("stops scheduled cleanup after the quarantine lifetime budget", async () => { vi.useFakeTimers(); try { let executionCloseCount = 0; let admissionAttempt = 0; let scheduledAttempt = 0; const lifetimeIdentity = { ...identity, companyId: "company-cleanup-lifetime", }; const close = vi.fn(({ reason }: { reason: string }) => { if (reason === "native session execution complete") { executionCloseCount += 1; return executionCloseCount === 1 ? Promise.reject(new Error("initial close failed")) : Promise.resolve(); } if ( reason.startsWith( "native session cleanup recovery after close failure", ) ) { return Promise.reject(new Error("bounded close recovery failed")); } if (reason === "native session quarantined cleanup recovery") { return Promise.reject(new Error("quarantined close recovery failed")); } if (reason === "native session quarantined admission recovery") { admissionAttempt += 1; return Promise.reject(new Error("admission cleanup failed")); } if ( reason === "native session scheduled quarantined cleanup recovery" ) { scheduledAttempt += 1; return Promise.reject(new Error("scheduled cleanup failed")); } return Promise.resolve(); }); const session: NativeSession = { identity: () => lifetimeIdentity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity: lifetimeIdentity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input: { ...input, binding: { ...input.binding, companyId: lifetimeIdentity.companyId, }, }, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await expect(execute()).resolves.toMatchObject({ result }); await vi.advanceTimersByTimeAsync(6_000); const recoveredExecution = expect(execute()).rejects.toThrow( "prior session cleanup remains incomplete", ); await vi.advanceTimersByTimeAsync(2_100); await recoveredExecution; expect(admissionAttempt).toBe(3); expect(scheduledAttempt).toBe(0); const block = { companyId: lifetimeIdentity.companyId, issueId: input.binding.issueId }; // Automatic recovery is still scheduled: the server may hold input. expect(nativeSessionCleanupBlock(block)).toBe("pending"); await vi.advanceTimersByTimeAsync(180_000); expect(scheduledAttempt).toBe(3); await vi.advanceTimersByTimeAsync(300_000); expect(scheduledAttempt).toBe(3); expect(openSession).toHaveBeenCalledOnce(); // The automatic budget is spent and nothing else is scheduled: waiting // would never end, so the task needs stopped-process recovery. expect(nativeSessionCleanupBlock(block)).toBe("operator_required"); } finally { vi.useRealTimers(); } }); it("fails admission within a bound when quarantined close never settles", async () => { vi.useFakeTimers(); try { let releaseBlockedClose = () => {}; let blockClose = false; const close = vi.fn(() => { if (blockClose) { return new Promise((resolve) => { releaseBlockedClose = resolve; }); } return Promise.reject(new Error("persistent close failure")); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await expect(execute()).resolves.toMatchObject({ result }); await vi.advanceTimersByTimeAsync(6_000); blockClose = true; const blockedAdmission = execute(); const blockedResult = expect(blockedAdmission).rejects.toThrow( "prior session cleanup exceeded the admission grace", ); await vi.advanceTimersByTimeAsync(31_100); await blockedResult; expect(openSession).toHaveBeenCalledOnce(); const isolatedSession: NativeSession = { ...session, identity: () => ({ ...identity, companyId: "company-isolated" }), close: vi.fn(async () => undefined), }; const isolatedOpenSession = vi.fn(async () => isolatedSession); await expect( executeNativeSession({ input: { ...input, binding: { ...input.binding, companyId: "company-isolated", }, }, backend: { ...backend, openSession: isolatedOpenSession }, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ result }); expect(isolatedOpenSession).toHaveBeenCalledOnce(); blockClose = false; releaseBlockedClose(); close.mockResolvedValue(undefined); await vi.runAllTimersAsync(); await expect(execute()).resolves.toMatchObject({ result }); } finally { vi.useRealTimers(); } }); it("quarantines a pending first close before another provider session can open", async () => { vi.useFakeTimers(); try { let releaseClose = () => {}; const pendingClose = new Promise((resolve) => { releaseClose = resolve; }); const close = vi.fn(() => pendingClose); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const openSession = vi.fn(async () => session); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); const firstExecution = execute(); await vi.advanceTimersByTimeAsync(100); await expect(firstExecution).resolves.toMatchObject({ result }); expect(close).toHaveBeenCalledOnce(); const blockedAdmission = execute(); const blockedResult = expect(blockedAdmission).rejects.toThrow( "prior session cleanup exceeded the admission grace", ); await vi.advanceTimersByTimeAsync(31_100); await blockedResult; expect(openSession).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); releaseClose(); await vi.runAllTimersAsync(); } finally { vi.useRealTimers(); } }); it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => { const digest = "0".repeat(64); const context = { prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest(), }, instructions: { entryPath: "AGENTS.md", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/paperclip/context/instructions", fileCount: 1, totalBytes: 1, }, }, skills: [], mcp: { assignmentSetId: "none", digest, bindingId: null }, } satisfies RuntimeContextFields; const openSession = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "future-provider", version: "1", capabilities: { resume: false, typedEvents: true, steering: false, interruption: false, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input: { ...input, schema: "paperclip.native-execution-input.v3", executionMode: "default", planningContext: null, runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context), }, }, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).rejects.toThrow("does not natively realize instructions, skills, mcp"); expect(openSession).not.toHaveBeenCalled(); }); it("does not admit a fresh run when provider session initialization fails", async () => { const providerFailure = new Error("provider initialization failed"); const openSession = vi.fn(async () => { throw providerFailure; }); const openRun = vi.fn(async () => undefined); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "fresh-backend", version: "1", capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).rejects.toBe(providerFailure); expect(openSession).toHaveBeenCalledWith( expect.objectContaining({ identity, workingDirectory: input.workspace.cwd, signal: expect.any(AbortSignal), }), ); expect(openRun).not.toHaveBeenCalled(); }); it.each(["synchronous failure", "expired deadline"].flatMap(scenario => [false, true].map(retained => ({ scenario, retained }))))("cleans up startup cancellation after $scenario (retained: $retained)", async ({ scenario, retained }) => { const abort = new AbortController(); const remove = vi.spyOn(abort.signal, "removeEventListener"); const failure = new Error("synchronous backend failure"); const openSession = vi.fn((): Promise => { throw failure; }); const attachRun = vi.fn((): Promise => { throw failure; }); const openRun = vi.fn(async () => undefined); const capabilities = { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: `startup-${scenario}`, version: "1", capabilities }; }, openSession, }; const existingSession: NativeSession = { identity: () => ({ ...identity, runId: "prior-run" }), async capabilities() { return capabilities; }, attachRun, async *events() {}, async startTurn() { throw new Error("unexpected turn"); }, async result() { return null; }, async snapshot() { throw new Error("unexpected snapshot"); }, async close() {}, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", ...(retained ? { existingSession } : {}), startup: { deadlineAtMs: Date.now() + (scenario === "expired deadline" ? -1 : 60_000), signal: abort.signal }, })).rejects.toThrow(scenario === "expired deadline" ? retained ? "native_execution_startup_expired" : "native session bootstrap timed out after 0ms" : failure.message); expect(openSession).toHaveBeenCalledTimes(!retained && scenario !== "expired deadline" ? 1 : 0); expect(attachRun).toHaveBeenCalledTimes(retained && scenario !== "expired deadline" ? 1 : 0); expect(openRun).not.toHaveBeenCalled(); if (scenario === "synchronous failure") expect(remove).toHaveBeenCalledWith("abort", expect.any(Function)); abort.abort(new Error("after completion")); await Promise.resolve(); }); it("retains late bootstrap cleanup through the next admission", async () => { vi.useFakeTimers(); let releaseClose = () => {}; try { let resolveBootstrap = (_value: NativeSession) => {}; const stalledBootstrap = new Promise((resolve) => { resolveBootstrap = resolve; }); let markBootstrapStarted = () => {}; const bootstrapStarted = new Promise((resolve) => { markBootstrapStarted = resolve; }); let markCloseStarted = () => {}; const closeStarted = new Promise((resolve) => { markCloseStarted = resolve; }); const closeReleased = new Promise((resolve) => { releaseClose = resolve; }); const close = vi.fn(async () => { markCloseStarted(); await closeReleased; }); const lateSession: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() {}, async startTurn() { throw new Error("late fresh session must not start"); }, async result() { return null; }, async snapshot() { throw new Error("late fresh session must not snapshot"); }, close, }; let bootstrapSignal: AbortSignal | undefined; let bootstrapCount = 0; const openSession = vi.fn( (bootstrapInput: { identity: NativeRunIdentity; workingDirectory?: string; signal?: AbortSignal; }) => { bootstrapCount += 1; if (bootstrapCount > 1) { throw new Error("replacement bootstrap launched"); } bootstrapSignal = bootstrapInput.signal; bootstrapInput.signal?.addEventListener( "abort", () => resolveBootstrap(lateSession), { once: true }, ); markBootstrapStarted(); return stalledBootstrap; }, ); const openRun = vi.fn(async () => undefined); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "fresh-stalled-backend", version: "1", capabilities: await lateSession.capabilities(), }; }, openSession, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 5, }); const execution = execute(); const rejection = expect(execution).rejects.toThrow( "native session bootstrap timed out after 5ms", ); await bootstrapStarted; expect(bootstrapSignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(5); await rejection; expect(bootstrapSignal?.aborted).toBe(true); expect(openRun).not.toHaveBeenCalled(); await closeStarted; expect(close).toHaveBeenCalledWith({ reason: "native session bootstrap timed out", }); // Even after the detached disposer exhausts its short settlement grace, // the exact close remains admission-visible until it releases provider // resources. A replacement bootstrap cannot start concurrently. await vi.advanceTimersByTimeAsync(101); const blockedAdmission = execute(); const blockedAdmissionRejection = expect( blockedAdmission, ).rejects.toThrow("replacement bootstrap launched"); await vi.advanceTimersByTimeAsync(1_000); expect(openSession).toHaveBeenCalledOnce(); releaseClose(); await vi.advanceTimersByTimeAsync(0); await blockedAdmissionRejection; expect(openSession).toHaveBeenCalledTimes(2); expect(openRun).not.toHaveBeenCalled(); } finally { releaseClose(); vi.useRealTimers(); } }); it("waits for controller ownership publication before dispatching a turn", async () => { let release!: () => void; const published = new Promise((resolve) => { release = resolve; }); const snapshotFailure = new Error("stop after ownership publication"); const snapshot = vi.fn(async () => { throw snapshotFailure; }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() {}, async startTurn() { throw new Error("unexpected turn"); }, async result() { return null; }, snapshot, close: vi.fn(async () => undefined), }; const onSession = vi.fn(async (current: NativeSession | null) => { if (current) await published; }); const running = executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "owner-barrier", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; } }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {} }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", onSession, }); const rejected = expect(running).rejects.toBe(snapshotFailure); await vi.waitFor(() => expect(onSession).toHaveBeenCalledWith(session)); expect(snapshot).not.toHaveBeenCalled(); release(); await rejected; }); it("closes the provider when owner quarantine notification throws", async () => { const snapshotFailure = new Error("snapshot failed"); const close = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() {}, async startTurn() { throw new Error("unexpected turn"); }, async result() { return null; }, async snapshot() { throw snapshotFailure; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "owner-notification-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const retainedSessions: Array = []; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", keepSessionOpen: true, onSession(current) { retainedSessions.push(current); if (current === null) throw new Error("owner notification failed"); }, }), ).rejects.toBe(snapshotFailure); expect(retainedSessions).toEqual([session, null]); expect(close).toHaveBeenCalledOnce(); }); it.each(["control-plane checkpoint", "owner checkpoint"] as const)( "aborts consumption and closes the provider when the startup %s never settles", async (stalledBoundary) => { vi.useFakeTimers(); let releaseStream = () => {}; try { const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); const never = new Promise(() => undefined); let markCheckpointStalled = () => {}; const checkpointStalled = new Promise((resolve) => { markCheckpointStalled = resolve; }); let checkpointSignal: AbortSignal | undefined; let controlPlaneCheckpointCount = 0; const checkpointSession: NonNullable< ControlPlanePort["checkpointSession"] > = async (_snapshot, checkpointOptions) => { controlPlaneCheckpointCount += 1; if ( stalledBoundary === "control-plane checkpoint" && controlPlaneCheckpointCount === 2 ) { checkpointSignal = checkpointOptions?.signal; markCheckpointStalled(); await never; } }; let ownerCheckpointCount = 0; const onCheckpoint: NonNullable< ExecuteNativeSessionOptions["onCheckpoint"] > = async (_snapshot, checkpointOptions) => { ownerCheckpointCount += 1; if ( stalledBoundary === "owner checkpoint" && ownerCheckpointCount === 2 ) { checkpointSignal = checkpointOptions?.signal; markCheckpointStalled(); await never; } }; const startTurn = vi.fn(async () => ({ turnId: "turn-checkpoint-stalled", })); const close = vi.fn(async () => { releaseStream(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; }, async *events() { await streamReleased; }, startTurn, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-checkpoint-stalled", cursor: "0", activeTurnId: "turn-checkpoint-stalled", pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "checkpoint-stalled-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const openRun = vi.fn(async () => undefined); const port: ControlPlanePort = { openRun, checkpointSession, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const retainedSessions: Array = []; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1_000, checkpointTimeoutMs: 1, keepSessionOpen: true, onCheckpoint, onSession: (current) => retainedSessions.push(current), }); const rejection = expect(execution).rejects.toThrow( "native session checkpoint timed out after 1ms", ); await checkpointStalled; expect(checkpointSignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(1); await rejection; expect(checkpointSignal?.aborted).toBe(true); expect(openRun).toHaveBeenCalledOnce(); expect(startTurn).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); expect(retainedSessions).toEqual([session, null]); } finally { releaseStream(); vi.useRealTimers(); } }, ); it.each([ "provider result", "completion checkpoint", "control-plane replay", "final event append", "run completion", ] as const)( "bounds post-terminal finalization when %s never settles", async (stalledBoundary) => { vi.useFakeTimers(); try { const never = new Promise(() => undefined); let markFinalizationStalled = () => {}; const finalizationStalled = new Promise((resolve) => { markFinalizationStalled = resolve; }); let stalledSignal: AbortSignal | undefined; let resultCalls = 0; let resultResolved = false; let completionCheckpointCalls = 0; const close = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { resultCalls += 1; if (stalledBoundary === "provider result") { markFinalizationStalled(); return await never; } resultResolved = true; return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "finalization-timeout-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession(_snapshot, operationOptions) { if (stalledBoundary === "completion checkpoint" && resultResolved) { completionCheckpointCalls += 1; stalledSignal = operationOptions?.signal; markFinalizationStalled(); await never; } }, async appendEvent(event, operationOptions) { if ( stalledBoundary === "final event append" && (event as PrpEvent).sourceKind === "control_plane" ) { stalledSignal = operationOptions?.signal; markFinalizationStalled(); return await never; } return { cursor: 1, highestContiguousSourceSeq: (event as PrpEvent).sourceSeq, disposition: "committed", }; }, async replayEvents(_replay, operationOptions) { if (stalledBoundary === "control-plane replay") { stalledSignal = operationOptions?.signal; markFinalizationStalled(); return await never; } return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun(_completion, operationOptions) { if (stalledBoundary === "run completion") { stalledSignal = operationOptions?.signal; markFinalizationStalled(); await never; } }, }; const retainedSessions: Array = []; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 10, keepSessionOpen: true, onSession: (current) => retainedSessions.push(current), }); const rejection = expect(execution).rejects.toThrow( "native session finalization timed out after 10ms", ); await finalizationStalled; if (stalledBoundary !== "provider result") { expect(stalledSignal?.aborted).toBe(false); } await vi.advanceTimersByTimeAsync(20); await rejection; if (stalledBoundary !== "provider result") { expect(stalledSignal?.aborted).toBe(true); } expect(resultCalls).toBe(1); if (stalledBoundary === "completion checkpoint") { expect(completionCheckpointCalls).toBe(1); } expect(close).toHaveBeenCalledOnce(); expect(retainedSessions).toEqual([session, null]); } finally { vi.useRealTimers(); } }, ); it.each(["final event append", "run completion"] as const)( "confirms durable completion when %s commits before its acknowledgement stalls", async (stalledBoundary) => { vi.useFakeTimers(); try { const never = new Promise(() => undefined); let markFinalizationStalled = () => {}; const finalizationStalled = new Promise((resolve) => { markFinalizationStalled = resolve; }); let stalledOnce = false; let stalledSignal: AbortSignal | undefined; const events: PrpEvent[] = []; let durableCompletion: unknown = null; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "durable-finalization-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event, operationOptions) { const appended = structuredClone(event as PrpEvent); const existing = events.find( (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId && candidate.sourceSeq === appended.sourceSeq, ); if (existing === undefined) events.push(appended); if ( stalledBoundary === "final event append" && appended.sourceKind === "control_plane" && !stalledOnce ) { stalledOnce = true; stalledSignal = operationOptions?.signal; markFinalizationStalled(); return await never; } const sourceEvents = events.filter( (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId, ); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), disposition: existing === undefined ? "committed" : "duplicate", }; }, async replayEvents(replay) { const sourceEvents = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId, ); return { events: structuredClone( sourceEvents.filter( (event) => event.sourceSeq > replay.afterSourceSeq, ), ), highestContiguousSourceSeq: highestContiguous(sourceEvents), }; }, async completeRun(completion, operationOptions) { if (durableCompletion === null) { durableCompletion = structuredClone(completion); } else { expect(completion).toEqual(durableCompletion); } if (stalledBoundary === "run completion" && !stalledOnce) { stalledOnce = true; stalledSignal = operationOptions?.signal; markFinalizationStalled(); await never; } }, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 10, }); await finalizationStalled; expect(stalledSignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(10); await expect(execution).resolves.toMatchObject({ result, terminal, nativeEventCount: 3, }); expect(stalledSignal?.aborted).toBe(true); expect(durableCompletion).toMatchObject({ result, terminal }); expect( events.filter((event) => event.sourceKind === "control_plane"), ).toHaveLength(2); } finally { vi.useRealTimers(); } }, ); it.each([ { boundary: "control-plane replay", fault: "typed", admitted: false }, { boundary: "final event append", fault: "typed", admitted: false }, { boundary: "lost final event acknowledgement", fault: "typed", admitted: false, }, { boundary: "control-plane replay", fault: "lookalike", admitted: true }, { boundary: "control-plane replay", fault: "generic", admitted: true }, { boundary: "completion invoked before commit", fault: "typed", admitted: true, }, { boundary: "lost completion acknowledgement", fault: "typed", admitted: true, }, ] as const)( "observes integrity before completion admission without revoking admitted completion (%j)", async ({ boundary, fault, admitted }) => { vi.useFakeTimers(); let releaseBoundary = () => {}; const boundaryReleased = new Promise((resolve) => { releaseBoundary = resolve; }); let markBoundaryReached = () => {}; const boundaryReached = new Promise((resolve) => { markBoundaryReached = resolve; }); const failure = fault === "typed" ? new NativeSessionProtocolIntegrityError( "semantic_input_digest_mismatch", ) : fault === "lookalike" ? Object.assign(new Error("untrusted transport error"), { code: "native_event_replay_conflict", reason: "semantic_input_digest_mismatch", }) : new Error("snapshot temporarily unavailable"); let latchedFailure: Error | null = null; let boundaryBlocked = false; const waitAtBoundary = async () => { if (boundaryBlocked) return; boundaryBlocked = true; markBoundaryReached(); await boundaryReleased; }; const events: PrpEvent[] = []; let durableCompletion: unknown = null; const close = vi.fn(async () => undefined); const resolveResult = vi.fn(async () => ({ result, terminal, turnId: "turn-recovery", })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, result: resolveResult, async snapshot() { if (latchedFailure !== null) throw latchedFailure; return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: `completion-integrity-${boundary}-${fault}`, version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const completeRun = vi.fn( async (completion) => { if (boundary === "completion invoked before commit") await waitAtBoundary(); if (durableCompletion === null) durableCompletion = structuredClone(completion); else expect(completion).toEqual(durableCompletion); if ( boundary === "lost completion acknowledgement" && !boundaryBlocked ) { await waitAtBoundary(); await new Promise(() => undefined); } }, ); const checkpointSession = vi.fn>( async () => undefined, ); const port: ControlPlanePort = { async openRun() {}, checkpointSession, async appendEvent(rawEvent) { const event = structuredClone(rawEvent as PrpEvent); const existing = events.some( (candidate) => candidate.sourceInstanceId === event.sourceInstanceId && candidate.sourceSeq === event.sourceSeq, ); if (!existing) events.push(event); if ( boundary === "final event append" && event.eventType === "run.terminal" ) { await waitAtBoundary(); } if ( boundary === "lost final event acknowledgement" && event.eventType === "run.terminal" && !boundaryBlocked ) { await waitAtBoundary(); await new Promise(() => undefined); } return { cursor: events.length, highestContiguousSourceSeq: highestContiguous( events.filter( (candidate) => candidate.sourceInstanceId === event.sourceInstanceId, ), ), disposition: existing ? "duplicate" : "committed", }; }, async replayEvents(replay) { if ( boundary === "control-plane replay" && replay.sourceInstanceId === "control-recovery" ) { await waitAtBoundary(); } const sourceEvents = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId, ); return { events: structuredClone( sourceEvents.filter( (event) => event.sourceSeq > replay.afterSourceSeq, ), ), highestContiguousSourceSeq: highestContiguous(sourceEvents), }; }, completeRun, }; const outcome = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 10, requireSessionCloseBeforeReturn: true, }).then( (value) => ({ value, error: null }), (error: unknown) => ({ value: null, error }), ); try { await boundaryReached; const checkpointsBeforeFault = checkpointSession.mock.calls.length; if (boundary === "completion invoked before commit") { expect(completeRun).toHaveBeenCalledOnce(); expect(durableCompletion).toBeNull(); } latchedFailure = failure; releaseBoundary(); if ( boundary === "lost completion acknowledgement" || boundary === "lost final event acknowledgement" ) { await vi.advanceTimersByTimeAsync(10); } const settled = await outcome; if (!admitted) { expect(settled.error).toBe(failure); expect(settled.value).toBeNull(); expect(completeRun).not.toHaveBeenCalled(); expect(durableCompletion).toBeNull(); } else { expect(settled.error).toBeNull(); expect(settled.value).toMatchObject({ result, terminal, nativeEventCount: 3, }); expect(durableCompletion).toMatchObject({ result, terminal }); expect(completeRun).toHaveBeenCalledTimes( boundary === "lost completion acknowledgement" ? 2 : 1, ); } expect( events.filter((event) => event.sourceKind === "control_plane"), ).toHaveLength(2); expect(checkpointSession).toHaveBeenCalledTimes(checkpointsBeforeFault); expect(resolveResult).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); } finally { releaseBoundary(); await vi.advanceTimersByTimeAsync(30); await outcome; vi.useRealTimers(); } }, ); it.each([ "provider snapshot", "post-completion checkpoint", "provider usage", ] as const)( "preserves durable completion when %s never settles", async (stalledBoundary) => { vi.useFakeTimers(); try { const never = new Promise(() => undefined); let markEnrichmentStalled = () => {}; const enrichmentStalled = new Promise((resolve) => { markEnrichmentStalled = resolve; }); let stalledSignal: AbortSignal | undefined; let runCompleted = false; const close = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot(snapshotOptions) { if (stalledBoundary === "provider snapshot" && runCompleted) { stalledSignal = snapshotOptions?.signal; markEnrichmentStalled(); return await never; } return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, async usage() { if (stalledBoundary === "provider usage" && runCompleted) { markEnrichmentStalled(); return await never; } return { driverVersion: "2" }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "post-completion-enrichment-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const completeRun = vi.fn(async () => { runCompleted = true; }); const port: ControlPlanePort = { async openRun() {}, async checkpointSession(_snapshot, operationOptions) { if ( stalledBoundary === "post-completion checkpoint" && runCompleted ) { stalledSignal = operationOptions?.signal; markEnrichmentStalled(); await never; } }, async appendEvent(event) { return { cursor: 1, highestContiguousSourceSeq: (event as PrpEvent).sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; const retainedSessions: Array = []; const enrichmentFailures: Array<"checkpoint" | "usage"> = []; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 10, keepSessionOpen: true, onSession: (current) => retainedSessions.push(current), onPostCompletionEnrichmentFailure: ({ stage }) => { enrichmentFailures.push(stage); }, }); await enrichmentStalled; if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(false); await vi.advanceTimersByTimeAsync(10); await expect(execution).resolves.toMatchObject({ result, terminal, providerSessionId: "provider-recovery", driverVersion: "1", usage: null, }); if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(true); expect(completeRun).toHaveBeenCalledOnce(); expect(enrichmentFailures).toEqual([ stalledBoundary === "provider usage" ? "usage" : "checkpoint", ]); if (stalledBoundary === "provider usage") { expect(close).not.toHaveBeenCalled(); expect(retainedSessions).toEqual([session]); } else { expect(close).toHaveBeenCalledOnce(); expect(retainedSessions).toEqual([session, null]); } } finally { vi.useRealTimers(); } }, ); it("contains a consumer rejection when starting the turn fails first", async () => { let markAppendStarted = () => {}; const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); let releaseAppend = () => {}; const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); let appendCommitted = false; const close = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.started"); }, async startTurn() { await appendStarted; throw new Error("start turn failed"); }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(_event, options) { markAppendStarted(); await Promise.race([ appendReleased, new Promise((_resolve, reject) => { const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); if (options?.signal.aborted) rejectAbort(); else options?.signal.addEventListener("abort", rejectAbort, { once: true, }); }), ]); appendCommitted = true; return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" as const, }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await appendStarted; await expect(execution).rejects.toThrow("start turn failed"); expect(close).toHaveBeenCalled(); expect(appendCommitted).toBe(false); releaseAppend(); await new Promise((resolve) => setImmediate(resolve)); expect(appendCommitted).toBe(false); }); it("stops and closes a timed-out consumer even when the caller requested a warm session", async () => { let releaseAppend = () => {}; const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); let releaseTeardown = () => {}; const teardownReleased = new Promise((resolve) => { releaseTeardown = resolve; }); const iteratorTeardown = vi.fn(); let appendCommitted = false; const appendEvent = vi.fn( async (_event: PrpEvent, options?: { signal: AbortSignal }) => { await Promise.race([ appendReleased, new Promise((_resolve, reject) => { const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); if (options?.signal.aborted) rejectAbort(); else options?.signal.addEventListener("abort", rejectAbort, { once: true, }); }), ]); appendCommitted = true; return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" as const, }; }, ); const cancel = vi.fn(() => { releaseTeardown(); return { cleanup: Promise.resolve() }; }); const close = vi.fn(async () => { releaseTeardown(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { try { yield runnerEvent(1, "turn.completed"); } finally { iteratorTeardown(); await teardownReleased; } }, async startTurn() { return { turnId: "turn-recovery" }; }, cancel, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, appendEvent, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; // Reach the pending append before expiring the consumer. A real 1 ms // deadline can expire during setup before the consumer reaches this case. vi.useFakeTimers(); try { const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1, keepSessionOpen: true, }); const rejection = expect(execution).rejects.toThrow( "native session timed out", ); await vi.advanceTimersByTimeAsync(0); expect(appendEvent).toHaveBeenCalledOnce(); await vi.advanceTimersByTimeAsync(1); await rejection; expect(iteratorTeardown).toHaveBeenCalledOnce(); expect(cancel).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalled(); expect(appendEvent).toHaveBeenCalledOnce(); expect(appendCommitted).toBe(false); releaseAppend(); await vi.advanceTimersByTimeAsync(0); expect(appendCommitted).toBe(false); } finally { releaseAppend(); releaseTeardown(); vi.useRealTimers(); } }); it("closes a failed session while retaining an uncancellable event read", async () => { let releaseStream = () => {}; const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); const close = vi.fn(async () => { releaseStream(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; }, async *events() { await streamReleased; yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1, keepSessionOpen: true, }), ).rejects.toThrow("native session timed out"); expect(close).toHaveBeenCalledOnce(); }); it("commits cancellation before bounding failed provider cleanup", async () => { let releaseStream = () => {}; const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); let releaseCancellation = () => {}; const cancellationReleased = new Promise((resolve) => { releaseCancellation = resolve; }); const interrupt = vi.fn(() => cancellationReleased); let cancellationCommitted = false; const cancel = vi.fn(() => { cancellationCommitted = true; return { cleanup: cancellationReleased }; }); const close = vi.fn(async () => { releaseStream(); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { await streamReleased; yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, interrupt, cancel, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1, keepSessionOpen: true, }); await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); expect(interrupt).not.toHaveBeenCalled(); expect(cancel).toHaveBeenCalledOnce(); await expect(execution).rejects.toThrow("native session timed out"); expect(cancellationCommitted).toBe(true); releaseCancellation(); }); it("bounds failure when iterator teardown and provider close never settle", async () => { const never = new Promise(() => undefined); let releaseClose = () => {}; const pendingClose = new Promise((resolve) => { releaseClose = resolve; }); const close = vi.fn(() => pendingClose); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; }, async *events() { await never; yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1, keepSessionOpen: true, }), ).rejects.toThrow("native session timed out"); expect(close).toHaveBeenCalledOnce(); releaseClose(); await pendingClose; }); it("preserves durable success when provider close never settles", async () => { let releaseClose = () => {}; const pendingClose = new Promise((resolve) => { releaseClose = resolve; }); const close = vi.fn(() => pendingClose); const completeRun = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }, }; }, async openSession() { return session; }, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ result, terminal }); expect(completeRun).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); releaseClose(); await pendingClose; }); it.each([false, true])( "waits for a required backend checkpoint close when enrichment failure=%s", async (enrichmentFails) => { let releaseClose = () => {}; const pendingClose = new Promise((resolve) => { releaseClose = resolve; }); const close = vi.fn(() => pendingClose); let runCompleted = false; let enrichmentFailureObserved = false; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { // Exercise only the best-effort enrichment snapshot after the // control plane has durably committed the run result. if (enrichmentFails && runCompleted) { enrichmentFailureObserved = true; throw new Error("checkpoint enrichment failed"); } return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }, }; }, async openSession() { return session; }, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() { runCompleted = true; }, }; let resolved = false; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", requireSessionCloseBeforeReturn: true, }).then((value) => { resolved = true; return value; }); await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); await new Promise((resolve) => setTimeout(resolve, 150)); expect(resolved).toBe(false); releaseClose(); await expect(execution).resolves.toMatchObject({ result, terminal }); expect(resolved).toBe(true); expect(enrichmentFailureObserved).toBe(enrichmentFails); }, ); it.each([ { typed: true, closeFails: false }, { typed: true, closeFails: true }, { typed: false, closeFails: true }, { typed: true, closeFails: true, startupRace: true }, ])( "preserves the primary execution failure through required cleanup (%j)", async ({ typed, closeFails, startupRace = false }) => { const failure = typed ? new NativeSessionProtocolIntegrityError( "semantic_input_digest_mismatch", ) : Object.assign(new Error("ordinary provider connection failed"), { code: "native_event_replay_conflict", recovery: "operator_required", }); const closeFailure = new NativeSessionCloseUnrecoverableError(); let observeClose = () => {}; const closeStarted = new Promise((resolve) => { observeClose = resolve; }); const close = vi.fn(async () => { observeClose(); if (closeFails) throw closeFailure; }); const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; const session: NativeSession = { identity: () => identity, capabilities: async () => capabilities, async *events() { throw failure; }, startTurn: async () => { if (startupRace) { await closeStarted; throw new Error( "provider_transport_failed: startup raced with close", ); } return { turnId: "turn-recovery" }; }, result: vi.fn(async () => null), snapshot: async () => ({ backendKind: "mock", sessionId: "driver-integrity", identity, providerSessionId: "provider-integrity", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }), close, }; const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: `integrity-${typed}-${closeFails}-${startupRace}`, version: "1", capabilities, }), openSession: vi.fn(async () => session), }; const events: PrpEvent[] = []; const port: ControlPlanePort = { openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async (event) => { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0, }), completeRun: vi.fn(async () => {}), }; const onSession = vi.fn(); const onSessionAdmission = vi.fn(async () => {}); const options: ExecuteNativeSessionOptions = { onSessionAdmission, input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", onSession, requireSessionCloseBeforeReturn: true, timeoutMs: 900_000, }; await expect(executeNativeSession(options)).rejects.toBe( failure, ); if (closeFails) expect(failure).toHaveProperty("cleanupError", closeFailure); expect(close).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenLastCalledWith(null); expect(port.completeRun).not.toHaveBeenCalled(); expect(session.result).not.toHaveBeenCalled(); expect(events.some((event) => event.sourceKind === "runner")).toBe(false); if (closeFails) { await expect(executeNativeSession(options)).rejects.toBeInstanceOf( NativeSessionCleanupQuarantinedError, ); expect(backend.openSession).toHaveBeenCalledOnce(); expect(onSessionAdmission).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); } }, ); it("collects a failed run's private instructions only after its owned provider close joins", async () => { const scopedIdentity = { ...identity, companyId: "instruction-close-company", runId: "instruction-close-run" }; let finishClose!: () => void; const closed = new Promise((resolve) => { finishClose = resolve; }); const order: string[] = []; const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const session: NativeSession = { identity: () => scopedIdentity, capabilities: async () => capabilities, async *events() { throw new Error("provider failed after editing instructions"); }, startTurn: async () => ({ turnId: "instruction-turn" }), result: async () => null, snapshot: async () => ({ backendKind: "runner", sessionId: "instruction-session", identity: scopedIdentity, providerSessionId: "instruction-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }), close: async () => { order.push("closing"); await closed; order.push("stopped"); }, }; const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "runner", name: "instruction-close-test", version: "1", capabilities }), openSession: async () => session, }; const controlPlane: ControlPlanePort = { openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: async () => {}, }; const options = { input: { ...input, binding: { ...input.binding, companyId: scopedIdentity.companyId, runId: scopedIdentity.runId } }, backend, controlPlane, runnerInstanceId: "instruction-runner", controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true, onSessionClosed: async () => { order.push("collected"); }, }; const execution = executeNativeSession(options); const failed = expect(execution).rejects.toThrow("provider failed after editing instructions"); await vi.waitFor(() => expect(order).toEqual(["closing"])); finishClose(); await failed; expect(order).toEqual(["closing", "stopped", "collected"]); }); it("retires only the terminated remote resource, including two sandboxes for one run", async () => { const scopedIdentity = { ...identity, companyId: "remote-stop-company", runId: "remote-stop-run" }; const binding = { ...scopedIdentity, remoteCleanupScope: "first-sandbox" }; const scopedInput = { ...input, binding: { ...input.binding, companyId: scopedIdentity.companyId, runId: scopedIdentity.runId } }; const failure = new NativeSessionCloseUnrecoverableError(); const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const session: NativeSession = { identity: () => scopedIdentity, capabilities: async () => capabilities, async *events() { throw new Error("cancelled remote transport"); }, startTurn: async () => ({ turnId: "remote-turn" }), result: async () => null, async snapshot() { return { backendKind: "mock", sessionId: "stop-session", identity: this.identity(), providerSessionId: "stop-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(async () => { throw failure; }), }; const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "remote", name: "remote-stop-test", version: "1", capabilities }), openSession: vi.fn(async () => session), }; const controlPlane: ControlPlanePort = { openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: vi.fn(async () => {}), }; const options = { input: scopedInput, backend, controlPlane, runnerInstanceId: "remote-runner", controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true, remoteCleanupScope: binding.remoteCleanupScope }; await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "cancelled remote transport", cleanupError: failure }); expect(completeTerminatedRemoteNativeSessionCleanup({ ...binding, runId: "other-run" })).toBe(true); expect(completeTerminatedRemoteNativeSessionCleanup({ ...binding, companyId: "other-company" })).toBe(true); await expect(executeNativeSession(options)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(backend.openSession).toHaveBeenCalledOnce(); // A separate sandbox can start without inheriting this process quarantine. const independent = { ...scopedIdentity, sessionId: "other-sandbox-session" }; const independentSession = { ...session, identity: () => independent }; const independentBackend = { ...backend, openSession: vi.fn(async () => independentSession) }; await expect(executeNativeSession({ ...options, remoteCleanupScope: "other-sandbox", input: { ...scopedInput, binding: { ...scopedInput.binding, runId: independent.runId } }, backend: independentBackend })).rejects.toMatchObject({ message: "cancelled remote transport", cleanupError: failure }); expect(independentBackend.openSession).toHaveBeenCalledOnce(); expect(completeTerminatedRemoteNativeSessionCleanup(binding)).toBe(true); // Same company/run, different sandbox: its quarantine must remain intact. await expect(executeNativeSession({ ...options, remoteCleanupScope: "other-sandbox", backend: independentBackend })).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(independentBackend.openSession).toHaveBeenCalledOnce(); // Reopening is now possible; the old failure/result was never rewritten. await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "cancelled remote transport", cleanupError: failure }); expect(backend.openSession).toHaveBeenCalledTimes(2); expect(controlPlane.completeRun).not.toHaveBeenCalled(); completeTerminatedRemoteNativeSessionCleanup(binding); completeTerminatedRemoteNativeSessionCleanup({ ...binding, remoteCleanupScope: "other-sandbox" }); }); it("retires local quarantine only for the stopped run and runner instance", async () => { const scopedIdentity = { ...identity, companyId: "local-stop-company", runId: "local-stop-run" }; const binding = { ...scopedIdentity, runnerInstanceId: "local-runner" }; const scopedInput = { ...input, binding: { ...input.binding, companyId: scopedIdentity.companyId, runId: scopedIdentity.runId } }; const failure = new NativeSessionCloseUnrecoverableError(); const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const session: NativeSession = { identity: () => scopedIdentity, capabilities: async () => capabilities, async *events() { throw new Error("local transport stopped"); }, startTurn: async () => ({ turnId: "local-turn" }), result: async () => null, async snapshot() { return { backendKind: "mock", sessionId: "stop-session", identity: this.identity(), providerSessionId: "stop-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(async () => { throw failure; }), }; const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "runner", name: "local-stop-test", version: "1", capabilities }), openSession: vi.fn(async () => session), }; const controlPlane: ControlPlanePort = { openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: vi.fn(async () => {}), }; const options = { input: scopedInput, backend, controlPlane, runnerInstanceId: binding.runnerInstanceId, controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true }; await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "local transport stopped", cleanupError: failure }); completeTerminatedLocalNativeSessionCleanup({ ...binding, companyId: "other-company" }); completeTerminatedLocalNativeSessionCleanup({ ...binding, runId: "other-run" }); expect(completeTerminatedLocalNativeSessionCleanup({ ...binding, runnerInstanceId: "other-runner" })).toBe(false); await expect(executeNativeSession(options)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(backend.openSession).toHaveBeenCalledOnce(); expect(completeTerminatedLocalNativeSessionCleanup(binding)).toBe(true); await expect(executeNativeSession(options)).rejects.toMatchObject({ message: "local transport stopped", cleanupError: failure }); expect(backend.openSession).toHaveBeenCalledTimes(2); expect(controlPlane.completeRun).not.toHaveBeenCalled(); completeTerminatedLocalNativeSessionCleanup(binding); }); it("announces every cleanup quarantine with its cause when it is created", async () => { const quarantined = vi.fn(); const unsubscribe = onNativeSessionCleanupQuarantined(quarantined); const scopedIdentity = { ...identity, companyId: "announce-company", runId: "announce-run", issueId: "announce-issue", sessionId: "announce-session" }; const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const settlement = { runnerSuspended: false, suspensionState: { commandStatus: "failed", runnerLifecycle: "suspended" } }; const session: NativeSession = { identity: () => scopedIdentity, capabilities: async () => capabilities, async *events() { throw new Error("announce transport stopped"); }, startTurn: async () => ({ turnId: "announce-turn" }), result: async () => null, async snapshot() { return { backendKind: "mock", sessionId: "announce-driver", identity: scopedIdentity, providerSessionId: "announce-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(async () => { throw new NativeSessionCloseUnrecoverableError(settlement); }), }; const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "runner", name: "announce-backend", version: "1", capabilities }), openSession: vi.fn(async () => session), }; try { await expect(executeNativeSession({ input: { ...input, binding: { ...input.binding, companyId: scopedIdentity.companyId, runId: scopedIdentity.runId, issueId: scopedIdentity.issueId }, session: { ...input.session, normalizedSessionId: scopedIdentity.sessionId } }, backend, runnerInstanceId: "announce-runner", controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true, controlPlane: { openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: async () => {} }, })).rejects.toThrow("announce transport stopped"); expect(quarantined).toHaveBeenCalledOnce(); expect(quarantined).toHaveBeenCalledWith({ companyId: scopedIdentity.companyId, runId: scopedIdentity.runId, issueId: scopedIdentity.issueId, sessionId: scopedIdentity.sessionId, operatorRecoveryRequired: true, cause: expect.stringContaining("native_session_close_unrecoverable"), settlement, }); } finally { unsubscribe(); completeTerminatedLocalNativeSessionCleanup({ companyId: scopedIdentity.companyId, runId: scopedIdentity.runId, runnerInstanceId: "announce-runner" }); } }); it("keeps an unrecoverable close from quarantining unrelated tasks of the same backend", async () => { const companyId = "unrecoverable-scope-company"; const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const scoped = (suffix: string, overrides: { issueId?: string; sessionId?: string } = {}) => { const scopedIdentity = { ...identity, companyId, runId: `run-${suffix}`, issueId: overrides.issueId ?? `issue-${suffix}`, sessionId: overrides.sessionId ?? `session-${suffix}` }; const close = vi.fn(async () => {}); const session: NativeSession = { identity: () => scopedIdentity, capabilities: async () => capabilities, async *events() { throw new Error(`transport ${suffix} stopped`); }, startTurn: async () => ({ turnId: `turn-${suffix}` }), result: async () => null, async snapshot() { return { backendKind: "mock", sessionId: `driver-${suffix}`, identity: scopedIdentity, providerSessionId: `provider-${suffix}`, cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close, }; const backend: NativeSessionBackend = { // Every task shares the backend kind/name, as all Codex runs of a company do. descriptor: async () => ({ kind: "runner", name: "shared-codex-backend", version: "1", capabilities }), openSession: vi.fn(async () => session), }; const controlPlane: ControlPlanePort = { openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: vi.fn(async () => {}), }; const options = { input: { ...input, binding: { ...input.binding, companyId, runId: scopedIdentity.runId, issueId: scopedIdentity.issueId }, session: { ...input.session, normalizedSessionId: scopedIdentity.sessionId } }, backend, controlPlane, runnerInstanceId: "shared-runner", controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true, }; return { scopedIdentity, close, backend, options }; }; const quarantined = scoped("a"); quarantined.close.mockRejectedValue(new NativeSessionCloseUnrecoverableError()); await expect(executeNativeSession(quarantined.options)).rejects.toMatchObject({ message: "transport a stopped", cleanupError: expect.any(NativeSessionCloseUnrecoverableError) }); // A different task and conversation shares no resource with the retained owner. const unrelated = scoped("b"); await expect(executeNativeSession(unrelated.options)).rejects.toThrow("transport b stopped"); expect(unrelated.backend.openSession).toHaveBeenCalledOnce(); // The retained owner may still write its own task and conversation: both stay gated. for (const sameResource of [scoped("c", { issueId: quarantined.scopedIdentity.issueId }), scoped("d", { sessionId: quarantined.scopedIdentity.sessionId })]) { await expect(executeNativeSession(sameResource.options)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(sameResource.backend.openSession).not.toHaveBeenCalled(); } expect(completeTerminatedLocalNativeSessionCleanup({ companyId, runId: quarantined.scopedIdentity.runId, runnerInstanceId: "shared-runner" })).toBe(true); }); it("propagates an exhausted required backend checkpoint close", async () => { vi.useFakeTimers(); try { const closeFailure = new Error("required remote checkpoint close failed"); const onSessionClosed = vi.fn(async () => {}); const close = vi.fn(({ reason }: { reason: string }) => reason === "native session quarantined cleanup recovery" ? Promise.resolve() : Promise.reject(closeFailure), ); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true, }, }; }, async openSession() { return session; }, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", requireSessionCloseBeforeReturn: true, onSessionClosed, }), ).rejects.toThrow(closeFailure); await vi.advanceTimersByTimeAsync(3_000); await execution; expect(close).toHaveBeenCalledTimes(5); expect(onSessionClosed).not.toHaveBeenCalled(); } finally { vi.useRealTimers(); } }); it("closes after a synchronous governed-wait probe returns no result", async () => { const resolveGovernedWait = vi.fn(() => null); const lifecycle: string[] = []; const close = vi.fn(async () => { lifecycle.push("closed"); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "item.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: "turn-recovery", pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 5, resolveGovernedWait, }); await expect(execution).rejects.toThrow("before a turn terminal fact"); expect(resolveGovernedWait).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalled(); expect(lifecycle).toEqual(["closed"]); }); it("settles an intentional governed park through the actual Codex session facade", async () => { const transport = new FakeCodexTransport(); const request = transport.request.bind(transport); transport.request = async (method, params) => { const response = await request(method, params); if (method === "turn/interrupt") setImmediate(() => transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "interrupted", error: null, items: [] }, })); return response; }; const executionInput = { ...input, workspace: { ...input.workspace, cwd: WORKSPACE } }; const backend = createCodexNativeSessionBackend(executionInput, { runnerInstanceId: "runner-recovery", transportFactory: () => transport, }); const events: PrpEvent[] = []; const deferredEventTypes: string[] = []; const completeRun = vi.fn(async () => undefined); const execution = executeNativeSession({ input: executionInput, backend, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 5_000, controlPlane: { async openRun() {}, async checkpointSession() {}, completeRun, async appendEvent(event: PrpEvent, options) { if (options?.deferPostCommitObservers) deferredEventTypes.push(event.eventType); events.push(structuredClone(event)); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, }, resolveGovernedWait: ({ event }) => event.eventType === "item.completed" && event.payload.kind === "agentMessage" ? yieldedResult : null, }); const outcome = expect(execution).resolves.toMatchObject({ result: yieldedResult, terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" } }); await vi.waitFor(() => expect(transport.calls.some(call => call.method === "turn/start")).toBe(true)); transport.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); transport.push("item/completed", { threadId: "thread-1", turnId: "turn-1", item: { id: "waiting-message", type: "agentMessage", channel: "final", text: "The human-only question is saved." } }); await outcome; expect(events.find(event => event.eventType === "turn.interrupted")?.payload).toMatchObject({ error: null, status: "interrupted" }); expect(deferredEventTypes).toEqual(["turn.interrupted"]); expect(events.some(event => event.eventType === "item.completed")).toBe(true); expect(completeRun).toHaveBeenCalledOnce(); expect(transport.calls.filter(call => call.method === "turn/start")).toHaveLength(1); }); it("commits a proved governed wait without waiting for abort-insensitive provider cleanup", async () => { vi.useFakeTimers(); try { const lifecycle: string[] = []; let cancellationSignal: AbortSignal | undefined; const cancel = vi.fn(({ signal }: { signal: AbortSignal }) => { cancellationSignal = signal; lifecycle.push("cancelled"); return { cleanup: new Promise(() => undefined) }; }); const close = vi.fn(async () => { lifecycle.push("closed"); }); const events: PrpEvent[] = []; const retainedSessions: Array = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "item.completed"); expect(cancel).toHaveBeenCalledOnce(); yield runnerEvent(2, "turn.interrupted", { providerTerminalObserved: true }); }, async startTurn() { return { turnId: "turn-recovery" }; }, cancel, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: null, activeTurnId: "turn-recovery", pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 5, resolveGovernedWait: () => yieldedResult, keepSessionOpen: true, onSession: (current) => retainedSessions.push(current), }); let settled = false; const completed = execution.then(value => { settled = true; return value; }); void completed.catch(() => undefined); await vi.advanceTimersByTimeAsync(9_900); expect(settled).toBe(false); await vi.advanceTimersByTimeAsync(200); await expect(completed).resolves.toMatchObject({ result: yieldedResult }); expect(cancel).toHaveBeenCalledOnce(); expect(cancellationSignal?.aborted).toBe(true); expect(close).toHaveBeenCalledOnce(); expect(lifecycle).toEqual(["cancelled", "closed"]); expect(retainedSessions.at(-1)).toBeNull(); expect(events.map((event) => event.eventType)).toEqual([ "item.completed", "turn.interrupted", "run.result.accepted", "run.terminal", ]); } finally { vi.useRealTimers(); } }); it("uses the execution timeout when a completion report has no provider terminal", async () => { const lifecycle: string[] = []; let releaseProvider = () => {}; const providerReleased = new Promise((resolve) => { releaseProvider = resolve; }); const cancel = vi.fn(() => { lifecycle.push("cancelled"); return { cleanup: new Promise((resolve) => setTimeout(() => { releaseProvider(); resolve(); }, 150), ), }; }); const providerResult = vi.fn(async () => null); const close = vi.fn(async () => { lifecycle.push("closed"); }); const events: PrpEvent[] = []; const retainedSessions: Array = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "run.result.proposed", result); await providerReleased; }, async startTurn() { return { turnId: "turn-recovery" }; }, cancel, result: providerResult, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: "turn-recovery", pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "semantic-result-terminal-stall-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(structuredClone(event as PrpEvent)); const sourceEvents = events.filter( (candidate) => candidate.sourceInstanceId === event.sourceInstanceId, ); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), disposition: "committed", }; }, async replayEvents(replay) { const sourceEvents = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId, ); return { events: structuredClone( sourceEvents.filter( (event) => event.sourceSeq > replay.afterSourceSeq, ), ), highestContiguousSourceSeq: highestContiguous(sourceEvents), }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 20, keepSessionOpen: true, onSession: (current) => retainedSessions.push(current), }), ).rejects.toThrow("native session timed out after 20ms"); expect(cancel).toHaveBeenCalledWith({ reason: "Native session event consumption failed.", signal: expect.any(AbortSignal), failureCleanup: true, }); expect(providerResult).not.toHaveBeenCalled(); expect(events.map((event) => event.eventType)).toEqual([ "run.result.proposed", ]); expect(lifecycle).toContain("cancelled"); expect(close).toHaveBeenCalledOnce(); expect(retainedSessions.at(-1)).toBeNull(); }); it.each([false, true])("retains a reusable session while terminal teardown settles (governed=%s)", async (governed) => { const close = vi.fn(async () => undefined); const retainedSessions: Array = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { try { yield { ...runnerEvent(1, governed ? "item.completed" : "run.result.proposed", governed ? { kind: "dynamicToolCall" } : result), turnId: "turn-recovery" }; yield { ...runnerEvent(2, governed ? "turn.interrupted" : "turn.completed"), turnId: "turn-recovery", }; } finally { await new Promise((resolve) => setTimeout(resolve, 150)); } }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "2", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, cancel: () => ({ cleanup: Promise.resolve() }), close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "semantic-terminal-subscription-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: governed ? () => yieldedResult : undefined, keepSessionOpen: true, onSession: (current) => retainedSessions.push(current), }), ).resolves.toMatchObject({ result: governed ? yieldedResult : result }); expect(close).not.toHaveBeenCalled(); expect(retainedSessions).toEqual([session]); }); it.each([ ["turn.completed", "succeeded"], ["turn.failed", "failed"], ["turn.cancelled", "cancelled"], ["turn.interrupted", "cancelled"], ["stream.closed", null], ] as const)("persists a delayed final answer before settling %s", async (eventType, runTerminalState) => { vi.useFakeTimers(); try { let releasePersistence!: () => void; const persistence = new Promise((resolve) => { releasePersistence = resolve; }); let persistingAnswer = false; const cancel = vi.fn(() => ({ cleanup: Promise.resolve() })); const close = vi.fn(async () => undefined); const events: PrpEvent[] = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "run.result.proposed", result); await new Promise((resolve) => setTimeout(resolve, 6_000)); yield runnerEvent(2, "item.completed", { kind: "agentMessage", channel: "final", text: "Final response.", }); if (eventType !== "stream.closed") yield runnerEvent(3, eventType); }, async startTurn() { return { turnId: "turn-recovery" }; }, cancel, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "3", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "semantic-result-final-response-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { if (event.eventType === "item.completed") { persistingAnswer = true; await persistence; } events.push(structuredClone(event as PrpEvent)); const sourceEvents = events.filter( (candidate) => candidate.sourceInstanceId === event.sourceInstanceId, ); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), disposition: "committed", }; }, async replayEvents(replay) { const sourceEvents = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId, ); return { events: structuredClone( sourceEvents.filter( (event) => event.sourceSeq > replay.afterSourceSeq, ), ), highestContiguousSourceSeq: highestContiguous(sourceEvents), }; }, async completeRun() {}, }; let completed = false; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }).then((value) => { completed = true; return value; }); await vi.waitFor(() => expect(events).toHaveLength(1)); await vi.advanceTimersByTimeAsync(6_001); expect(persistingAnswer).toBe(true); expect(completed).toBe(false); expect(events.map((event) => event.eventType)).toEqual(["run.result.proposed"]); expect(cancel).not.toHaveBeenCalled(); releasePersistence(); if (eventType === "stream.closed") { await expect(execution).rejects.toThrow("native event stream closed before a turn terminal fact"); expect(events.map((event) => event.eventType)).toEqual(["run.result.proposed", "item.completed"]); return; } await expect(execution).resolves.toMatchObject({ result, terminal: { runTerminalState } }); expect(cancel).not.toHaveBeenCalled(); expect(close).toHaveBeenCalledOnce(); expect(events.map((event) => event.eventType)).toEqual([ "run.result.proposed", "item.completed", eventType, "run.result.accepted", "run.terminal", ]); } finally { vi.useRealTimers(); } }); it("rejects a mismatched checkpoint before it mutates control-plane state", async () => { const openRun = vi.fn(async () => undefined); const checkpointSession = vi.fn(async () => undefined); const openSession = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { openRun, async loadSessionCheckpoint() { return { backendKind: "mock", sessionId: "driver-recovery", identity: { ...identity, companyId: "other-company" }, }; }, checkpointSession, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).rejects.toThrow("native_session_checkpoint_binding_mismatch"); expect(openRun).not.toHaveBeenCalled(); expect(checkpointSession).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); }); it("rejects a mismatched existing session before opening control-plane state", async () => { const openRun = vi.fn(async () => undefined); const attachRun = vi.fn(async () => undefined); const existingSession: NativeSession = { identity: () => ({ ...identity, companyId: "other-company" }), async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, attachRun, async *events() {}, async startTurn() { return { turnId: "unexpected" }; }, async result() { return null; }, async snapshot() { throw new Error("unexpected snapshot"); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "existing-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected open"); }, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", existingSession, }), ).rejects.toThrow("native_session_attach_binding_mismatch"); expect(openRun).not.toHaveBeenCalled(); expect(attachRun).not.toHaveBeenCalled(); }); it.each(["codex", "copilot"] as const)("quarantines a retained %s session when attachment partially mutates then fails", async (provider) => { const qualified = resolveQualifiedAcpxProfile("copilot", "selected-model"); const executionInput: NativeExecutionInputV1 = provider === "copilot" ? { ...input, session: { ...input.session, driverKind: "acpx_runtime" }, provider: { kind: "acpx", agent: "copilot", model: "selected-model", permissionPolicy: "interactive", profile: { driverKind: qualified.driverKind, protocolVersion: qualified.protocolVersion, acpxVersion: qualified.acpxVersion, agent: qualified.agent, agentProfileVersion: qualified.agentProfileVersion, agentServerPackage: qualified.agentServerPackage, agentServerVersion: qualified.agentServerVersion, agentRuntimePackage: qualified.agentRuntimePackage, agentRuntimeVersion: qualified.agentRuntimeVersion, commandDigest: qualified.commandDigest, } }, } : input; const attachmentFailure = new Error("provider attachment failed"); const openRun = vi.fn(async () => undefined); let retainedIdentity = { ...identity, runId: "run-previous" }; const attachRun = vi.fn(async (input: { identity: NativeRunIdentity }) => { retainedIdentity = structuredClone(input.identity); throw attachmentFailure; }); const close = vi.fn(() => new Promise(() => {})); const startTurn = vi.fn(async () => ({ turnId: "unexpected" })); const onSession = vi.fn(); const existingSession: NativeSession = { identity: () => structuredClone(retainedIdentity), async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, attachRun, async *events() {}, startTurn, async result() { return null; }, async snapshot() { throw new Error("unexpected snapshot"); }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: `attachment-failure-backend-${provider}`, version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected open"); }, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", existingSession, onSession, }), ).rejects.toBe(attachmentFailure); // The attaching run's own contract travels with the attachment. const completionContract = { revision: "1", criterionIds: ["objective"] }; expect(attachRun).toHaveBeenCalledWith(provider === "copilot" ? { identity, completionContract, currentRunGrant: { runtimeContext: null, instructions: expect.any(String) } } : { identity, completionContract }); expect(retainedIdentity).toEqual(identity); expect(onSession).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenCalledWith(null); expect(close).toHaveBeenCalledWith({ reason: "native session attachment failed", }); expect(openRun).not.toHaveBeenCalled(); expect(startTurn).not.toHaveBeenCalled(); }); it("quarantines an attached session when control-plane run admission fails", async () => { const admissionFailure = new Error("control-plane admission failed"); const openRun = vi.fn(async () => { throw admissionFailure; }); const attachRun = vi.fn(async () => undefined); const close = vi.fn(async () => undefined); const startTurn = vi.fn(async () => ({ turnId: "unexpected" })); const onSession = vi.fn(); const existingSession: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, attachRun, async *events() {}, startTurn, async result() { return null; }, async snapshot() { throw new Error("unexpected snapshot"); }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "control-plane-admission-failure-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected open"); }, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", existingSession, onSession, }), ).rejects.toBe(admissionFailure); expect(attachRun).toHaveBeenCalledWith({ identity, completionContract: { revision: "1", criterionIds: ["objective"] } }); expect(openRun).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenCalledWith(null); expect(close).toHaveBeenCalledWith({ reason: "native control-plane run admission failed", }); expect(startTurn).not.toHaveBeenCalled(); }); it("rejects checkpoint adoption when the requested session id is absent", async () => { const openRun = vi.fn(async () => undefined); const openSession = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, }; const port: ControlPlanePort = { openRun, async loadSessionCheckpoint() { return { backendKind: "mock", sessionId: "driver-other-session", identity: { ...identity, sessionId: "other-session" }, }; }, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input: { ...input, session: { ...input.session, normalizedSessionId: null }, }, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).rejects.toThrow("native_session_checkpoint_binding_mismatch"); expect(openRun).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); }); it("proves required provider recovery before re-opening the durable run", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-unrecoverable", identity, providerSessionId: "provider-unrecoverable", providerRecoveryPolicy: "same_session_only", cursor: "0", activeTurnId: "turn-unrecoverable", pendingRuntimeRequests: [], lineage: [], }; const openRun = vi.fn(async () => undefined); const completeRun = vi.fn(async () => undefined); const recoverSession = vi.fn(async () => ({ recovered: false as const, reason: "provider session no longer exists", })); const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); }); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, recoverSession, }; const port: ControlPlanePort = { openRun, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).rejects.toThrow( "native_session_recovery_failed: provider session no longer exists", ); expect(recoverSession).toHaveBeenCalledOnce(); expect(openRun).not.toHaveBeenCalled(); expect(completeRun).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); }); it("types a runner-proven incompatible session when same-session recovery is required", async () => { for (const providerRecoveryPolicy of ["same_session_only", "allow_replacement_after_governed_wait"] as const) { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-incompatible", identity, providerSessionId: "provider-incompatible", providerRecoveryPolicy, cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const openRun = vi.fn(async () => undefined); const openSession = vi.fn(); const openReplacementSession = vi.fn(); const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: "incompatible", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), openSession, openReplacementSession, recoverSession: async () => ({ recovered: false, reason: "saved launch profile changed", failureClass: PROVIDER_STATE_INCOMPATIBLE }), }; const port: ControlPlanePort = { openRun, appendEvent: vi.fn(), completeRun: vi.fn(), loadSessionCheckpoint: async () => structuredClone(checkpoint), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), }; const failure = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }).catch((error: unknown) => error); expect(failure, providerRecoveryPolicy).toBeInstanceOf(NativeProviderStateIncompatibleError); expect((failure as Error).message).toBe("native_session_recovery_failed: saved launch profile changed"); expect(openRun).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); expect(openReplacementSession).not.toHaveBeenCalled(); } }); // Staging class P after #20757: a follow-up after a governed (approval) wait // rebinds the previous run's session, but the new sandbox holds no saved // Runner state for it. The follow-up has not started a turn, so a fresh // provider session cannot repeat provider work or a tool side effect; only // the earlier conversation context is lost. The run continues in a fresh // session with the full task, the answer and the stored handoff. describe("a governed-wait follow-up whose saved provider state is gone", () => { const approval = { interactionId: "interaction-approval", kind: "request_confirmation" as const, response: { outcome: "accepted", note: "APPROVED_BY_BOARD" } }; const followUpInput = { ...input, task: { ...input.task, prompt: "FULL_TASK_CONTEXT\nAPPROVAL_FOLLOW_UP" }, interactionResponses: [approval], }; const reboundCheckpoint = (): PersistedNativeSession => ({ backendKind: "mock", driverKind: "codex_app_server", sessionId: "driver-lost", identity, providerSessionId: "provider-lost", providerRecoveryPolicy: "allow_replacement_after_governed_wait", cursor: null, semanticResult: null, terminal: null, activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], dispositionOnlyRecoveryConsumed: false, dispositionOnlyRecoveryTurnId: null, lineage: [], }); const stateUnavailable: NativeSessionRecoveryResult = { recovered: false, failureClass: PROVIDER_STATE_INCOMPATIBLE, resumeStateUnavailable: true, reason: "Error: NativeRunnerResumeStateUnavailableError: PRP provider resume state is unavailable", }; function harness(options: { checkpoint?: PersistedNativeSession; recovery?: Awaited>>; proof?: boolean | null | "rejects" | "hangs"; replay?: PrpEvent[]; } = {}) { const startTurn = vi.fn(async () => ({ turnId: "turn-fresh" })); const replacementSnapshot: PersistedNativeSession = { ...reboundCheckpoint(), sessionId: "driver-fresh", providerSessionId: "provider-fresh", }; const replacementSession: NativeSession = { identity: () => identity, capabilities: async () => ({ resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }), async *events() { yield runnerEvent(1, "turn.completed"); }, startTurn, result: async () => ({ result, terminal, turnId: "turn-fresh" }), snapshot: async () => structuredClone(replacementSnapshot), close: async () => {}, }; const openReplacementSession = vi.fn(async () => replacementSession); const openSession = vi.fn(async () => { throw new Error("replacement seam must be used"); }); const recoverSession = vi.fn(async () => options.recovery ?? stateUnavailable); const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: "governed-follow-up", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), openSession, openReplacementSession, recoverSession, }; const events: PrpEvent[] = []; const provesNoRunProviderWork = vi.fn(async ({ signal }: { signal: AbortSignal }) => { if (options.proof === "rejects") throw new Error("database unavailable"); if (options.proof === "hangs") return await new Promise((_resolve, reject) => signal.addEventListener("abort", () => reject(signal.reason), { once: true })); return options.proof === true; }); const openRun = vi.fn(async () => undefined); const replay = options.replay ?? []; const controlPlane: ControlPlanePort = { openRun, loadSessionCheckpoint: async () => structuredClone(options.checkpoint ?? reboundCheckpoint()), checkpointSession: async () => {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed" }; }, replayEvents: async ({ afterSourceSeq }) => { const page = replay.filter(event => event.sourceSeq > afterSourceSeq); return { events: page, highestContiguousSourceSeq: highestContiguous(page) }; }, completeRun: async () => {}, ...(options.proof === null ? {} : { provesNoRunProviderWork }), }; const onContinuityBreak = vi.fn(async () => undefined); const getFreshSessionHandoff = vi.fn(async () => "FRESH_HANDOFF: prior conversation summary"); const run = () => executeNativeSession({ input: followUpInput, backend, controlPlane, getFreshSessionHandoff, onContinuityBreak, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); return { run, startTurn, openReplacementSession, openSession, recoverSession, provesNoRunProviderWork, openRun, onContinuityBreak, getFreshSessionHandoff }; } it("continues in a fresh provider session with the task, the answer and the stored handoff", async () => { const h = harness({ proof: true }); await expect(h.run()).resolves.toMatchObject({ providerSessionId: "provider-fresh" }); expect(h.recoverSession).toHaveBeenCalledOnce(); expect(h.provesNoRunProviderWork).toHaveBeenCalledOnce(); expect(h.openReplacementSession).toHaveBeenCalledOnce(); expect(h.openSession).not.toHaveBeenCalled(); expect(h.startTurn).toHaveBeenCalledOnce(); const envelope = JSON.parse(h.startTurn.mock.calls[0]![0].message.text) as { task: { prompt: string }; interactionResponses: unknown[]; }; expect(envelope.task.prompt).toBe("FRESH_HANDOFF: prior conversation summary\n\nFULL_TASK_CONTEXT\nAPPROVAL_FOLLOW_UP"); expect(envelope.interactionResponses).toEqual([approval]); expect(h.startTurn.mock.calls[0]![0]).not.toHaveProperty("continuation"); // The user sees that the earlier conversation context was reset. expect(h.onContinuityBreak).toHaveBeenCalledWith({ reason: expect.stringContaining("saved provider state is unavailable, so the follow-up continues in a fresh provider session without the earlier conversation context"), previousDriverSessionId: "driver-lost", previousProviderSessionId: "provider-lost", replacementDriverSessionId: "driver-fresh", replacementProviderSessionId: "provider-fresh", }); }); it.each([ ["the control plane cannot prove the run started no provider work", { proof: false }], ["the control plane predates the provider-work proof", { proof: null }], ["the provider-work proof fails", { proof: "rejects" }], ["the saved state is incompatible for another reason", { proof: true, recovery: { recovered: false as const, failureClass: PROVIDER_STATE_INCOMPATIBLE, reason: "saved launch profile changed" } }], ["the checkpoint requires the same session", { proof: true, checkpoint: { ...reboundCheckpoint(), providerRecoveryPolicy: "same_session_only" as const } }], ["this run already completed a turn", { proof: true, checkpoint: { ...reboundCheckpoint(), cursor: "2", terminalTurns: [{ turnId: "turn-own", fingerprint: "completed" }] } }], ["this run already advanced the durable cursor", { proof: true, checkpoint: { ...reboundCheckpoint(), cursor: "3" } }], ["this run already recorded a provider event", { proof: true, replay: [runnerEvent(1, "turn.started")] }], ["this run already proposed a result", { proof: true, checkpoint: { ...reboundCheckpoint(), semanticResult: result } }], ["the checkpoint was saved without the at-most-once recovery marker", { proof: true, checkpoint: (() => { const { dispositionOnlyRecoveryConsumed: _omitted, ...older } = reboundCheckpoint(); return older; })() }], ] as const)("keeps the named refusal and a working Retry when %s", async (_label, options) => { const h = harness(options as Parameters[0]); const failure = await h.run().catch((error: unknown) => error); expect(failure).toBeInstanceOf(NativeProviderStateIncompatibleError); expect((failure as NativeProviderStateIncompatibleError).code).toBe("native_provider_state_incompatible"); expect(h.openReplacementSession).not.toHaveBeenCalled(); expect(h.openSession).not.toHaveBeenCalled(); expect(h.openRun).not.toHaveBeenCalled(); expect(h.startTurn).not.toHaveBeenCalled(); }); it("never replaces a session whose checkpoint records this run's own governed wait", async () => { const h = harness({ proof: true, checkpoint: { ...reboundCheckpoint(), governedWait: { sourceEvent: runnerEvent(1, "turn.started"), result: yieldedResult } } }); await expect(h.run()).rejects.toThrow(); expect(h.openReplacementSession).not.toHaveBeenCalled(); expect(h.startTurn).not.toHaveBeenCalled(); }); it("keeps the named refusal when the provider-work proof does not answer in time", async () => { vi.useFakeTimers(); try { const h = harness({ proof: "hangs" }); const failure = h.run().catch((error: unknown) => error); await vi.advanceTimersByTimeAsync(30_001); expect(await failure).toBeInstanceOf(NativeProviderStateIncompatibleError); expect(h.provesNoRunProviderWork).toHaveBeenCalledOnce(); expect(h.openReplacementSession).not.toHaveBeenCalled(); } finally { vi.useRealTimers(); } }); it("stops by name when the follow-up's saved session still owns a runtime request", async () => { const h = harness({ proof: true, recovery: { recovered: false as const, failureClass: PROVIDER_SIDE_EFFECT_PENDING, reason: "native_runner_source_runtime_reconciliation_required" } }); await expect(h.run()).rejects.toBeInstanceOf(NativeProviderSideEffectPendingError); expect(h.provesNoRunProviderWork).not.toHaveBeenCalled(); expect(h.openReplacementSession).not.toHaveBeenCalled(); }); }); // A remote workspace epoch was abandoned, so the run was assigned a new // sibling root while its checkpoint still names the old one. A checkpoint // that requires the same session (an unfinished goal) ends with the // permanent named code, never a generic retryable failure; a checkpoint that // allows replacement continues in a fresh session in the assigned root. it.each([ ["same_session_only", false, false], ["allow_replacement_after_governed_wait", false, false], ["allow_replacement_after_resume_failure", true, false], // An in-flight turn may have an unsettled external effect: never replaced, under any policy. ["same_session_only", false, true], ["allow_replacement_after_resume_failure", false, true], ] as const)("handles a %s checkpoint whose remote workspace root moved (replaces=%s, in flight=%s)", async (providerRecoveryPolicy, replaces, inFlight) => { const parent = "/home/daytona/paperclip-workspace/.paperclip-workspaces"; const savedRoot = `${parent}/a5a34403-04fe-4e7e-97cf-d8f371a802a4-1-80214561-2151-4112-a43a-7d65f3071b56`; const assignedRoot = `${parent}/a5a34403-04fe-4e7e-97cf-d8f371a802a4-5-ef79592c-6784-438f-8a17-9800dd2fa2fc`; const checkpoint: PersistedNativeSession = { backendKind: "runner", sessionId: "thread-1", identity, providerSessionId: "provider-session-1", workingDirectory: savedRoot, providerRecoveryPolicy, cursor: "0", activeTurnId: inFlight ? "turn-in-flight" : null, terminalTurns: [], pendingRuntimeRequests: [], lineage: [], }; const transportFactory = vi.fn(() => new FakeCodexTransport()); const backend = createCodexNativeSessionBackend({ ...input, workspace: { ...input.workspace, cwd: assignedRoot } }, { runnerInstanceId: "runner-recovery", transportFactory, workingDirectoryAuthority: "remote_runner", environment: { PAPERCLIP_WORKSPACE_CWD: assignedRoot }, }); const replacementAttempted = new Error("replacement attempted"); const openReplacementSession = vi.fn(async (sessionInput: { workingDirectory?: string }) => { expect(sessionInput.workingDirectory).toBe(assignedRoot); throw replacementAttempted; }); const openRun = vi.fn(async () => undefined); const failure = await executeNativeSession({ input: { ...input, workspace: { ...input.workspace, cwd: assignedRoot } }, backend: { descriptor: () => backend.descriptor(), openSession: sessionInput => backend.openSession(sessionInput), openReplacementSession, recoverSession: (snapshot, options) => backend.recoverSession!(snapshot, options) }, controlPlane: { openRun, appendEvent: vi.fn(), completeRun: vi.fn(), loadSessionCheckpoint: async () => structuredClone(checkpoint), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }).catch((error: unknown) => error); if (replaces) { expect(failure).toBe(replacementAttempted); expect(openReplacementSession).toHaveBeenCalledOnce(); } else { if (inFlight) { expect(failure).toBeInstanceOf(NativeProviderSideEffectPendingError); expect((failure as NativeProviderSideEffectPendingError).code).toBe("native_provider_side_effect_pending"); } else { expect(failure).toBeInstanceOf(NativeProviderStateIncompatibleError); expect((failure as NativeProviderStateIncompatibleError).code).toBe("native_provider_state_incompatible"); } expect(openReplacementSession).not.toHaveBeenCalled(); expect(openRun).not.toHaveBeenCalled(); } // No Runner or provider process starts for a session that cannot resume here. expect(transportFactory).not.toHaveBeenCalled(); }); it("never replaces a session after a deterministic, non-destructive recovery refusal", async () => { const outcomes = [ { failureClass: PROVIDER_ATTACH_REFUSED, reason: "run.attach cannot change protected launch arguments", expected: NativeProviderStateIncompatibleError }, { failureClass: PROVIDER_SIDE_EFFECT_PENDING, reason: "native_runner_source_semantic_reconciliation_required", expected: NativeProviderSideEffectPendingError }, ] as const; for (const outcome of outcomes) { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-refused", identity, providerSessionId: "provider-refused", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const openRun = vi.fn(async () => undefined); const openSession = vi.fn(); const openReplacementSession = vi.fn(); const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: "refused", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), openSession, openReplacementSession, recoverSession: async () => ({ recovered: false, reason: outcome.reason, failureClass: outcome.failureClass }), }; const port: ControlPlanePort = { openRun, appendEvent: vi.fn(), completeRun: vi.fn(), loadSessionCheckpoint: async () => structuredClone(checkpoint), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), }; const failure = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }).catch((error: unknown) => error); expect(failure, outcome.failureClass).toBeInstanceOf(outcome.expected); expect((failure as Error).message).toBe(`native_session_recovery_failed: ${outcome.reason}`); expect(openReplacementSession, outcome.failureClass).not.toHaveBeenCalled(); expect(openSession, outcome.failureClass).not.toHaveBeenCalled(); expect(openRun, outcome.failureClass).not.toHaveBeenCalled(); } }); it("still replaces a session after an unclassified resume failure when the checkpoint allows it", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-transient", identity, providerSessionId: "provider-transient", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const replacementAttempted = new Error("replacement attempted"); const openReplacementSession = vi.fn(async () => { throw replacementAttempted; }); const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: "transient", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), openSession: vi.fn(), openReplacementSession, recoverSession: async () => ({ recovered: false, reason: "runnerd transport closed while waiting for run.attach" }), }; const port: ControlPlanePort = { openRun: vi.fn(async () => undefined), appendEvent: vi.fn(), completeRun: vi.fn(), loadSessionCheckpoint: async () => structuredClone(checkpoint), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), }; await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", })).rejects.toBe(replacementAttempted); expect(openReplacementSession).toHaveBeenCalledOnce(); }); it("continues in a fresh session when a new run's saved session is incompatible and the checkpoint allows replacement", async () => { // A provider release changed between turns (for example an agent-server // bump). The next run's checkpoint allows replacement, so the user's task // continues automatically in a fresh provider session, with no Retry. const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-upgraded", identity, providerSessionId: "provider-upgraded", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const replacementAttempted = new Error("replacement attempted"); const openReplacementSession = vi.fn(async () => { throw replacementAttempted; }); const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: "upgraded", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), openSession: vi.fn(), openReplacementSession, recoverSession: async () => ({ recovered: false, failureClass: PROVIDER_STATE_INCOMPATIBLE, reason: "ACPX durable launch profile digest does not match runner startup" }), }; const port: ControlPlanePort = { openRun: vi.fn(async () => undefined), appendEvent: vi.fn(), completeRun: vi.fn(), loadSessionCheckpoint: async () => structuredClone(checkpoint), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), }; await expect(executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", })).rejects.toBe(replacementAttempted); expect(openReplacementSession).toHaveBeenCalledOnce(); }); it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { vi.useFakeTimers(); let rejectStalledReplay = (_error: Error) => {}; try { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-replay-stalled", identity, providerSessionId: "provider-replay-stalled", providerRecoveryPolicy: "same_session_only", cursor: "0", activeTurnId: "turn-replay-stalled", pendingRuntimeRequests: [], lineage: [], }; const stalledReplay = new Promise((_resolve, reject) => { rejectStalledReplay = reject; }); let markSecondPageStarted = () => {}; const secondPageStarted = new Promise((resolve) => { markSecondPageStarted = resolve; }); const replaySignals: AbortSignal[] = []; const replayEvents = vi.fn( async (replay, operationOptions) => { expect(operationOptions?.signal).toBeInstanceOf(AbortSignal); replaySignals.push(operationOptions!.signal); if (replay.afterSourceSeq === 0) { return { events: [runnerEvent(1, "item.completed", { kind: "progress" })], highestContiguousSourceSeq: 1, }; } markSecondPageStarted(); return await stalledReplay; }, ); const recoverSession = vi.fn(); const openRun = vi.fn(async () => undefined); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected replacement"); }, recoverSession, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, replayEvents, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession: structuredClone(checkpoint), timeoutMs: 5, }); const rejection = expect(execution).rejects.toThrow( "native session recovery replay timed out after 5ms", ); await secondPageStarted; expect(replayEvents).toHaveBeenCalledTimes(2); expect(replaySignals).toHaveLength(2); expect(replaySignals[1]).toBe(replaySignals[0]); expect(replaySignals[0]?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(5); await rejection; expect(replaySignals[0]?.aborted).toBe(true); expect(recoverSession).not.toHaveBeenCalled(); expect(openRun).not.toHaveBeenCalled(); // A broken adapter may ignore abort and reject later. The bounded helper // keeps that losing operation observed after execution already rejected. rejectStalledReplay(new Error("late recovery replay failure")); await Promise.resolve(); } finally { vi.useRealTimers(); } }); it("bounds provider recovery and closes a session returned after timeout", async () => { vi.useFakeTimers(); try { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery-stalled", identity, providerSessionId: "provider-recovery-stalled", providerRecoveryPolicy: "same_session_only", cursor: "0", activeTurnId: "turn-recovery-stalled", pendingRuntimeRequests: [], lineage: [], }; let resolveRecovery = (_value: { recovered: true; session: NativeSession; }) => {}; const stalledRecovery = new Promise<{ recovered: true; session: NativeSession; }>((resolve) => { resolveRecovery = resolve; }); let markRecoveryStarted = () => {}; const recoveryStarted = new Promise((resolve) => { markRecoveryStarted = resolve; }); let markCloseStarted = () => {}; const closeStarted = new Promise((resolve) => { markCloseStarted = resolve; }); const close = vi.fn(async () => { markCloseStarted(); }); const lateSession: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() {}, async startTurn() { throw new Error("late recovery session must not start"); }, async result() { return null; }, async snapshot() { return structuredClone(checkpoint); }, close, }; let recoverySignal: AbortSignal | undefined; const recoverSession = vi.fn( ( _checkpoint: PersistedNativeSession, recoveryOptions: { signal: AbortSignal }, ) => { recoverySignal = recoveryOptions.signal; recoveryOptions.signal.addEventListener( "abort", () => resolveRecovery({ recovered: true, session: lateSession }), { once: true }, ); markRecoveryStarted(); return stalledRecovery; }, ); const openRun = vi.fn(async () => undefined); const onSession = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected replacement"); }, recoverSession, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession: structuredClone(checkpoint), timeoutMs: 5, onSession, }); const rejection = expect(execution).rejects.toThrow( "native session provider recovery timed out after 5ms", ); await recoveryStarted; expect(recoverySignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(5); await rejection; expect(recoverySignal?.aborted).toBe(true); expect(openRun).not.toHaveBeenCalled(); expect(onSession).not.toHaveBeenCalled(); await closeStarted; expect(close).toHaveBeenCalledWith({ reason: "native session provider recovery timed out", }); expect(onSession).not.toHaveBeenCalled(); } finally { vi.useRealTimers(); } }); it("bounds replacement bootstrap and closes a session returned after timeout", async () => { vi.useFakeTimers(); try { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-replacement-stalled", identity, providerSessionId: "provider-replacement-stalled", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; let resolveReplacement = (_value: NativeSession) => {}; const stalledReplacement = new Promise((resolve) => { resolveReplacement = resolve; }); let markReplacementStarted = () => {}; const replacementStarted = new Promise((resolve) => { markReplacementStarted = resolve; }); let markCloseStarted = () => {}; const closeStarted = new Promise((resolve) => { markCloseStarted = resolve; }); const close = vi.fn(async () => { markCloseStarted(); }); const lateSession: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() {}, async startTurn() { throw new Error("late replacement session must not start"); }, async result() { return null; }, async snapshot() { return structuredClone(checkpoint); }, close, }; let replacementSignal: AbortSignal | undefined; const openReplacementSession = vi.fn( (replacementInput: { identity: NativeRunIdentity; workingDirectory?: string; signal?: AbortSignal; }) => { replacementSignal = replacementInput.signal; replacementInput.signal?.addEventListener( "abort", () => resolveReplacement(lateSession), { once: true }, ); markReplacementStarted(); return stalledReplacement; }, ); const openRun = vi.fn(async () => undefined); const onSession = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "replacement-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("replacement seam must be used"); }, async recoverSession() { return { recovered: false, reason: "provider session is missing" }; }, openReplacementSession, }; const port: ControlPlanePort = { openRun, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", persistedSession: structuredClone(checkpoint), timeoutMs: 5, onSession, }); const rejection = expect(execution).rejects.toThrow( "native session replacement bootstrap timed out after 5ms", ); await replacementStarted; expect(replacementSignal?.aborted).toBe(false); await vi.advanceTimersByTimeAsync(5); await rejection; expect(replacementSignal?.aborted).toBe(true); expect(openRun).not.toHaveBeenCalled(); expect(onSession).not.toHaveBeenCalled(); await closeStarted; expect(close).toHaveBeenCalledWith({ reason: "native session replacement bootstrap timed out", }); expect(onSession).not.toHaveBeenCalled(); } finally { vi.useRealTimers(); } }); it("does not persist a reconciled recovery cursor when provider recovery rejects", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery-rejects", identity, providerSessionId: "provider-recovery-rejects", providerRecoveryPolicy: "same_session_only", cursor: "0", activeTurnId: "turn-recovery-rejects", pendingRuntimeRequests: [], lineage: [], }; const recoveryFailure = new Error("provider recovery rejected"); const openRun = vi.fn(async () => undefined); const checkpointSession = vi.fn(async () => undefined); const onCheckpoint = vi.fn(async () => undefined); const recoverSession = vi.fn( async (recoveryCheckpoint: PersistedNativeSession) => { expect(recoveryCheckpoint.cursor).toBe("1"); throw recoveryFailure; }, ); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected replacement"); }, recoverSession, }; const port: ControlPlanePort = { openRun, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, checkpointSession, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents(replay) { return { events: replay.afterSourceSeq === 0 ? [runnerEvent(1, "turn.started")] : [], highestContiguousSourceSeq: 1, }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", onCheckpoint, }), ).rejects.toBe(recoveryFailure); expect(recoverSession).toHaveBeenCalledOnce(); expect(openRun).not.toHaveBeenCalled(); expect(checkpointSession).not.toHaveBeenCalled(); expect(onCheckpoint).not.toHaveBeenCalled(); }); it("does not persist a reconciled recovery cursor when run admission rejects", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-admission-rejects", identity, providerSessionId: "provider-admission-rejects", providerRecoveryPolicy: "same_session_only", cursor: "0", activeTurnId: "turn-admission-rejects", pendingRuntimeRequests: [], lineage: [], }; const admissionFailure = new Error("control-plane admission rejected"); const openRun = vi.fn(async () => { throw admissionFailure; }); const checkpointSession = vi.fn(async () => undefined); const onCheckpoint = vi.fn(async () => undefined); const onSession = vi.fn(); const close = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() {}, async startTurn() { throw new Error("unexpected turn"); }, async result() { return null; }, async snapshot() { throw new Error("unexpected snapshot"); }, close, }; const recoverSession = vi.fn( async (recoveryCheckpoint: PersistedNativeSession) => { expect(recoveryCheckpoint.cursor).toBe("1"); return { recovered: true as const, session }; }, ); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("unexpected replacement"); }, recoverSession, }; const port: ControlPlanePort = { openRun, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, checkpointSession, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents(replay) { return { events: replay.afterSourceSeq === 0 ? [runnerEvent(1, "turn.started")] : [], highestContiguousSourceSeq: 1, }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", onCheckpoint, onSession, }), ).rejects.toBe(admissionFailure); expect(recoverSession).toHaveBeenCalledOnce(); expect(openRun).toHaveBeenCalledOnce(); expect(checkpointSession).not.toHaveBeenCalled(); expect(onCheckpoint).not.toHaveBeenCalled(); expect(onSession).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenCalledWith(null); expect(close).toHaveBeenCalledWith({ reason: "native control-plane run admission failed", }); }); it("continues a provider-reported active turn without starting a duplicate turn", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const providerSnapshot: PersistedNativeSession = { ...checkpoint, cursor: "1", activeTurnId: "turn-recovery", }; const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: "provider-recovery:1", sourceSeq: 1, sourceInstanceId: "provider-recovery", sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-recovery", eventType: "turn.completed", schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:00.000Z", payload: {}, }; const bySource = new Map(); const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" })); const openSession = vi.fn(async () => { throw new Error("must recover the provider session"); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield terminalEvent; }, startTurn, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return structuredClone(providerSnapshot); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent(event: PrpEvent) { const list = bySource.get(event.sourceInstanceId) ?? []; list.push(structuredClone(event)); bySource.set(event.sourceInstanceId, list); return { cursor: list.length, highestContiguousSourceSeq: highestContiguous(list), disposition: "committed", }; }, async replayEvents(replay) { const list = bySource.get(replay.sourceInstanceId) ?? []; return { events: structuredClone( list.filter((event) => event.sourceSeq > replay.afterSourceSeq), ), highestContiguousSourceSeq: highestContiguous(list), }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ turnId: "turn-recovery", providerSessionId: "provider-recovery", }); expect(openSession).not.toHaveBeenCalled(); expect(startTurn).not.toHaveBeenCalled(); }); it.each([ { checkpointCursor: "12", expectedCursor: "41", terminalSequence: 42 }, { checkpointCursor: "50", expectedCursor: "50", terminalSequence: 51 }, ])( "seeds recovery from the larger of checkpoint $checkpointCursor and the persisted source high-water mark", async ({ checkpointCursor, expectedCursor, terminalSequence }) => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: checkpointCursor, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const runnerEvents = [ runnerEvent(13, "item.completed", { kind: "progress" }), runnerEvent(41, "item.completed", { kind: "progress" }), ]; const terminalEvent = runnerEvent(terminalSequence, "turn.completed"); const controlEvents: PrpEvent[] = []; const checkpoints: PersistedNativeSession[] = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield terminalEvent; }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { ...checkpoint, cursor: String(terminalSequence), activeTurnId: null, }; }, async close() {}, }; const recoverSession = vi.fn( async (recoveryCheckpoint: PersistedNativeSession) => { expect(recoveryCheckpoint.cursor).toBe(expectedCursor); return { recovered: true, session }; }, ); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("must recover the provider session"); }, recoverSession, }; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, async appendEvent(event: PrpEvent) { const target = event.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; if ( target.some((existing) => existing.sourceSeq === event.sourceSeq) ) { throw new Error(`native_event_replay_conflict:${event.sourceSeq}`); } target.push(structuredClone(event)); return { cursor: target.length, highestContiguousSourceSeq: highestContiguous(target), disposition: "committed", }; }, async replayEvents(replay) { const source = replay.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; const events = source .filter((event) => event.sourceSeq > replay.afterSourceSeq) .sort((left, right) => left.sourceSeq - right.sourceSeq) .slice(0, replay.limit); return { events: structuredClone(events), highestContiguousSourceSeq: highestContiguous(source), }; }, async completeRun() {}, }; const getFreshSessionHandoff = vi.fn(async () => "FRESH_HANDOFF_ONLY"); await expect( executeNativeSession({ input, getFreshSessionHandoff, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ turnId: "turn-recovery" }); expect(getFreshSessionHandoff).not.toHaveBeenCalled(); expect(recoverSession).toHaveBeenCalledOnce(); expect( runnerEvents.some((event) => event.sourceSeq === terminalSequence), ).toBe(true); if (checkpointCursor === "12") { expect(checkpoints[0]).toMatchObject({ cursor: "41" }); } }, ); it("attempts exact recovery before opening an observable replacement session", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-old", identity, providerSessionId: "provider-old", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const replacementSnapshot: PersistedNativeSession = { ...checkpoint, sessionId: "driver-new", providerSessionId: "provider-new", providerRecoveryPolicy: "same_session_only", }; const replacementSession: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async startTurn() { return { turnId: "turn-replacement" }; }, async result() { return { result, terminal, turnId: "turn-replacement" }; }, async snapshot() { return structuredClone(replacementSnapshot); }, async close() {}, }; const recoverSession = vi.fn(async () => ({ recovered: false as const, reason: "provider reported the prior session missing", })); const openReplacementSession = vi.fn(async () => replacementSession); const onContinuityBreak = vi.fn(async () => undefined); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "replacement-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("replacement seam must be used"); }, recoverSession, openReplacementSession, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(structuredClone(event)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-replacement", controlPlaneInstanceId: "control-replacement", onContinuityBreak, }), ).resolves.toMatchObject({ providerSessionId: "provider-new" }); expect(recoverSession).toHaveBeenCalledOnce(); expect(openReplacementSession).toHaveBeenCalledOnce(); expect(onContinuityBreak).toHaveBeenCalledWith({ reason: "provider reported the prior session missing", previousDriverSessionId: "driver-old", previousProviderSessionId: "provider-old", replacementDriverSessionId: "driver-new", replacementProviderSessionId: "provider-new", }); }); it("does not replace a failed provider session when recovery policy forbids it", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-failed-constrained", identity, providerSessionId: "provider-failed-constrained", providerRecoveryPolicy: "same_session_only", cursor: null, activeTurnId: null, semanticResult: null, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "yielded", }, terminalTurns: [{ turnId: "turn-failed", fingerprint: "failed" }], pendingRuntimeRequests: [], lineage: [], }; const openRun = vi.fn(async () => undefined); const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); }); const recoverSession = vi.fn(); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "constrained-recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, recoverSession, }; const port: ControlPlanePort = { openRun, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async appendEvent() { throw new Error("unexpected event"); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; await expect( executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-constrained-recovery", controlPlaneInstanceId: "control-constrained-recovery", }), ).rejects.toMatchObject({ code: "native_provider_terminal_failed", providerCode: "provider_checkpoint_failed_terminal", recoverable: false, }); expect(recoverSession).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); expect(openRun).not.toHaveBeenCalled(); }); it.each([ { preparedMode: false, interrupted: false }, { preparedMode: true, interrupted: false }, { preparedMode: false, interrupted: true }, { preparedMode: true, interrupted: true }, ])("replaces a failed provider with full task context (prepared: $preparedMode, interrupted: $interrupted)", async ({ preparedMode, interrupted }) => { const digest = "0".repeat(64); const skill = { key: "company/recovery-skill", runtimeName: "recovery-skill", versionId: "recovery-skill-v1", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/skills/recovery-skill", fileCount: 1, totalBytes: 1, }, } as const; const context = { prompt: { revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, text: PAPERCLIP_EXECUTION_PROMPT, digest: nativeRuntimePromptDigest() }, instructions: { entryPath: "AGENTS.md", bundle: { schema: NATIVE_RUNTIME_ASSET_SCHEMA, digest, manifestDigest: digest, rootPath: "/runtime/instructions", fileCount: 1, totalBytes: 1 } }, skills: [skill], mcp: { assignmentSetId: "none", digest, bindingId: null }, } satisfies RuntimeContextFields; const legacyContext = { ...context, skills: [] } satisfies RuntimeContextFields; const prepared = preparedInput(); const executionInput = preparedMode ? { ...prepared, task: { ...prepared.task, description: "Complete after recovery with $recovery-skill.", prompt: "FULL_ASSIGNMENT_CONTEXT\nCURRENT_EVENT_CONTEXT", }, runtimeContext: { ...context, aggregateDigest: canonicalNativeRuntimeContextDigest(context), }, continuationPrompt: "ONLY_NEW_COMMENT", } as const : { ...input, task: { ...input.task, prompt: "FULL_ASSIGNMENT_CONTEXT\nCURRENT_EVENT_CONTEXT" }, schema: "paperclip.native-execution-input.v4" as const, executionMode: "default" as const, planningContext: null, provider: { kind: "codex" as const, model: null, approvalPolicy: "never" as const }, runtimeContext: { ...legacyContext, aggregateDigest: canonicalNativeRuntimeContextDigest(legacyContext), }, continuationPrompt: "ONLY_NEW_COMMENT", } as const; const checkpoint: PersistedNativeSession = { backendKind: "mock", driverKind: "codex_app_server", sessionId: "driver-failed", identity, providerSessionId: "provider-failed", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: null, activeTurnId: null, semanticResult: null, terminal: { schema: "paperclip.prp.terminal.v1", turnTerminalState: "failed", runTerminalState: "failed", reportedWorkDisposition: "yielded", }, terminalTurns: [{ turnId: "turn-failed", fingerprint: interrupted ? JSON.stringify({ terminalState: "failed", result: null, error: { code: "provider_turn_lost_on_restore", recoverable: true }, }) : "failed" }], pendingRuntimeRequests: [], lineage: [], }; const replacementSnapshot: PersistedNativeSession = { ...checkpoint, sessionId: "driver-replacement", providerSessionId: "provider-replacement", terminal: null, terminalTurns: [], }; const startTurn = vi.fn(async () => ({ turnId: "turn-replacement" })); const replacementSession: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, startTurn, async result() { return { result, terminal, turnId: "turn-replacement" }; }, async snapshot() { return structuredClone(replacementSnapshot); }, async close() {}, }; const recoverSession = vi.fn(async () => ({ recovered: false as const, reason: "provider thread is unavailable", })); const openReplacementSession = vi.fn(async () => replacementSession); const onContinuityBreak = vi.fn(async () => undefined); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "replacement-backend", version: "1", runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" }, capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("replacement seam must be used"); }, recoverSession, openReplacementSession, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(structuredClone(event)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const getFreshSessionHandoff = vi.fn(async () => "FRESH_HANDOFF: original goal, prior answers, and next step"); await expect( executeNativeSession({ input: executionInput, getFreshSessionHandoff, backend, controlPlane: port, runnerInstanceId: "runner-replacement", controlPlaneInstanceId: "control-replacement", onContinuityBreak, }), ).resolves.toMatchObject({ providerSessionId: "provider-replacement" }); expect(getFreshSessionHandoff).toHaveBeenCalledOnce(); expect(recoverSession).toHaveBeenCalledTimes(interrupted ? 1 : 0); expect(openReplacementSession).toHaveBeenCalledOnce(); const replacementEnvelope = JSON.parse( startTurn.mock.calls[0]![0].message.text, ) as { schema: string; requestedSkills?: string[]; task: { prompt: string }; completionContract: unknown; }; expect(replacementEnvelope).toMatchObject({ task: { prompt: "FRESH_HANDOFF: original goal, prior answers, and next step\n\nFULL_ASSIGNMENT_CONTEXT\nCURRENT_EVENT_CONTEXT" }, completionContract: executionInput.completionContract.contract, }); expect(replacementEnvelope.schema).toBe( preparedMode ? "paperclip.native-model-envelope.v3" : "paperclip.native-model-envelope.v2", ); if (preparedMode) expect(replacementEnvelope.requestedSkills).toEqual(["recovery-skill"]); else expect(replacementEnvelope).not.toHaveProperty("requestedSkills"); expect(JSON.stringify(replacementEnvelope)).not.toContain("ONLY_NEW_COMMENT"); expect(startTurn.mock.calls[0]![0]).not.toHaveProperty("continuation"); expect(onContinuityBreak).toHaveBeenCalledWith({ reason: interrupted ? "provider thread is unavailable" : "provider session ended with a failed terminal", previousDriverSessionId: "driver-failed", previousProviderSessionId: "provider-failed", replacementDriverSessionId: "driver-replacement", replacementProviderSessionId: "provider-replacement", }); }); it.each([false, true])("only replays the original ACPX envelope for a proven effect-free initial turn (prepared: %s)", async (prepared) => { const executionInput = prepared ? preparedInput() : input; const checkpoint: PersistedNativeSession = { backendKind: "mock", driverKind: "acpx_runtime", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, terminalTurns: [ { turnId: "turn-work", fingerprint: "terminal-fingerprint" }, ], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-missing-disposition", pendingRuntimeRequests: [], lineage: [], }; const recoveredSnapshot: PersistedNativeSession = { ...checkpoint, dispositionOnlyRecoveryConsumed: false, dispositionOnlyRecoveryTurnId: null, }; const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: "provider-recovery:2", sourceSeq: 2, sourceInstanceId: "provider-recovery", sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-continuation", eventType: "turn.completed", schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:01.000Z", payload: {}, }; const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield terminalEvent; }, startTurn, async result() { return { result, terminal, turnId: "turn-continuation" }; }, async snapshot() { return structuredClone(recoveredSnapshot); }, async close() {}, }; const backend: NativeSessionBackend = { preparedTaskConstraints: ["Retain the original workspace rule."], async descriptor() { return { kind: "mock", runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" }, name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("must recover the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const bySource = new Map(); const replayedPages: PrpEvent[][] = []; const replayEvents = vi.fn( async (replay: Parameters[0]) => { const list = bySource.get(replay.sourceInstanceId) ?? []; const events = structuredClone( list.filter((event) => event.sourceSeq > replay.afterSourceSeq), ); replayedPages.push(events); return { events, highestContiguousSourceSeq: highestContiguous(list), }; }, ); const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent(event: PrpEvent) { const list = bySource.get(event.sourceInstanceId) ?? []; list.push(structuredClone(event)); bySource.set(event.sourceInstanceId, list); return { cursor: list.length, highestContiguousSourceSeq: highestContiguous(list), disposition: "committed", }; }, replayEvents, async completeRun() {}, }; const submittedTurn = runnerEvent(1, "turn.submitted"); delete submittedTurn.turnId; const effectFreeTurn = [ submittedTurn, { ...runnerEvent(2, "turn.started", { status: "inProgress" }), turnId: "turn-work", }, { ...runnerEvent(3, "turn.accepted"), turnId: "turn-work" }, { ...runnerEvent(4, "item.completed", { kind: "usage", usage: { total: { requests: 1, inputTokens: 0, outputTokens: 0, activeSeconds: 0, providerCostUsd: 0, cacheReadTokens: 0, cacheWriteTokens: 0, }, runDelta: { requests: 1, inputTokens: 0, outputTokens: 0, activeSeconds: 0, providerCostUsd: 0, cacheReadTokens: 0, cacheWriteTokens: 0, }, }, }), turnId: "turn-work", }, { ...runnerEvent(5, "turn.completed", { status: "completed", error: null, }), turnId: "turn-work", }, ]; bySource.set("runner-recovery", effectFreeTurn); await expect( executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery", }); expect(startTurn).toHaveBeenCalledOnce(); expect(replayEvents).toHaveBeenCalledWith({ runId: identity.runId, sourceInstanceId: "runner-recovery", afterSourceSeq: 0, limit: 1_000, }); expect( replayedPages.some( (events) => events.length === effectFreeTurn.length && events.every( (event, index) => event.sourceSeq === effectFreeTurn[index]!.sourceSeq, ), ), ).toBe(true); const recoveryEnvelope = JSON.parse( startTurn.mock.calls[0]![0].message.text, ) as { task: { prompt: string } }; expect(recoveryEnvelope.task.prompt).toBe(executionInput.task.prompt); if (prepared) { expect(recoveryEnvelope).toMatchObject({ schema: "paperclip.native-model-envelope.v3", constraints: ["Retain the original workspace rule."], completionContract: { criteria: [{ id: "objective", source: { id: identity.issueId, location: "task.prompt" } }] }, }); expect(recoveryEnvelope.task).not.toHaveProperty("description"); } startTurn.mockClear(); bySource.set("runner-recovery", [ ...effectFreeTurn.slice(0, 3), { ...runnerEvent(4, "item.completed", { kind: "agentMessage", text: "Work may already have been performed.", }), turnId: "turn-work", }, { ...runnerEvent(5, "turn.completed", { status: "completed", error: null, }), turnId: "turn-work", }, ]); await expect( executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery", }); const dispositionEnvelope = JSON.parse( startTurn.mock.calls[0]![0].message.text, ) as { task: { prompt: string } }; expect(dispositionEnvelope.task.prompt).toContain( "semantic-result recovery for a prior completed provider turn", ); expect(dispositionEnvelope.task.prompt).toContain( "Do not repeat implementation, tests, research, or the final answer", ); expect(dispositionEnvelope.task.prompt).not.toContain(input.task.prompt); expect(dispositionEnvelope).toHaveProperty("completionContract", input.completionContract.contract); expect(dispositionEnvelope).not.toHaveProperty("constraints"); if (prepared) expect(dispositionEnvelope).toHaveProperty("requestedSkills", []); checkpoint.dispositionOnlyRecoveryTurnId = undefined; recoveredSnapshot.dispositionOnlyRecoveryTurnId = undefined; startTurn.mockClear(); bySource.clear(); bySource.set("runner-recovery", [ { ...terminalEvent, sourceEventId: "runner-recovery:stale-terminal", turnId: "turn-stale-unbound", }, ]); await expect( executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }), ).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery", }); expect(startTurn).toHaveBeenCalledOnce(); }); it.each( [true, false].flatMap((dispositionRecovery) => (["turn.completed", "turn.interrupted"] as const).flatMap( (terminalType) => [false, true].map((failInitialAppend) => ({ dispositionRecovery, terminalType, failInitialAppend, })), ), ), )( "consumes adopted $terminalType without resending (disposition: $dispositionRecovery, failed first append: $failInitialAppend)", async ({ dispositionRecovery, terminalType, failInitialAppend }) => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: dispositionRecovery ? null : "turn-disposition", terminalTurns: dispositionRecovery ? [{ turnId: "turn-work", fingerprint: "work-terminal" }] : [], dispositionOnlyRecoveryConsumed: false, pendingRuntimeRequests: [], lineage: [], }; const recoveredSnapshot: PersistedNativeSession = { ...checkpoint, cursor: "2", activeTurnId: null, terminalTurns: [ ...checkpoint.terminalTurns!, { turnId: "turn-disposition", fingerprint: "disposition-terminal" }, ], dispositionOnlyRecoveryConsumed: dispositionRecovery, }; const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: "provider-recovery:2", sourceSeq: 2, sourceInstanceId: "provider-recovery", sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-disposition", eventType: terminalType, schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:01.000Z", payload: {}, }; const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); const close = vi.fn(async () => undefined); const completeRun = vi.fn(async () => undefined); const appendFailure = new Error("adopted terminal append failed"); let failNextAppend = failInitialAppend; let durableCheckpoint = structuredClone(checkpoint); const recoveryCheckpoints: PersistedNativeSession[] = []; let dispositionTerminalCommitted = false; let prematureDispositionCheckpoint = false; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield terminalEvent; }, startTurn, async result() { return null; }, async snapshot() { return structuredClone(recoveredSnapshot); }, close, }; const events: PrpEvent[] = []; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("must recover the provider session"); }, async recoverSession(snapshot) { recoveryCheckpoints.push(structuredClone(snapshot)); return { recovered: true, session: { ...session } }; }, }; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(durableCheckpoint); }, async checkpointSession(snapshot) { if ( snapshot.terminalTurns?.some( (turn) => turn.turnId === "turn-disposition", ) && !dispositionTerminalCommitted ) prematureDispositionCheckpoint = true; durableCheckpoint = structuredClone(snapshot); }, async appendEvent(event: PrpEvent) { if (event.eventType === terminalType && failNextAppend) { failNextAppend = false; throw appendFailure; } events.push(structuredClone(event)); if ( event.eventType === terminalType && event.turnId === "turn-disposition" ) { dispositionTerminalCommitted = true; } return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents(replay) { return { events: structuredClone( events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq, ), ), highestContiguousSourceSeq: highestContiguous(events), }; }, completeRun, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveMissingResult: async () => result, }); if (failInitialAppend) { await expect(execute()).rejects.toBe(appendFailure); expect(events).toEqual([]); expect(startTurn).not.toHaveBeenCalled(); expect(completeRun).not.toHaveBeenCalled(); expect(close).toHaveBeenCalledOnce(); expect(prematureDispositionCheckpoint).toBe(false); expect(durableCheckpoint.activeTurnId).toBe(checkpoint.activeTurnId); expect(durableCheckpoint.terminalTurns).toEqual( checkpoint.terminalTurns, ); expect(durableCheckpoint.identity).toEqual(checkpoint.identity); } await expect(execute()).resolves.toMatchObject({ result, turnId: "turn-disposition", terminal: { turnTerminalState: terminalType === "turn.completed" ? "completed" : "interrupted", runTerminalState: terminalType === "turn.completed" ? "succeeded" : "cancelled", }, }); expect(recoveryCheckpoints).toHaveLength(failInitialAppend ? 2 : 1); for (const recoveredCheckpoint of recoveryCheckpoints) { expect(recoveredCheckpoint.activeTurnId).toBe(checkpoint.activeTurnId); expect(recoveredCheckpoint.terminalTurns).toEqual( checkpoint.terminalTurns, ); expect(recoveredCheckpoint.identity).toEqual(checkpoint.identity); } expect(startTurn).not.toHaveBeenCalled(); expect(completeRun).toHaveBeenCalledOnce(); expect(prematureDispositionCheckpoint).toBe(false); expect(events.map((event) => event.eventType)).toEqual([ terminalType, "run.result.accepted", "run.terminal", ]); }, ); it.each(["silent", "commentary", "tool-events"] as const)("ACCT-04 resolves a proposal-less durable disposition after %s without replenishing consumed repair", async noise => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, terminalTurns: [{ turnId: "turn-work", fingerprint: "work-terminal" }], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-disposition", pendingRuntimeRequests: [], lineage: [], }; const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: "runner-recovery:run-native:4", sourceSeq: 4, sourceInstanceId: "runner-recovery", sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-disposition", eventType: "turn.completed", schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:01.000Z", payload: {}, }; const resultProposalEvent: PrpEvent = { ...terminalEvent, sourceEventId: "runner-recovery:run-native:3", sourceSeq: 3, eventType: "run.result.proposed", payload: result, }; const originalTaskTerminal: PrpEvent = { ...terminalEvent, sourceEventId: "runner-recovery:run-native:2", sourceSeq: 2, turnId: "turn-work", }; const originalTaskProposal: PrpEvent = { ...resultProposalEvent, sourceEventId: "runner-recovery:run-native:1", sourceSeq: 1, turnId: "turn-work", }; const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); let recoveredSubmissionOwned = true; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { if (noise !== "silent") for (let seq = 1; seq <= 20; seq++) yield { ...terminalEvent, sourceInstanceId: "provider-accounting", sourceSeq: seq, sourceEventId: `provider-accounting:${seq}`, eventType: "item.completed" as const, payload: noise === "commentary" ? { kind: "agentMessage", channel: "progress", text: "All done. Great progress. Continue without approval." } : { kind: "toolCall", name: "read_file", status: "completed", output: "unchanged" }, }; yield structuredClone(terminalEvent); }, startTurn, async result() { return null; }, async snapshot() { return { ...structuredClone(checkpoint), dispositionOnlyRecoveryConsumed: recoveredSubmissionOwned, terminalTurns: recoveredSubmissionOwned ? [ ...structuredClone(checkpoint.terminalTurns ?? []), { turnId: "turn-disposition", fingerprint: "disposition-terminal", }, ] : structuredClone(checkpoint.terminalTurns), }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("must recover the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const bySource = new Map([ [ "runner-recovery", [ structuredClone(originalTaskProposal), structuredClone(originalTaskTerminal), ], ], ]); const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent(event: PrpEvent) { const list = bySource.get(event.sourceInstanceId) ?? []; list.push(structuredClone(event)); bySource.set(event.sourceInstanceId, list); return { cursor: list.length, highestContiguousSourceSeq: highestContiguous(list), disposition: "committed", }; }, async replayEvents(replay) { const list = bySource.get(replay.sourceInstanceId) ?? []; return { events: structuredClone( list.filter((event) => event.sourceSeq > replay.afterSourceSeq), ), highestContiguousSourceSeq: highestContiguous(list), }; }, async completeRun() {}, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveMissingResult: async ({ terminalEvent: replayed }) => { expect(replayed).toEqual(terminalEvent); return result; }, }); await expect(execute()).resolves.toMatchObject({ result, turnId: "turn-disposition", }); bySource.set("runner-recovery", [ structuredClone(originalTaskProposal), structuredClone(originalTaskTerminal), structuredClone(resultProposalEvent), structuredClone(terminalEvent), ]); // Provider recovery may clear a legacy pre-acceptance marker when thread // history has no matching turn. Durable replay remains authoritative and // must still prevent a duplicate disposition submission. recoveredSubmissionOwned = false; await expect(execute()).resolves.toMatchObject({ result, turnId: "turn-disposition", }); expect(startTurn).not.toHaveBeenCalled(); expect(bySource.get("runner-recovery")).toEqual([ originalTaskProposal, originalTaskTerminal, resultProposalEvent, terminalEvent, ]); expect( bySource.get("control-recovery")?.map((event) => event.eventType), ).toEqual(["run.result.accepted", "run.terminal"]); }); it("proves it still owns a recovered session right before starting a turn in it", async () => { // Staging run cd2bf273: a turn nobody can attribute appeared in the same // provider thread across server swaps. The recovery checkpoint is fenced // (controller boot, generation, lease), but preparing the turn's input can // take a while and another generation may take the run over meanwhile. // The fenced checkpoint is repeated right before the submit, so the // replaced server submits nothing. const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], lineage: [], }; const startTurn = vi.fn(async () => ({ turnId: "late-turn" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, events: () => (async function* () {})(), startTurn, async result() { return null; }, async snapshot() { return structuredClone(checkpoint); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }; }, async openSession() { throw new Error("must recover the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; let takenOver = false; const checkpointSession = vi.fn(async () => { if (takenOver) throw new Error("native_runner_checkpoint_owner_lost"); }); const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, checkpointSession, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 0, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun: vi.fn(async () => undefined), }; await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", // Another server generation claims the run while this one prepares the turn. prepareWakeAttachmentPrompt: async () => { takenOver = true; return ""; } }).catch(() => undefined); expect(takenOver).toBe(true); expect(startTurn).not.toHaveBeenCalled(); }); it.each([ { label: "at startup", interruptedAtStartup: true }, { label: "when the lost turn's terminal arrives", interruptedAtStartup: false }, ])("proves it still owns a recovered session right before a restart continuation $label", async ({ interruptedAtStartup }) => { // Same takeover window as the startup turn, on the restart-continuation // path: steered messages and attachments are prepared before the submit. const failure = { code: "provider_turn_lost_on_restore", recoverable: true }; const checkpoint: PersistedNativeSession = { backendKind: "runner", driverKind: "acpx_runtime", sessionId: "record-recovery", identity, providerSessionId: "provider-recovery", cursor: "0", activeTurnId: "turn-recovery", terminalTurns: [], semanticResult: null, pendingRuntimeRequests: [], lineage: [], }; const interrupted: PersistedNativeSession = { ...checkpoint, activeTurnId: null, terminalTurns: [{ turnId: "turn-recovery", fingerprint: JSON.stringify({ terminalState: "failed", error: failure, result: null }) }], }; const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() { if (!interruptedAtStartup) yield runnerEvent(1, "turn.failed", { status: "failed", error: failure }); }, startTurn, async result() { return null; }, async snapshot() { return structuredClone(interrupted); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "runner", name: "restart", version: "1", capabilities: await session.capabilities() }; }, async openSession() { throw new Error("must preserve the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; let takenOver = false; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() { if (takenOver) throw new Error("native_runner_checkpoint_owner_lost"); }, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const prepareWakeAttachmentPrompt = vi.fn(async () => { takenOver = true; return ""; }); await executeNativeSession({ input, backend, controlPlane: port, persistedSession: interruptedAtStartup ? interrupted : checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resumeInterruptedTurn: true, // Another server generation claims the run while this one prepares the continuation. prepareWakeAttachmentPrompt, }).catch(() => undefined); expect(prepareWakeAttachmentPrompt).toHaveBeenCalledOnce(); expect(startTurn).not.toHaveBeenCalled(); }); it("resolves a checkpointed result-less disposition without resubmitting when its terminal event is missing", async () => { const workProposal: PrpEvent = { ...runnerEvent(1, "run.result.proposed", result), turnId: "turn-work", }; const workTerminal: PrpEvent = { ...runnerEvent(2, "turn.completed"), turnId: "turn-work", }; const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "3", activeTurnId: null, terminalTurns: [ { turnId: "turn-work", fingerprint: "work-terminal" }, { turnId: "turn-disposition", fingerprint: "disposition-terminal" }, ], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-disposition", pendingRuntimeRequests: [], lineage: [], }; const recoveredCheckpoint: PersistedNativeSession = structuredClone(checkpoint); const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); const events = vi.fn(() => (async function* () { throw new Error("checkpoint fallback must not consume provider events"); })(), ); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, events, startTurn, async result() { return null; }, async snapshot() { return structuredClone(recoveredCheckpoint); }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { throw new Error("must recover the provider session"); }, async recoverSession() { return { recovered: true, session }; }, }; const bySource = new Map([ ["runner-recovery", [workProposal, workTerminal]], ]); const completeRun = vi.fn(async () => undefined); const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession() {}, async appendEvent(event: PrpEvent) { const list = bySource.get(event.sourceInstanceId) ?? []; list.push(structuredClone(event)); bySource.set(event.sourceInstanceId, list); return { cursor: list.length, highestContiguousSourceSeq: highestContiguous(list), disposition: "committed", }; }, async replayEvents(replay) { const list = bySource.get(replay.sourceInstanceId) ?? []; return { events: structuredClone( list.filter((event) => event.sourceSeq > replay.afterSourceSeq), ), highestContiguousSourceSeq: highestContiguous(list), }; }, completeRun, }; const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveMissingResult: async ({ turnId, terminalEvent }) => { expect(turnId).toBe("turn-disposition"); expect(terminalEvent).toMatchObject({ sourceInstanceId: "control-recovery", sourceKind: "control_plane", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-disposition", eventType: "turn.completed", payload: { recovery: "checkpointed_resultless_disposition", terminalFingerprint: "disposition-terminal", }, }); return result; }, }); await expect(execute()).resolves.toMatchObject({ result, turnId: "turn-disposition", }); expect(startTurn).not.toHaveBeenCalled(); expect(events).not.toHaveBeenCalled(); expect(completeRun).toHaveBeenCalledOnce(); expect(bySource.get("runner-recovery")).toEqual([ workProposal, workTerminal, ]); expect( bySource.get("control-recovery")?.map((event) => event.eventType), ).toEqual(["run.result.accepted", "run.terminal"]); recoveredCheckpoint.terminalTurns![1]!.fingerprint = "conflicting-terminal"; await expect(execute()).rejects.toThrow( "native_disposition_recovery_checkpoint_conflict", ); expect(startTurn).not.toHaveBeenCalled(); expect(events).not.toHaveBeenCalled(); expect(completeRun).toHaveBeenCalledOnce(); }); it.each([false, true])("confirms a redacted completion after a lost acknowledgement (conflict: %s)", async conflict => { vi.useFakeTimers(); try { const completion = { ...result, summary: JSON.stringify({ operationId: "call_api", data: "private-fixture-value" }) }; const history: PrpEvent[] = []; const normalize = (payload: PrpEvent["payload"]): PrpEvent["payload"] => JSON.parse(JSON.stringify(payload).replaceAll("private-fixture-value", "***REDACTED***")); let committed!: () => void; const saved = new Promise(resolve => { committed = resolve; }); let lostAcknowledgement = false; const completeRun = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async startTurn() { return { turnId: "turn-recovery" }; }, async *events() { yield runnerEvent(1, "turn.completed"); }, async result() { return { result: completion, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: null }; }, async close() {}, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, normalizeReplayPayload: normalize, completeRun, async appendEvent(event) { const row = event as PrpEvent; history.push({ ...row, payload: normalize(row.payload) }); if (row.eventType === "run.result.accepted" && !lostAcknowledgement) { lostAcknowledgement = true; if (conflict) history.at(-1)!.payload = { ...history.at(-1)!.payload, changed: true }; committed(); return await new Promise(() => undefined); } return { cursor: history.length, highestContiguousSourceSeq: row.sourceSeq, disposition: "committed" }; }, async replayEvents(request) { const events = history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq); return { events, highestContiguousSourceSeq: events.at(-1)?.sourceSeq ?? request.afterSourceSeq }; }, }; const execution = executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "redacted-completion", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; }, }, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 20 }); const outcome = conflict ? expect(execution).rejects.toThrow("native_control_event_replay_conflict:1") : expect(execution).resolves.toMatchObject({ result: completion }); await saved; await vi.advanceTimersByTimeAsync(25); await outcome; expect(completeRun).toHaveBeenCalledTimes(conflict ? 0 : 1); expect(JSON.stringify(history)).not.toContain("private-fixture-value"); expect(history.filter(event => event.eventType === "run.result.accepted")).toHaveLength(1); } finally { vi.useRealTimers(); } }); it("keeps a reconstructed semantic result on its matched terminal turn", async () => { const semanticFingerprint = canonicalTestJson(result); const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "4", semanticResult: result, terminal, activeTurnId: null, terminalTurns: [ { turnId: "turn-with-result", fingerprint: JSON.stringify({ status: "completed", semanticResult: semanticFingerprint, }), }, { turnId: "turn-later-failed", fingerprint: JSON.stringify({ status: "failed" }), }, ], pendingRuntimeRequests: [], lineage: [], }; const events: PrpEvent[] = [ { ...controlEvent(1, "run.result.accepted", { result }), turnId: "turn-with-result", }, ]; const checkpoints: PersistedNativeSession[] = []; const completeRun = vi.fn(async () => undefined); const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); const openSession = vi.fn(async () => { throw new Error( "a recovered run must not open a second provider session", ); }); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() {}, startTurn, async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return structuredClone(checkpoint); }, async close() {}, }; const recoverSession = vi.fn(async () => ({ recovered: true, session })); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, openSession, recoverSession, }; const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents(replay) { const replayed = events.filter( (event) => event.sourceSeq > replay.afterSourceSeq, ); return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events), }; }, completeRun, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); expect(openSession).not.toHaveBeenCalled(); expect(recoverSession).toHaveBeenCalledOnce(); expect(startTurn).not.toHaveBeenCalled(); expect(events.map((event) => event.eventType)).toEqual([ "run.result.accepted", "run.terminal", ]); expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]); expect(events.map((event) => event.turnId)).toEqual([ "turn-with-result", "turn-with-result", ]); expect(completeRun).toHaveBeenCalledOnce(); expect(completeRun).toHaveBeenCalledWith( expect.objectContaining({ turnId: "turn-with-result" }), expect.anything(), ); expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2, }); expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal, }); }); it("rejects visible provider final text without an accepted current-turn completion report", async () => { const capabilities = { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; const events: PrpEvent[] = []; const completeRun = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return capabilities; }, async *events() { yield runnerEvent(1, "item.completed", { kind: "agentMessage", channel: "final", text: "Contacts retrieved: Ada Fixture. Verification code: CONTACTS_fixture." }); yield runnerEvent(2, "turn.completed"); }, async startTurn() { return { turnId: "turn-recovery" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-recovery", cursor: "2", activeTurnId: "turn-recovery", pendingRuntimeRequests: [], lineage: [] }; }, async close() {}, }; await expect(executeNativeSession({ input, backend: { async descriptor() { return { kind: "mock", name: "missing-current-completion", version: "1", capabilities }; }, async openSession() { return session; } }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(structuredClone(event as PrpEvent)); return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 1000, resolveMissingResult: async () => null, })).rejects.toThrow("native_finalization_missing"); expect(completeRun).not.toHaveBeenCalled(); expect(events.some(event => event.eventType === "item.completed")).toBe(true); expect(events.some(event => event.eventType === "run.result.accepted")).toBe(false); }); it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => { const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", sourceEventId: "provider-recovery:1", sourceSeq: 1, sourceInstanceId: "provider-recovery", sourceKind: "runner", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-waiting", eventType: "turn.completed", schemaVersion: 1, priority: 0, emittedAt: "2026-08-09T00:00:00.000Z", payload: {}, }; const yielded: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Waiting for the requested response.", completionClaim: { contractRevision: "1", objectiveSatisfied: false, criteria: [ { criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"], }, ], remainingWork: [ { description: "Resume after the response.", blocksCompletion: true }, ], }, evidence: [{ ref: "interaction:pending" }], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "response_wake", summary: "Resume from the answer.", idempotencyKey: "interaction-response:pending", }, }; const events: PrpEvent[] = []; const completeRun = vi.fn(async () => undefined); const resolveMissingResult = vi.fn(async () => yielded); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield terminalEvent; }, async startTurn() { return { turnId: "turn-waiting" }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-waiting", cursor: "1", activeTurnId: "turn-waiting", pendingRuntimeRequests: [], lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "governed-wait-backend", version: "1", capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents(replay) { const replayed = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq, ); return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed), }; }, completeRun, }; const completed = await executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveMissingResult, }); expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent, }); expect(completed).toMatchObject({ result: yielded, terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded", }, turnId: "turn-waiting", }); expect(completeRun).toHaveBeenCalledWith( expect.objectContaining({ result: yielded }), { signal: expect.any(AbortSignal) }, ); expect(events.map((event) => event.eventType)).toEqual([ "turn.completed", "run.result.accepted", "run.terminal", ]); }); it("settles a governed wait through real Runner shutdown after a missing Codex terminal", async () => { const directory = await mkdtemp(join(tmpdir(), "governed-wait-stop-")); const executionInput = { ...input, workspace: { ...input.workspace, cwd: directory } }; const bundle = createCapabilityRunnerdCodexTransport({ runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(directory, "--emit-tool-call", "--hold-turn", "--accept-interrupt-without-terminal"), stateDirectory: directory, lifecyclePolicy: input.session.lifecyclePolicy, prpIdentity: { runnerInstanceId: "runner-recovery", environmentLeaseId: "lease-recovery", runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: "turn-bootstrap", itemId: "item-bootstrap" }, }); const tool = vi.fn(async () => { await writeFile(join(directory, "saved-work.txt"), "The work and question were saved."); return { success: true, contentItems: [{ type: "inputText", text: "Saved question" }] }; }); const backend = createRunnerdNativeSessionBackend(executionInput, { runnerInstanceId: "runner-recovery", transportFactory: () => bundle.transport, dynamicTools: [{ name: "get_task_context", description: "Persist the fixture handoff", inputSchema: { type: "object", properties: {} } }], dynamicToolHandler: tool, }); const events: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; const complete = vi.fn(async () => undefined); const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); }, async appendEvent(event: PrpEvent) { events.push(event); return { cursor: events.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents(request) { return { events: events.filter(event => event.sourceInstanceId === request.sourceInstanceId && event.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: 0 }; }, }; try { const result = await executeNativeSession({ input: executionInput, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: 30_000, requireSessionCloseBeforeReturn: true, onSession(session) { if (!session?.cancel) return; const cancel = session.cancel.bind(session); session.cancel = request => ({ ...cancel(request), terminalDrainTimeoutMs: 100 }); }, resolveGovernedWait: ({ event }) => event.eventType === "item.completed" && event.payload.kind === "dynamicToolCall" ? yieldedResult : null, }); expect(result).toMatchObject({ result: yieldedResult, terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" } }); expect(tool).toHaveBeenCalledOnce(); expect(complete).toHaveBeenCalledOnce(); expect(await readFile(join(directory, "saved-work.txt"), "utf8")).toBe("The work and question were saved."); expect(checkpoint?.governedWait?.settlement).toMatchObject({ runId: identity.runId, normalizedSessionId: identity.sessionId }); expect(events.some(event => ["turn.completed", "turn.failed"].includes(event.eventType))).toBe(false); const journal = JSON.parse(await readFile(join(directory, "control-plane/control-plane-state.json"), "utf8")); expect(journal.commands.filter((command: { type: string }) => command.type === "turn.start")).toHaveLength(1); expect(journal.commands.find((command: { type: string }) => command.type === "turn.stop")).toMatchObject({ status: "completed", result: { result: { providerExitConfirmed: true } }, }); expect(JSON.parse(await readFile(join(directory, "runner/runner-state.json"), "utf8"))).toMatchObject({ lifecycle: "suspended" }); } finally { await bundle.transport.close().catch(() => undefined); await rm(directory, { recursive: true, force: true }); } }, 45_000); it.each(["confirmed", "restart", "foreign-turn", "no-proof", "stop-failed", "timeout", "provider-failed", "terminal-write-failed"] as const)("settles only verified governed shutdown (%s)", async fault => { vi.useFakeTimers(); try { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; let releaseStream!: () => void; const streamClosed = new Promise(resolve => { releaseStream = resolve; }); let confirmStop!: () => void; const stopConfirmed = new Promise(resolve => { confirmStop = resolve; }); const complete = vi.fn(async () => undefined); const start = vi.fn(async () => ({ turnId: trigger.turnId })); const settleCancelledTurn = vi.fn(async () => { await stopConfirmed; releaseStream(); if (fault === "stop-failed") throw new Error("shutdown unconfirmed"); if (fault === "no-proof") return null; return { runId: identity.runId, normalizedSessionId: identity.sessionId, turnId: fault === "foreign-turn" ? "another-turn" : trigger.turnId }; }); let crashed = false; const session: NativeSession & { settleCancelledTurn: typeof settleCancelledTurn } = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, startTurn: start, cancel() { return { cleanup: Promise.resolve() }; }, settleCancelledTurn, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: null }; }, async result() { return null; }, async close() { releaseStream(); }, async *events() { yield trigger; yield { ...runnerEvent(2, "item.completed", { kind: "interrupt_acknowledgement" }), turnId: trigger.turnId }; await streamClosed; if (fault === "provider-failed" || fault === "terminal-write-failed") yield { ...runnerEvent(3, "turn.failed", { error: { code: "provider_failed" } }), turnId: trigger.turnId }; }, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "verified-stop", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); if (fault === "restart" && value.governedWait?.settlement && !crashed) { crashed = true; throw new Error("controller stopped after settlement checkpoint"); } }, async appendEvent(event: PrpEvent) { if (fault === "terminal-write-failed" && event.eventType === "turn.failed") throw new Error("terminal persistence unavailable"); history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents(request) { return { events: history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: 0 }; }, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: () => yieldedResult }); void execution.catch(() => undefined); await vi.advanceTimersByTimeAsync(15_001); expect(checkpoint?.governedWait).toMatchObject({ sourceEvent: trigger, result: yieldedResult }); expect(complete).not.toHaveBeenCalled(); expect(settleCancelledTurn).toHaveBeenCalledOnce(); if (fault === "timeout") { await vi.advanceTimersByTimeAsync(30_500); await expect(execution).rejects.toMatchObject({ code: "native_governed_wait_settlement_pending" }); confirmStop(); await vi.advanceTimersByTimeAsync(1); expect(complete).not.toHaveBeenCalled(); expect(checkpoint?.governedWait?.settlement).toBeUndefined(); return; } confirmStop(); await vi.advanceTimersByTimeAsync(1); if (["foreign-turn", "no-proof", "stop-failed", "provider-failed", "terminal-write-failed"].includes(fault)) { await expect(execution).rejects.toMatchObject({ code: fault === "foreign-turn" ? "native_event_replay_conflict" : fault === "provider-failed" ? "native_provider_terminal_failed" : "native_governed_wait_settlement_pending" }); expect(complete).not.toHaveBeenCalled(); expect(checkpoint?.governedWait?.settlement).toBeUndefined(); return; } if (fault === "restart") { await expect(execution).rejects.toThrow("controller stopped after settlement checkpoint"); expect(complete).not.toHaveBeenCalled(); const answerAlreadySaved = vi.fn(() => null); await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: answerAlreadySaved })) .resolves.toMatchObject({ result: yieldedResult }); expect(answerAlreadySaved).not.toHaveBeenCalled(); expect(settleCancelledTurn).toHaveBeenCalledOnce(); } else { expect(await execution).toMatchObject({ result: yieldedResult, terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" } }); } expect(complete).toHaveBeenCalledOnce(); expect(start).toHaveBeenCalledOnce(); expect(history.filter(event => event.eventType === "run.result.accepted")).toHaveLength(1); expect(history.some(event => event.eventType === "turn.completed")).toBe(false); } finally { vi.useRealTimers(); } }); it.each(["end", "throw", "timeout"])("keeps a governed wait unfinalized when complete usage is followed by %s without a provider terminal", async fault => { vi.useFakeTimers(); try { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const usage = { ...runnerEvent(2, "item.completed", { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } }), turnId: trigger.turnId }; const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; let releaseStream!: () => void; const streamClosed = new Promise(resolve => { releaseStream = resolve; }); const complete = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async startTurn() { return { turnId: trigger.turnId }; }, cancel() { return { cleanup: Promise.resolve() }; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId }; }, async result() { return null; }, async close() { releaseStream(); }, async *events() { yield trigger; yield usage; if (fault === "throw") throw new Error("provider stream lost"); if (fault === "timeout") await streamClosed; }, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "missing-stop", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); }, async appendEvent(event: PrpEvent) { history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents(request) { return { events: history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: 0 }; }, }; const outcome = expect(executeNativeSession({ input, backend, controlPlane: port, timeoutMs: 1_000, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: () => yieldedResult })) .rejects.toMatchObject({ code: "native_governed_wait_settlement_pending" }); void outcome.catch(() => undefined); await vi.advanceTimersByTimeAsync(600); await outcome; expect(checkpoint?.governedWait).toEqual({ sourceEvent: trigger, result: yieldedResult }); expect(complete).not.toHaveBeenCalled(); expect(history).toEqual([trigger, usage]); } finally { vi.useRealTimers(); } }); it.each([ { name: "default", bound: undefined, limit: 15_000 }, { name: "not-finite", bound: Number.NaN, limit: 15_000 }, { name: "infinite", bound: Infinity, limit: 15_000 }, { name: "negative", bound: -1, limit: 15_000 }, { name: "nonpositive", bound: 0, limit: 15_000 }, { name: "oversized", bound: 30_001, limit: 15_000 }, { name: "fractional", bound: 30_000.5, limit: 15_000 }, { name: "extended-missing", bound: 30_000, limit: 30_000 }, { name: "short-operation", bound: 30_000, limit: 250, operationMs: 1_000 }, { name: "foreign-terminal", bound: 30_000, limit: 30_000, fault: "foreign" }, { name: "unproven-terminal", bound: 30_000, limit: 0, fault: "unproven" }, { name: "startup-abort", bound: 30_000, limit: 100, fault: "abort" }, ])("bounds governed terminal observation without finalizing an unproven wait: $name", async ({ bound, limit, operationMs, fault }) => { vi.useFakeTimers(); try { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; let releaseStream!: () => void; const streamClosed = new Promise(resolve => { releaseStream = resolve; }); let cancelAt = 0; const cancel = vi.fn(({ signal }: { signal: AbortSignal }) => { cancelAt = Date.now(); signal.addEventListener("abort", releaseStream, { once: true }); return { cleanup: Promise.resolve(), terminalDrainTimeoutMs: bound }; }); const complete = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async startTurn() { if (fault === "abort") await new Promise((_, reject) => setTimeout(() => reject(new Error("controlled startup abort")), 100)); return { turnId: trigger.turnId }; }, cancel, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId }; }, async result() { return null; }, async close() { releaseStream(); }, async *events() { yield trigger; if (fault === "foreign" || fault === "unproven") yield { ...runnerEvent(2, "turn.interrupted", { providerTerminalObserved: fault !== "unproven" }), turnId: fault === "foreign" ? "foreign-turn" : trigger.turnId }; await streamClosed; }, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "bounded-cancellation", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; } }; const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); }, async appendEvent(event: PrpEvent) { history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents(request) { return { events: history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: 0 }; }, }; const execution = executeNativeSession({ input, backend, controlPlane: port, timeoutMs: operationMs ?? 120_000, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: () => yieldedResult }); let settled = false; void execution.then(() => { settled = true; }, () => { settled = true; }); await vi.waitFor(() => expect(cancel).toHaveBeenCalledOnce()); if (limit > 0) { await vi.advanceTimersByTimeAsync(Math.max(0, limit - 1 - (Date.now() - cancelAt))); expect(settled).toBe(false); } await vi.advanceTimersByTimeAsync(fault === "abort" ? 1_002 : 2); if (fault === "abort") await expect(execution).rejects.toThrow("controlled startup abort"); else await expect(execution).rejects.toMatchObject({ code: fault === "unproven" ? "native_provider_terminal_failed" : "native_governed_wait_settlement_pending" }); expect(checkpoint?.governedWait).toEqual({ sourceEvent: trigger, result: yieldedResult }); expect(complete).not.toHaveBeenCalled(); expect(history.some(event => event.eventType === "run.terminal")).toBe(false); if (fault === "foreign") expect(history).toEqual([trigger]); } finally { vi.useRealTimers(); } }); it.each(["interrupt_acknowledgement", "runtime_request.cancelled", "runtime_request.expired"])("retains %s in contiguous governed-wait history for recovery", async cancellationKind => { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const acknowledgement: PrpEvent = { ...runnerEvent(2, cancellationKind === "interrupt_acknowledgement" ? "item.completed" : cancellationKind as "runtime_request.cancelled" | "runtime_request.expired", cancellationKind === "interrupt_acknowledgement" ? { kind: cancellationKind } : { requestId: "request-1" }), turnId: trigger.turnId }; const usage = { ...runnerEvent(3, "item.completed", { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } }), turnId: trigger.turnId }; const stopped = { ...runnerEvent(4, "turn.interrupted", { providerTerminalObserved: true }), turnId: trigger.turnId }; const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; const start = vi.fn(async () => ({ turnId: trigger.turnId })); const cancel = vi.fn(() => ({ cleanup: Promise.resolve() })); const complete = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, startTurn: start, cancel, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId }; }, async result() { return null; }, async close() {}, async *events() { yield trigger; yield acknowledgement; yield usage; yield stopped; }, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "cancel-history", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; }, async recoverSession() { return { recovered: true, session }; }, }; const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); }, async appendEvent(event: PrpEvent) { const duplicate = history.some(row => row.sourceEventId === event.sourceEventId); if (!duplicate) history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: duplicate ? "duplicate" : "committed" }; }, async replayEvents(request) { const events = history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq); return { events, highestContiguousSourceSeq: events.at(-1)?.sourceSeq ?? request.afterSourceSeq }; }, }; const options = { input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }; await expect(executeNativeSession({ ...options, resolveGovernedWait: () => yieldedResult })) .resolves.toMatchObject({ result: yieldedResult }); expect(history.filter(event => event.sourceInstanceId === trigger.sourceInstanceId)).toEqual([trigger, acknowledgement, usage, stopped]); await expect(executeNativeSession({ ...options, persistedSession: checkpoint })).resolves.toMatchObject({ result: yieldedResult }); expect(start).toHaveBeenCalledOnce(); expect(cancel).toHaveBeenCalledOnce(); }); it.each(["trigger", "usage", "terminal"])("recovers a journaled governed wait after its answer is saved (last committed: %s)", async lastCommitted => { const terminalCommitted = lastCommitted === "terminal"; const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall", item: { id: "ask-1", name: "request_human_input" } }), turnId: "turn-waiting" }; const usage = { ...runnerEvent(2, "item.completed", { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } }), turnId: trigger.turnId }; const stopped = { ...runnerEvent(3, "turn.interrupted", { providerTerminalObserved: true }), turnId: trigger.turnId }; const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; let crashed = false; let interrupted = false; const revoke = vi.fn(); const cancel = vi.fn(() => { interrupted = true; return { cleanup: Promise.resolve() }; }); const start = vi.fn(async () => ({ turnId: trigger.turnId })); const complete = vi.fn(async () => undefined); const snapshot = (): PersistedNativeSession => ({ backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "same-provider", activeTurnId: trigger.turnId, cursor: String(history.at(-1)?.sourceSeq ?? 0) }); const makeSession = (recovering: boolean): NativeSession => ({ identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, startTurn: start, revokeTurnPublication: revoke, cancel, async snapshot() { return snapshot(); }, async result() { return null; }, async close() {}, async *events() { if (!recovering) { yield trigger; return; } // The restart resumes settlement only; it must not submit a model turn. expect(interrupted).toBe(true); yield usage; yield stopped; }, }); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "durable-wait", version: "1", capabilities: await makeSession(false).capabilities() }; }, async openSession() { return makeSession(false); }, async recoverSession() { return { recovered: true, session: makeSession(true) }; }, }; const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); if (value.governedWait && !crashed) { expect(revoke).toHaveBeenCalledOnce(); expect(cancel).not.toHaveBeenCalled(); crashed = true; throw new Error("controller lost after governed checkpoint"); } }, async appendEvent(event: PrpEvent) { if (!history.some(row => row.sourceEventId === event.sourceEventId)) history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents(request) { const events = history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq); return { events, highestContiguousSourceSeq: events.at(-1)?.sourceSeq ?? request.afterSourceSeq }; }, }; const options = { input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }; await expect(executeNativeSession({ ...options, resolveGovernedWait: () => yieldedResult })) .rejects.toThrow("controller lost after governed checkpoint"); expect(checkpoint?.governedWait).toEqual({ sourceEvent: trigger, result: yieldedResult }); expect(complete).not.toHaveBeenCalled(); // The failed controller's ordinary cleanup also invokes cancel. Its // checkpoint assertion above proves interruption followed persistence. cancel.mockClear(); interrupted = false; if (lastCommitted !== "trigger") history.push(usage); if (terminalCommitted) history.push(stopped); const answerAlreadySaved = vi.fn(() => null); await expect(executeNativeSession({ ...options, persistedSession: checkpoint, resolveGovernedWait: answerAlreadySaved })) .resolves.toMatchObject({ result: yieldedResult, turnId: trigger.turnId, terminal: { runTerminalState: "succeeded" } }); expect(start).toHaveBeenCalledOnce(); expect(cancel).toHaveBeenCalledTimes(terminalCommitted ? 0 : 1); expect(answerAlreadySaved).not.toHaveBeenCalled(); expect(complete).toHaveBeenCalledOnce(); expect(history.filter(event => event.payload.kind === "usage")).toEqual([usage]); }); it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "failed-terminal", "lost-not-recoverable", "different-turn", "foreign-settlement", "active-settled-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId, governedWait: { sourceEvent: trigger, result: yieldedResult } }; const history = fault === "missing-trigger" ? [] : [fault === "changed-trigger" ? { ...trigger, payload: {} } : trigger]; if (fault === "foreign-settlement" || fault === "active-settled-turn") checkpoint.governedWait!.settlement = { runId: fault === "foreign-settlement" ? "another-run" : identity.runId, normalizedSessionId: identity.sessionId, turnId: trigger.turnId, }; if (fault === "unproven-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "unproven", eventType: "turn.failed", payload: { providerTerminalObserved: false } }); // Only a recoverable restart loss proves the old provider lifetime ended. if (fault === "failed-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "failed", eventType: "turn.failed", payload: { status: "failed", error: { code: "model_not_found", recoverable: false } } }); if (fault === "lost-not-recoverable") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "lost", eventType: "turn.failed", payload: { status: "failed", error: { code: "provider_turn_lost_on_restore", recoverable: false } } }); if (fault === "different-turn") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "new-turn", turnId: "different", eventType: "turn.started" }); const start = vi.fn(); const complete = vi.fn(); const session: NativeSession = { identity: () => identity, startTurn: start, async result() { return null; }, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async snapshot() { return checkpoint; }, async *events() {}, async close() {} }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "unsafe-wait", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; }, async recoverSession() { return { recovered: true, session }; } }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected append"); }, completeRun: complete, async replayEvents(request) { return { events: history.filter(row => row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: history.at(-1)?.sourceSeq ?? 0 }; } }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow(/governed_wait|failed terminal|native_event_replay_conflict/); expect(start).not.toHaveBeenCalled(); expect(complete).not.toHaveBeenCalled(); }); // Restart chaos (Codex Q1 with PV1/PV2 + K1): the provider held or dropped // the parked turn's terminal and the controller died during the accounting // drain. The drain had already discarded numbered non-accounting events, so // the saved replay has gaps after the trigger. // Claude (ACPX) Q1 with PV1/PV2 + K1 (wave w-20261110T094930-s214): the // restored runner proves the old provider lifetime ended and reports the // parked turn as lost on restore instead of interrupted. it.each(["terminal-after-restart", "late-terminal-saved", "lost-on-restore-after-restart", "lost-on-restore-saved"])("settles a saved governed wait whose drain left sequence gaps before a restart (%s)", async variant => { const lateTerminalSaved = variant.endsWith("-saved"); const lostOnRestore = variant.startsWith("lost-on-restore"); const turn = (event: PrpEvent) => ({ ...event, turnId: "turn-waiting" }); const trigger = turn(runnerEvent(1, "item.completed", { kind: "dynamicToolCall", item: { id: "ask-1", name: "request_human_input" } })); const acknowledgement = turn(runnerEvent(2, "item.completed", { kind: "interrupt_acknowledgement", status: "acknowledged" })); const discardedMessage = turn(runnerEvent(3, "item.completed", { kind: "agentMessage", text: "I asked the board." })); const discardedToolEnd = turn(runnerEvent(4, "tool.execution.completed", { name: "request_human_input", status: "completed" })); const usage = turn(runnerEvent(5, "item.completed", { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } })); const stopped = lostOnRestore ? turn(runnerEvent(6, "turn.failed", { provider: "acpx", status: "failed", providerTerminalObserved: false, code: "provider_turn_lost_on_restore", error: { code: "provider_turn_lost_on_restore", recoverable: true } })) : turn(runnerEvent(6, "turn.interrupted", { providerTerminalObserved: true })); const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; const start = vi.fn(async () => ({ turnId: trigger.turnId })); const cancel = vi.fn(() => ({ cleanup: Promise.resolve() })); const complete = vi.fn(async () => undefined); const snapshot = (): PersistedNativeSession => ({ backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "same-provider", activeTurnId: trigger.turnId, cursor: String(history.at(-1)?.sourceSeq ?? 0) }); const makeSession = (recovering: boolean): NativeSession => ({ identity: () => identity, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, startTurn: start, revokeTurnPublication: vi.fn(), cancel, async snapshot() { return snapshot(); }, async result() { return null; }, async close() {}, async *events() { if (!recovering) { // The terminal is held (or dropped) and the process then dies. yield trigger; yield acknowledgement; yield discardedMessage; yield discardedToolEnd; yield usage; return; } if (!lateTerminalSaved) yield stopped; }, }); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "drain-gap", version: "1", capabilities: await makeSession(false).capabilities() }; }, async openSession() { return makeSession(false); }, async recoverSession() { return { recovered: true, session: makeSession(true) }; }, }; const port: ControlPlanePort = { async openRun() {}, completeRun: complete, async checkpointSession(value) { checkpoint = structuredClone(value); }, async appendEvent(event: PrpEvent) { const duplicate = history.some(row => row.sourceEventId === event.sourceEventId); if (!duplicate) history.push(event); return { cursor: history.length, highestContiguousSourceSeq: event.sourceSeq, disposition: duplicate ? "duplicate" : "committed" }; }, async replayEvents(request) { const events = history.filter(row => row.sourceInstanceId === request.sourceInstanceId && row.sourceSeq > request.afterSourceSeq) .sort((left, right) => left.sourceSeq - right.sourceSeq); return { events, highestContiguousSourceSeq: events.at(-1)?.sourceSeq ?? request.afterSourceSeq }; }, }; const options = { input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" }; await expect(executeNativeSession({ ...options, resolveGovernedWait: ({ event }) => event.sourceSeq === 1 ? yieldedResult : null })) .rejects.toThrow(); // The question is saved, and the drain kept only accounting facts. expect(checkpoint?.governedWait).toEqual({ sourceEvent: trigger, result: yieldedResult }); expect(history).toEqual([trigger, acknowledgement, usage]); expect(complete).not.toHaveBeenCalled(); cancel.mockClear(); if (lateTerminalSaved) history.push(stopped); const answerAlreadySaved = vi.fn(() => null); await expect(executeNativeSession({ ...options, persistedSession: checkpoint, resolveGovernedWait: answerAlreadySaved })) .resolves.toMatchObject({ result: yieldedResult, turnId: trigger.turnId, terminal: { runTerminalState: "succeeded" } }); expect(start).toHaveBeenCalledOnce(); // A saved terminal settles without interrupting provider work again. expect(cancel).toHaveBeenCalledTimes(lateTerminalSaved ? 0 : 1); expect(answerAlreadySaved).not.toHaveBeenCalled(); expect(complete).toHaveBeenCalledOnce(); expect(history.filter(event => event.sourceInstanceId === trigger.sourceInstanceId)).toEqual([trigger, acknowledgement, usage, stopped]); // A further recovery of the settled wait neither submits nor settles again. const settledCheckpoint = structuredClone(checkpoint); await expect(executeNativeSession({ ...options, persistedSession: settledCheckpoint })) .resolves.toMatchObject({ result: yieldedResult, turnId: trigger.turnId, terminal: { runTerminalState: "succeeded" } }); expect(start).toHaveBeenCalledOnce(); expect(cancel).toHaveBeenCalledTimes(lateTerminalSaved ? 0 : 1); expect(history.filter(event => event.sourceInstanceId === trigger.sourceInstanceId)).toEqual([trigger, acknowledgement, usage, stopped]); }); it.each([ "gap-before-trigger", "gap-foreign-run", "gap-foreign-session", "gap-foreign-instance", "gap-other-turn-usage", "gap-unaccounted-event", "unaccounted-event-after-gap", ])("keeps rejecting an unproven governed-wait replay with a sequence gap: %s", async fault => { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const afterGap = (sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record, overrides: Partial = {}) => ({ ...runnerEvent(sourceSeq, eventType, payload), turnId: trigger.turnId, ...overrides }) as PrpEvent; const usagePayload = { kind: "usage", usage: { runDelta: { inputTokens: 1, outputTokens: 1 }, runDeltaComplete: true } }; const history: PrpEvent[] = fault === "gap-before-trigger" ? [{ ...trigger, sourceSeq: 2, sourceEventId: "late-trigger" }] : [trigger]; if (fault === "gap-foreign-run") history.push(afterGap(3, "item.completed", usagePayload, { runId: "another-run" })); if (fault === "gap-foreign-session") history.push(afterGap(3, "item.completed", usagePayload, { normalizedSessionId: "another-session" })); if (fault === "gap-foreign-instance") history.push(afterGap(3, "item.completed", usagePayload, { sourceInstanceId: "another-runner" })); if (fault === "gap-other-turn-usage") history.push(afterGap(3, "item.completed", usagePayload, { turnId: "another-turn" })); if (fault === "gap-unaccounted-event") history.push(afterGap(3, "item.completed", { kind: "agentMessage", text: "more work" })); if (fault === "unaccounted-event-after-gap") { history.push(afterGap(3, "item.completed", usagePayload), afterGap(4, "item.delta", { kind: "agentMessage", delta: "more" })); } const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId, governedWait: { sourceEvent: trigger, result: yieldedResult } }; const start = vi.fn(); const complete = vi.fn(); const session: NativeSession = { identity: () => identity, startTurn: start, async result() { return null; }, async capabilities() { return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async snapshot() { return checkpoint; }, async *events() {}, async close() {} }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "unsafe-gap", version: "1", capabilities: await session.capabilities() }; }, async openSession() { return session; }, async recoverSession() { return { recovered: true, session }; } }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { throw new Error("unexpected append"); }, completeRun: complete, async replayEvents(request) { return { events: history.filter(row => row.sourceSeq > request.afterSourceSeq), highestContiguousSourceSeq: request.afterSourceSeq }; } }; await expect(executeNativeSession({ input, backend, controlPlane: port, persistedSession: checkpoint, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery" })).rejects.toThrow("native_governed_wait_replay_mismatch"); expect(start).not.toHaveBeenCalled(); expect(complete).not.toHaveBeenCalled(); }); it.each([0, 6_000, 12_000])("retains actual shutdown usage after %s ms before finalizing a governed wait", async (receiptDelayMs) => { vi.useFakeTimers(); try { const yielded: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "yielded", summary: "Waiting for the requested response.", completionClaim: { contractRevision: "1", objectiveSatisfied: false, criteria: [ { criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"], }, ], remainingWork: [ { description: "Resume after the response.", blocksCompletion: true }, ], }, evidence: [{ ref: "interaction:pending" }], verification: [], attentionRequests: [], artifacts: [], continuation: { kind: "response_wake", summary: "Resume from the answer.", idempotencyKey: "interaction-response:pending", }, }; const itemCompleted: PrpEvent = { ...controlEvent(1, "item.completed", { kind: "dynamicToolCall", item: { id: "ask-1", name: "ask_user_questions" }, }), sourceEventId: "provider-recovery:1", sourceInstanceId: "provider-recovery", sourceKind: "runner", turnId: "turn-waiting", }; const turnInterrupted: PrpEvent = { ...controlEvent(3, "turn.interrupted", { reason: "governed_wait" }), sourceEventId: "provider-recovery:3", sourceInstanceId: "provider-recovery", sourceKind: "runner", turnId: "turn-waiting", }; let releaseCancelled!: () => void; const cancelled = new Promise((resolve) => { releaseCancelled = resolve; }); const cancel = vi.fn(() => { releaseCancelled(); return { cleanup: Promise.resolve() }; }); const events: PrpEvent[] = []; const completeRun = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { yield itemCompleted; await cancelled; await new Promise(resolve => setTimeout(resolve, receiptDelayMs)); yield { ...itemCompleted, sourceSeq: 2, sourceEventId: "provider-recovery:2", payload: { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } } }; yield turnInterrupted; }, async startTurn() { return { turnId: "turn-waiting" }; }, cancel, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-waiting", cursor: "2", activeTurnId: "turn-waiting", pendingRuntimeRequests: [], lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "governed-wait-backend", version: "1", capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed", }; }, async replayEvents(replay) { const replayed = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq, ); return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed), }; }, completeRun, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", resolveGovernedWait: ({ event }) => event.eventType === "item.completed" ? yielded : null, }); await vi.advanceTimersByTimeAsync(receiptDelayMs + 1); const completed = await execution; expect(cancel).toHaveBeenCalledOnce(); expect(events.find(event => event.payload.kind === "usage")?.payload.usage).toMatchObject({ runDelta: { inputTokens: 7, outputTokens: 3 } }); expect(completed).toMatchObject({ result: yielded, terminal: { turnTerminalState: "completed", runTerminalState: "succeeded", reportedWorkDisposition: "yielded", }, }); expect(events.map((event) => event.eventType)).toEqual([ "item.completed", "item.completed", "turn.interrupted", "run.result.accepted", "run.terminal", ]); } finally { vi.useRealTimers(); } }); it("hands a committed structured input to the durable wait after its live window", async () => { vi.useFakeTimers(); try { const questionSet = { schema: "paperclip.question_set.v1" as const, questions: [ { id: "region", prompt: "Which region?", required: true, answerMode: "single_select" as const, options: [ { id: "us", label: "US" }, { id: "eu", label: "Europe" }, ], }, ], }; const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "input-1", type: "input", status: "pending", prompt: "Which region?", input: questionSet, origin: { adapter: "mock" }, turnId: "turn-waiting", itemId: "input-1", }; const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting", }; const expired = { ...runnerEvent(2, "runtime_request.expired", { requestId: "input-1", requestKind: "runtime", turnId: "turn-waiting", itemId: "input-1", reason: "durable_handoff", replayAllowed: false, requestType: "input", request, }), turnId: "turn-waiting", }; const interrupted = { ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), turnId: "turn-waiting", }; let releaseHandoff!: () => void; const handedOff = new Promise((resolve) => { releaseHandoff = resolve; }); let releaseCancelled!: () => void; const cancelled = new Promise((resolve) => { releaseCancelled = resolve; }); let releaseCreated!: () => void; const createdCommitted = new Promise((resolve) => { releaseCreated = resolve; }); const handoffRuntimeRequest = vi.fn(() => { releaseHandoff(); return { result: "handed_off" as const, cleanup: Promise.resolve() }; }); const cancel = vi.fn(() => { releaseCancelled(); return { cleanup: Promise.resolve() }; }); const events: PrpEvent[] = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, runtimeRequestHandoff: true, }; }, async *events() { yield created; await handedOff; yield expired; await cancelled; yield interrupted; }, async startTurn() { return { turnId: "turn-waiting" }; }, handoffRuntimeRequest, cancel, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-waiting", cursor: "3", activeTurnId: "turn-waiting", pendingRuntimeRequests: [], lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "runtime-input-wait-backend", version: "1", capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, runtimeRequestHandoff: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { events.push(structuredClone(event as PrpEvent)); if (event.eventType === "runtime_request.created") releaseCreated(); const sourceEvents = events.filter( (candidate) => candidate.sourceInstanceId === event.sourceInstanceId, ); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), disposition: "committed", }; }, async replayEvents(replay) { const replayed = events.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq, ); return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed), }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", runtimeInputLiveWindowMs: 120, resolveGovernedWait: ({ event }) => event.eventType === "runtime_request.expired" ? yieldedResult : null, }); await createdCommitted; expect(handoffRuntimeRequest).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(119); expect(handoffRuntimeRequest).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(1); await expect(execution).resolves.toMatchObject({ result: yieldedResult }); expect(handoffRuntimeRequest).toHaveBeenCalledWith({ requestId: "input-1", turnId: "turn-waiting", reason: "durable_handoff", signal: expect.any(AbortSignal), }); expect(cancel).toHaveBeenCalledOnce(); expect(events.map((event) => event.eventType)).toContain( "runtime_request.expired", ); } finally { vi.useRealTimers(); } }); it("aborts and bounds a durable handoff that never settles", async () => { const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "input-stalled", type: "input", status: "pending", prompt: "Which region?", input: { schema: "paperclip.question_set.v1", questions: [ { id: "region", prompt: "Which region?", required: true, answerMode: "text", }, ], }, origin: { adapter: "mock" }, turnId: "turn-stalled", itemId: "input-stalled", }; const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-stalled", }; let releaseEvents = () => {}; const eventsReleased = new Promise((resolve) => { releaseEvents = resolve; }); let markHandoffStarted = () => {}; const handoffStarted = new Promise((resolve) => { markHandoffStarted = resolve; }); let handoffSignal: AbortSignal | undefined; let releaseHandoff = () => {}; const close = vi.fn(async () => releaseEvents()); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, runtimeRequestHandoff: true, }; }, async *events() { yield created; await eventsReleased; }, async startTurn() { return { turnId: "turn-stalled" }; }, handoffRuntimeRequest(input) { handoffSignal = input.signal; markHandoffStarted(); return { result: "handed_off", cleanup: new Promise((resolve) => { releaseHandoff = resolve; }), }; }, cancel() { releaseEvents(); return { cleanup: Promise.resolve() }; }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-stalled", activeTurnId: "turn-stalled", }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "stalled-handoff-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", runtimeInputLiveWindowMs: 1, timeoutMs: 25, keepSessionOpen: true, }); await handoffStarted; await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); expect(handoffSignal?.aborted).toBe(true); await expect(execution).rejects.toThrow("native session timed out"); releaseHandoff(); expect(close).toHaveBeenCalledOnce(); }); it("preserves terminal success while iterator teardown remains pending", async () => { vi.useFakeTimers(); try { let releaseTeardown = () => {}; const teardownStarted = vi.fn(); const close = vi.fn(async () => undefined); const readResult = vi.fn(async () => ({ result, terminal, turnId: "turn-terminal", })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, }; }, async *events() { try { yield { ...runnerEvent(1, "turn.completed"), turnId: "turn-terminal", }; } finally { teardownStarted(); await new Promise((resolve) => { releaseTeardown = resolve; }); } }, async startTurn() { return { turnId: "turn-terminal" }; }, result: readResult, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-terminal", cursor: "1", activeTurnId: null, }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "slow-teardown-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const completeRun = vi.fn(async () => undefined); const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", }); await vi.waitFor(() => expect(teardownStarted).toHaveBeenCalledOnce()); await vi.advanceTimersByTimeAsync(100); await expect(execution).resolves.toMatchObject({ result }); expect(readResult).toHaveBeenCalledOnce(); expect(completeRun).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); releaseTeardown(); await Promise.resolve(); } finally { vi.useRealTimers(); } }); it("preserves terminal success while quarantining stalled handoff cleanup", async () => { const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "input-terminal", type: "input", status: "pending", prompt: "Which region?", input: { schema: "paperclip.question_set.v1", questions: [ { id: "region", prompt: "Which region?", required: true, answerMode: "text", }, ], }, origin: { adapter: "mock" }, turnId: "turn-terminal", itemId: "input-terminal", }; let markHandoffStarted = () => {}; const handoffStarted = new Promise((resolve) => { markHandoffStarted = resolve; }); let releaseHandoff = () => {}; let handoffSignal: AbortSignal | undefined; const close = vi.fn(async () => undefined); const onSession = vi.fn(); const completeRun = vi.fn(async () => undefined); const readResult = vi.fn(async () => ({ result, terminal, turnId: "turn-terminal", })); const providerEvents = [ { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-terminal", }, { ...runnerEvent(2, "turn.completed"), turnId: "turn-terminal" }, ]; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, runtimeRequestHandoff: true, }; }, async *events() { yield providerEvents[0]!; await handoffStarted; yield providerEvents[1]!; }, async startTurn() { return { turnId: "turn-terminal" }; }, handoffRuntimeRequest(input) { handoffSignal = input.signal; markHandoffStarted(); return { result: "handed_off", cleanup: new Promise((resolve) => { releaseHandoff = resolve; }), }; }, result: readResult, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-terminal", cursor: "2", activeTurnId: null, }; }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "terminal-handoff-backend", version: "1", capabilities: await session.capabilities(), }; }, async openSession() { return session; }, }; const appended: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event) { appended.push(structuredClone(event as PrpEvent)); return { cursor: appended.length, highestContiguousSourceSeq: highestContiguous(appended), disposition: "committed", }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, }; const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", runtimeInputLiveWindowMs: 1, keepSessionOpen: true, onSession, }); await handoffStarted; await vi.waitFor(() => expect(handoffSignal?.aborted).toBe(true)); await expect(execution).resolves.toMatchObject({ result }); expect(readResult).toHaveBeenCalledOnce(); expect(completeRun).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenLastCalledWith(null); releaseHandoff(); await Promise.resolve(); expect(close).toHaveBeenCalledOnce(); expect(completeRun).toHaveBeenCalledOnce(); }); it("keeps a settling structured input in the original turn while its append crosses expiry", async () => { const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", requestId: "input-live", type: "input", status: "pending", prompt: "Which region?", input: { schema: "paperclip.question_set.v1", questions: [ { id: "region", prompt: "Which region?", required: true, answerMode: "text", }, ], }, origin: { adapter: "mock" }, turnId: "turn-live", itemId: "input-live", }; const providerEvents = [ { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-live", }, { ...runnerEvent(2, "runtime_request.resolved", { requestId: "input-live", requestKind: "user_input", turnId: "turn-live", itemId: "input-live", action: "submit", requestType: "input", }), turnId: "turn-live", }, { ...runnerEvent(3, "turn.completed"), turnId: "turn-live" }, ]; const appended: PrpEvent[] = []; let markSettlementAppendStarted!: () => void; const settlementAppendStarted = new Promise((resolve) => { markSettlementAppendStarted = resolve; }); let releaseSettlementAppend!: () => void; const settlementAppendReleased = new Promise((resolve) => { releaseSettlementAppend = resolve; }); const handoffRuntimeRequest = vi.fn(() => ({ result: "handed_off" as const, cleanup: Promise.resolve(), })); const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, runtimeRequestHandoff: true, }; }, async *events() { yield* providerEvents; }, async startTurn() { return { turnId: "turn-live" }; }, handoffRuntimeRequest, async result() { return { result, terminal, turnId: "turn-live" }; }, async snapshot() { return { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-live", cursor: "3", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; }, async close() {}, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "runtime-input-live-backend", version: "1", capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true, runtimeRequestHandoff: true, }, }; }, async openSession() { return session; }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event: PrpEvent) { if (event.eventType === "runtime_request.resolved") { markSettlementAppendStarted(); await settlementAppendReleased; } appended.push(structuredClone(event as PrpEvent)); const sourceEvents = appended.filter( (candidate) => candidate.sourceInstanceId === event.sourceInstanceId, ); return { cursor: appended.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), disposition: "committed", }; }, async replayEvents(replay) { const replayed = appended.filter( (event) => event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq, ); return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed), }; }, async completeRun() {}, }; const originalSetTimeout = globalThis.setTimeout; let queuedHandoffCallback = null as (() => void) | null; const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((( callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[] ) => { if (delay === 123_456) { queuedHandoffCallback = () => callback(...args); const handle = originalSetTimeout(() => undefined, 60_000); handle.unref?.(); return handle; } return originalSetTimeout(callback, delay, ...args); }) as typeof setTimeout); try { const execution = executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", runtimeInputLiveWindowMs: 123_456, }); await settlementAppendStarted; expect(queuedHandoffCallback).not.toBeNull(); queuedHandoffCallback?.(); await Promise.resolve(); expect(handoffRuntimeRequest).not.toHaveBeenCalled(); releaseSettlementAppend(); await expect(execution).resolves.toMatchObject({ result }); queuedHandoffCallback?.(); await Promise.resolve(); expect(handoffRuntimeRequest).not.toHaveBeenCalled(); expect( appended.some((event) => event.eventType === "runtime_request.expired"), ).toBe(false); } finally { releaseSettlementAppend(); timeoutSpy.mockRestore(); } }); }); it("fails preconfigured recovery without replacing a missing provider session", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-preconfigured", identity, providerSessionId: "provider-missing", providerRecoveryPolicy: "allow_replacement_after_resume_failure", cursor: "0", activeTurnId: null, pendingRuntimeRequests: [], lineage: [], }; const openSession = vi.fn(); const openReplacementSession = vi.fn(); const backend: NativeSessionBackend = { descriptor: async () => ({ kind: "mock", name: "preconfigured", version: "1", runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" }, capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true } }), openSession, openReplacementSession, recoverSession: async () => ({ recovered: false, reason: "provider history is missing" }), }; const port: ControlPlanePort = { openRun: vi.fn(), appendEvent: vi.fn(), completeRun: vi.fn(), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), }; await expect(executeNativeSession({ input: { ...preparedInput(), schema: "paperclip.native-execution-input.v7", homeMode: "preconfigured" }, backend, controlPlane: port, runnerInstanceId: "runner-preconfigured", controlPlaneInstanceId: "control-preconfigured", persistedSession: checkpoint, })).rejects.toThrow("native_session_recovery_failed: provider history is missing"); expect(openSession).not.toHaveBeenCalled(); expect(openReplacementSession).not.toHaveBeenCalled(); }); describe("first-turn queued input", () => { async function startFirstTurn(executionInput: NativeExecutionInputV1 | NativeExecutionInputV5, queued: string | null) { const order: string[] = []; const sent: string[] = []; const session: NativeSession = { identity: () => identity, async capabilities() { return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; }, async *events() {}, async startTurn(turn) { order.push("turn.start"); sent.push(turn.message.text); throw new Error("stop after turn.start"); }, async result() { return null; }, async snapshot() { return { backendKind: "mock", sessionId: "driver-first-turn", identity, providerSessionId: "provider-first-turn", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(async () => undefined), }; const getFirstTurnQueuedInput = vi.fn(async () => { order.push("claim"); return queued; }); await expect(executeNativeSession({ input: executionInput, getFirstTurnQueuedInput, backend: { async descriptor() { return { kind: "mock", name: "first-turn", version: "1", capabilities: await session.capabilities(), runtimeContextCapabilities: { instructions: "native" as const, skills: "native" as const, mcp: "native" as const } }; }, async openSession() { return session; } }, controlPlane: { async openRun() {}, async checkpointSession() {}, async appendEvent() { return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" as const }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {} }, runnerInstanceId: "runner-first-turn", controlPlaneInstanceId: "control-first-turn", })).rejects.toThrow("stop after turn.start"); return { order, sent, getFirstTurnQueuedInput }; } it("adds messages queued during preparation to the first turn, sealed just before turn.start", async () => { const queued = "New messages from the user arrived while this run was starting.\n\n[Message 1 of 2]\nfirst\n\n[Message 2 of 2]\nsecond"; const { order, sent, getFirstTurnQueuedInput } = await startFirstTurn(preparedInput(), queued); expect(getFirstTurnQueuedInput).toHaveBeenCalledOnce(); expect(order).toEqual(["claim", "turn.start"]); const envelope = JSON.parse(sent[0]!) as { task: { prompt: string }; completionContract: { criteria: Array<{ source?: unknown }> } }; expect(envelope.task.prompt).toBe(`${preparedInput().task.prompt}\n\n${queued}`); // Appending input keeps the prompt-bound completion sources. expect(envelope.completionContract.criteria[0]!.source).toMatchObject({ location: "task.prompt" }); }); it("starts the first turn unchanged when nothing is queued", async () => { const { sent } = await startFirstTurn(input, null); expect((JSON.parse(sent[0]!) as { task: { prompt: string } }).task.prompt).toBe(input.task.prompt); }); }); describe("remote Runner retention after a failed run", () => { const port = (): ControlPlanePort => ({ async openRun() {}, async checkpointSession() {}, async completeRun() {}, async appendEvent(event: PrpEvent) { return { cursor: event.sourceSeq, highestContiguousSourceSeq: event.sourceSeq, disposition: "committed" }; }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, }); const startCodexRun = (transport: FakeCodexTransport, options: { timeoutMs?: number; onSession?: (session: NativeSession | null) => void } = {}) => { const close = vi.spyOn(transport, "close"); const executionInput = { ...input, workspace: { ...input.workspace, cwd: WORKSPACE } }; const backend = createCodexNativeSessionBackend(executionInput, { runnerInstanceId: "runner-recovery", transportFactory: () => transport, }); const execution = executeNativeSession({ input: executionInput, backend, controlPlane: port(), runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", timeoutMs: options.timeoutMs ?? 5_000, requireSessionCloseBeforeReturn: true, onSession: options.onSession }); void execution.catch(() => undefined); return { execution, close }; }; const retainedOn = (close: { mock: { calls: unknown[][] } }): Array => close.mock.calls.map((call: unknown[]) => (call[1] as { retainUnsettledRemoteRunner?: boolean } | undefined)?.retainUnsettledRemoteRunner); it("keeps the Runner for same-run reattachment when the turn start fails", async () => { const transport = new FakeCodexTransport(); transport.rejectMethods.set("turn/start", new Error("runnerd did not report the provider turn identity")); const { execution, close } = startCodexRun(transport); await expect(execution).rejects.toThrow("runnerd did not report the provider turn identity"); expect(retainedOn(close)).not.toHaveLength(0); expect(retainedOn(close).every(retained => retained === true)).toBe(true); }); it("keeps the Runner for same-run reattachment when event consumption fails", async () => { const transport = new FakeCodexTransport(); const { execution, close } = startCodexRun(transport, { timeoutMs: 100 }); await vi.waitFor(() => expect(transport.calls.some(call => call.method === "turn/start")).toBe(true)); transport.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); await expect(execution).rejects.toThrow("native session timed out after 100ms"); expect(retainedOn(close)).not.toHaveLength(0); expect(retainedOn(close).every(retained => retained === true)).toBe(true); }); it("still stops the Runner after a Stop, even when the stopped run then fails", async () => { const transport = new FakeCodexTransport(); let published: NativeSession | null = null; const { execution, close } = startCodexRun(transport, { timeoutMs: 300, onSession: session => { published ??= session; } }); await vi.waitFor(() => expect(transport.calls.some(call => call.method === "turn/start")).toBe(true)); transport.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); await vi.waitFor(() => expect(published).not.toBeNull()); void published!.cancel!({ reason: "Stop requested", signal: new AbortController().signal }).cleanup.catch(() => undefined); await expect(execution).rejects.toThrow(); expect(retainedOn(close)).not.toHaveLength(0); expect(retainedOn(close).every(retained => retained === false)).toBe(true); }); }); describe("same-run reattachment to a Runner the failed attempt retained", () => { const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; const controlPlane = (): ControlPlanePort => ({ openRun: async () => {}, checkpointSession: async () => {}, appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: vi.fn(async () => {}), }); const attemptSession = (runIdentity: NativeRunIdentity, failure: string, close: () => Promise): NativeSession => ({ identity: () => runIdentity, capabilities: async () => capabilities, async *events() { throw new Error(failure); }, startTurn: async () => ({ turnId: "retained-turn" }), result: async () => null, async snapshot() { return { backendKind: "mock", sessionId: runIdentity.sessionId, identity: runIdentity, providerSessionId: "retained-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, close: vi.fn(close), }); const setup = (name: string) => { const runIdentity = { ...identity, companyId: `${name}-company`, runId: `${name}-run` }; const scopedInput = { ...input, binding: { ...input.binding, companyId: runIdentity.companyId, runId: runIdentity.runId } }; const block = { companyId: runIdentity.companyId, issueId: scopedInput.binding.issueId }; const reattached = attemptSession(runIdentity, "reattached attempt reached the Runner", async () => {}); const options = (session: NativeSession, extra: Partial = {}): ExecuteNativeSessionOptions => ({ input: scopedInput, backend: { descriptor: async () => ({ kind: "runner", name: `${name}-backend`, version: "1", capabilities }), openSession: vi.fn(async () => session) }, controlPlane: controlPlane(), runnerInstanceId: `${name}-runner`, controlPlaneInstanceId: "control", requireSessionCloseBeforeReturn: true, remoteCleanupScope: `${name}-sandbox`, ...extra, }); return { runIdentity, scopedInput, block, reattached, options }; }; it("admits the run's own reattachment after a close that left the remote Runner alive and unsettled", async () => { const { runIdentity, scopedInput, block, reattached, options } = setup("retained-unsettled"); const retained = new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, runnerRetained: true }); const failed = attemptSession(runIdentity, "runnerd did not report the provider turn identity", async () => { throw retained; }); await expect(executeNativeSession(options(failed))).rejects.toMatchObject({ message: "runnerd did not report the provider turn identity", cleanupError: retained }); expect(nativeSessionCleanupBlock(block)).toBe("operator_required"); const otherRun = options(reattached, { retainedRunnerReattachment: true, input: { ...scopedInput, binding: { ...scopedInput.binding, runId: "another-run" } } }); await expect(executeNativeSession(otherRun)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); const reattachment = options(reattached, { retainedRunnerReattachment: true }); await expect(executeNativeSession(reattachment)).rejects.toThrow("reattached attempt reached the Runner"); expect(reattachment.backend.openSession).toHaveBeenCalledOnce(); expect(failed.close).toHaveBeenCalledOnce(); expect(nativeSessionCleanupBlock(block)).toBeNull(); }); it("keeps refusing a run that is not reattaching to its own retained remote Runner", async () => { const remote = setup("retained-without-takeover"); const retained = new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, runnerRetained: true }); const failed = attemptSession(remote.runIdentity, "transport interrupted", async () => { throw retained; }); await expect(executeNativeSession(remote.options(failed))).rejects.toThrow("transport interrupted"); const fresh = remote.options(remote.reattached); await expect(executeNativeSession(fresh)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(fresh.backend.openSession).not.toHaveBeenCalled(); expect(nativeSessionCleanupBlock(remote.block)).toBe("operator_required"); completeTerminatedRemoteNativeSessionCleanup({ companyId: remote.runIdentity.companyId, runId: remote.runIdentity.runId, remoteCleanupScope: "retained-without-takeover-sandbox" }); const local = setup("retained-local"); const localFailed = attemptSession(local.runIdentity, "local transport interrupted", async () => { throw retained; }); await expect(executeNativeSession(local.options(localFailed, { remoteCleanupScope: undefined }))) .rejects.toThrow("local transport interrupted"); const localReattachment = local.options(local.reattached, { remoteCleanupScope: undefined, retainedRunnerReattachment: true }); await expect(executeNativeSession(localReattachment)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); expect(localReattachment.backend.openSession).not.toHaveBeenCalled(); completeTerminatedLocalNativeSessionCleanup({ companyId: local.runIdentity.companyId, runId: local.runIdentity.runId, runnerInstanceId: "retained-local-runner" }); }); it("admits the run's own reattachment instead of waiting out close retries that replay one failed close", async () => { vi.useFakeTimers(); try { const { runIdentity, block, reattached, options } = setup("retained-replayed-close"); const drainTimedOut = new Error("Controller journal drain timed out; preserve the session before replacing its writer."); const failed = attemptSession(runIdentity, "runnerd did not report the provider turn identity", async () => { throw drainTimedOut; }); const first = expect(executeNativeSession(options(failed))).rejects.toThrow("runnerd did not report the provider turn identity"); await vi.advanceTimersByTimeAsync(10_000); await first; expect(nativeSessionCleanupBlock(block)).toBe("pending"); const closesBeforeReattachment = vi.mocked(failed.close).mock.calls.length; const reattachment = options(reattached, { retainedRunnerReattachment: true }); const admitted = expect(executeNativeSession(reattachment)).rejects.toThrow("reattached attempt reached the Runner"); await vi.advanceTimersByTimeAsync(2_000); await admitted; expect(reattachment.backend.openSession).toHaveBeenCalledOnce(); expect(nativeSessionCleanupBlock(block)).toBeNull(); await vi.advanceTimersByTimeAsync(600_000); expect(vi.mocked(failed.close).mock.calls.length).toBe(closesBeforeReattachment); } finally { vi.useRealTimers(); } }); it("lets a close that a deploy interrupted settle before the reattachment takes the Runner over", async () => { vi.useFakeTimers(); try { const { runIdentity, block, reattached, options } = setup("retained-deploy-mid-cleanup"); const interrupted = new Error("runner transport closed by deploy"); let rejectClose!: (error: Error) => void; const inFlightClose = new Promise((_resolve, reject) => { rejectClose = reject; }); void inFlightClose.catch(() => undefined); const failed = attemptSession(runIdentity, "server restarting for deploy", () => inFlightClose); const first = executeNativeSession(options(failed, { requireSessionCloseBeforeReturn: false })); void first.catch(() => undefined); await vi.waitFor(() => expect(failed.close).toHaveBeenCalledOnce()); expect(nativeSessionCleanupBlock(block)).toBe("pending"); const reattachment = options(reattached, { retainedRunnerReattachment: true }); let settled = false; const admitted = executeNativeSession(reattachment).catch((error: unknown) => { settled = true; return error; }); await vi.advanceTimersByTimeAsync(1_000); expect(settled).toBe(false); expect(reattachment.backend.openSession).not.toHaveBeenCalled(); rejectClose(interrupted); await vi.advanceTimersByTimeAsync(10_000); await expect(admitted).resolves.toMatchObject({ message: "reattached attempt reached the Runner" }); expect(reattachment.backend.openSession).toHaveBeenCalledOnce(); expect(nativeSessionCleanupBlock(block)).toBeNull(); const closesAfterTakeover = vi.mocked(failed.close).mock.calls.length; await vi.advanceTimersByTimeAsync(600_000); expect(vi.mocked(failed.close).mock.calls.length).toBe(closesAfterTakeover); } finally { vi.useRealTimers(); } }); });