diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs index a3232e33050..8bd51438e54 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-acpx-sidecar.rs @@ -10,6 +10,8 @@ use serde_json::{json, Value}; const PROJECTED_INPUT_PROVIDER_ID: &str = "input / réquest"; static BACKGROUND_TASKS: std::sync::OnceLock> = std::sync::OnceLock::new(); +static PROVIDER_TURNS_STARTED: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); fn background_tasks_mode(snapshot: bool) -> Option { let path = BACKGROUND_TASKS.get()?.as_ref()?; @@ -126,6 +128,7 @@ fn run() -> Result<(), Box> { let mut pending_provider_run: Option = None; let mut provider_rearmable = false; let mut source_pending_completion: Option = None; + let mut open_provider_turn: Option = None; let mut source_goal_revision = 0_u64; let mut runtime_directory: Option = None; // `--cancel-reply lost|late`: the provider stops its prompt, but the reply @@ -274,6 +277,7 @@ fn run() -> Result<(), Box> { next_sequence, &run, &mut source_pending_completion, + &mut open_provider_turn, )?; } } @@ -300,15 +304,31 @@ fn run() -> Result<(), Box> { )?; next_sequence += 1; } - if armed && tokens.iter().any(|token| token == "provider-turn-hold") { + if armed + && tokens.iter().any(|token| { + token == "provider-turn-hold" || token == "provider-turn-after-snapshot" + }) + { continue; } - if let (true, Some(run)) = (armed, pending_provider_run.take()) { + if let Some(delay) = tokens + .iter() + .find_map(|token| token.strip_prefix("provider-turn-delay-ms=")) + .and_then(|value| value.parse::().ok()) + .filter(|_| armed) + { + std::thread::sleep(Duration::from_millis(delay)); + } + let run = pending_provider_run.take().or_else(|| { + (provider_rearmable && chained_provider_turn_remains()).then(|| source_run.clone()) + }); + if let (true, Some(run)) = (armed, run) { next_sequence = write_provider_turn( &mut stdout, next_sequence, &run, &mut source_pending_completion, + &mut open_provider_turn, )?; provider_rearmable = tokens.iter().any(|token| token == "provider-terminal-live"); } else { @@ -338,7 +358,7 @@ fn run() -> Result<(), Box> { ); if command == "session.snapshot" { response["result"] = json!({"identity":session_identity, "runId":source_run, - "turnId":if fault == "active" {json!("unexpected-turn")} else {json!(source_pending_completion)}, + "turnId":if fault == "active" {json!("unexpected-turn")} else {json!(source_pending_completion.as_ref().or(open_provider_turn.as_ref()))}, "pendingToolCount":if fault == "tool" || source_pending_completion.is_some() {1} else {0}, "pendingInputCount":0, "pendingRuntimeRequests":[]}); } @@ -461,6 +481,24 @@ fn run() -> Result<(), Box> { "error":{"code":"session_open_failed","message":"The replacement provider could not open.","retryable":false}}); } write_json(&mut stdout, &response)?; + if command == "session.snapshot" && open_provider_turn.take().is_some() { + next_sequence = finish_provider_turn(&mut stdout, next_sequence, &source_run)?; + } + if command == "session.snapshot" + && provider_turn_tokens() + .iter() + .any(|token| token == "provider-turn-after-snapshot") + { + if let Some(run) = pending_provider_run.take() { + next_sequence = write_provider_turn( + &mut stdout, + next_sequence, + &run, + &mut source_pending_completion, + &mut open_provider_turn, + )?; + } + } if command == "session.open" && args.iter().any(|arg| arg == "--emit-process") { write_json( &mut stdout, @@ -1583,12 +1621,43 @@ fn bootstrap_success( } fn provider_turn_tokens() -> Vec { - BACKGROUND_TASKS + let mut tokens: Vec = BACKGROUND_TASKS .get() .and_then(Option::as_ref) .and_then(|path| std::fs::read_to_string(path).ok()) .map(|modes| modes.split_whitespace().map(str::to_owned).collect()) - .unwrap_or_default() + .unwrap_or_default(); + let index = PROVIDER_TURNS_STARTED + .load(std::sync::atomic::Ordering::SeqCst) + .max(1); + let spec = provider_turn_chain(&tokens).and_then(|chain| chain.into_iter().nth(index - 1)); + if let Some(spec) = spec { + tokens.push(format!("provider-turn-id=provider-{index}")); + tokens.extend( + spec.split('+') + .filter_map(|flag| match flag { + "live" => Some("provider-terminal-live"), + "result" => Some("provider-result"), + "resolved" => Some("provider-result-resolved"), + _ => None, + }) + .map(str::to_owned), + ); + } + tokens +} + +fn provider_turn_chain(tokens: &[String]) -> Option> { + tokens + .iter() + .find_map(|token| token.strip_prefix("provider-turns=")) + .map(|chain| chain.split('/').map(str::to_owned).collect()) +} + +fn chained_provider_turn_remains() -> bool { + provider_turn_chain(&provider_turn_tokens()).is_some_and(|chain| { + PROVIDER_TURNS_STARTED.load(std::sync::atomic::Ordering::SeqCst) < chain.len() + }) } fn offer_provider_turn( @@ -1603,7 +1672,7 @@ fn offer_provider_turn( return Ok(sequence); } if tokens.iter().any(|token| token == "provider-turn-unarmed") { - return write_provider_turn(output, sequence, default_run, pending_finish); + return write_provider_turn(output, sequence, default_run, pending_finish, &mut None); } *pending = Some(default_run.to_owned()); Ok(sequence) @@ -1614,7 +1683,11 @@ fn write_provider_turn( mut sequence: u64, default_run: &str, pending_finish: &mut Option, + open_turn: &mut Option, ) -> io::Result { + if provider_turn_chain(&provider_turn_tokens()).is_some() { + PROVIDER_TURNS_STARTED.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } let tokens = provider_turn_tokens(); if !tokens.iter().any(|token| token == "provider-turn") { return Ok(sequence); @@ -1689,6 +1762,37 @@ fn write_provider_turn( *pending_finish = Some(turn_id); return Ok(sequence + 1); } + if tokens + .iter() + .any(|token| token == "provider-turn-until-snapshot") + { + write_json( + output, + &frame( + sequence, + "runtime.event", + json!({"type":"text_delta","text":"Claude is still working."}), + ), + )?; + *open_turn = Some(turn_id); + return Ok(sequence + 1); + } + if let Some(gap) = tokens + .iter() + .find_map(|token| token.strip_prefix("provider-turn-gap-ms=")) + .and_then(|value| value.parse::().ok()) + { + write_json( + output, + &frame( + sequence, + "runtime.event", + json!({"type":"text_delta","text":"Claude is still working."}), + ), + )?; + sequence += 1; + std::thread::sleep(Duration::from_millis(gap)); + } finish_provider_turn(output, sequence, default_run) } @@ -1737,14 +1841,19 @@ fn finish_provider_turn( } else { 0 }; - write_json( - output, - &frame( - sequence, - "runtime.turn_terminal", - json!({"status":"completed","stopReason":"end_turn","backgroundTasks":{"live":live}}), - ), - )?; + let mut terminal = + json!({"status":"completed","stopReason":"end_turn","backgroundTasks":{"live":live}}); + if tokens + .iter() + .any(|token| token == "provider-goal-at-terminal") + { + terminal["goalAtTerminal"] = json!({"schema":"paperclip.session_goal.snapshot.v1", + "providerRevision":1_000_000,"goal":null,"workingNow":false, + "sessionGoals":{"availability":"unsupported","actions":[],"autonomousUpdates":false, + "persistentAcrossResume":false,"maxObjectiveChars":4000,"tokenBudgetControl":false, + "usageReporting":false}}); + } + write_json(output, &frame(sequence, "runtime.turn_terminal", terminal))?; Ok(sequence + 1) } diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index ac6562d55d6..c1c968ba15e 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -421,6 +421,32 @@ describe("HarnessDriverBackend", () => { }); }); + it("builds a native snapshot from the driver's boundary snapshot only when asked for that sequence", async () => { + const boundarySnapshot = vi.fn((sourceSeq: number): PersistedHarnessSession | null => sourceSeq !== 7 ? null : { + driverKind: "fake", driverSessionId: "driver-1", providerSessionId: "provider-1", providerIdentity, + activeTurnId: null, semanticResult: null, awaitingProviderTurn: true, lastSourceSequence: 7 }); + class BoundarySession extends FakeHarnessSession { + boundarySnapshot = boundarySnapshot; + override async snapshot(): Promise { + return { ...(await super.snapshot()), activeTurnId: "provider-1", lastSourceSequence: 9 }; + } + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new BoundarySession(); } }); + const session = await backend.openSession({ identity: { + runId: "run-boundary", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1", + } }); + const signal = new AbortController().signal; + await expect(session.snapshot({ signal, sourceSeq: 7 })).resolves.toMatchObject({ + cursor: "7", activeTurnId: null, semanticResult: null, awaitingProviderTurn: true }); + expect(boundarySnapshot).toHaveBeenCalledExactlyOnceWith(7); + const current = await session.snapshot(); + expect(current).toMatchObject({ cursor: "9", activeTurnId: "provider-1" }); + expect(current).not.toHaveProperty("awaitingProviderTurn"); + expect(boundarySnapshot).toHaveBeenCalledOnce(); + await expect(session.snapshot({ signal, sourceSeq: 8 })).resolves.toMatchObject({ cursor: "9", activeTurnId: "provider-1" }); + expect(boundarySnapshot).toHaveBeenLastCalledWith(8); + }); + it("preserves a consumed disposition allowance in native snapshots", async () => { class ConsumedRecoverySession extends FakeHarnessSession { override async snapshot(): Promise { @@ -703,6 +729,36 @@ describe("HarnessDriverBackend", () => { }); }); + async function terminalsAfterEachTurnEnd(turnEnds: Array>) { + class TurnEndsSession extends FakeHarnessSession { + override async *events() { + for (const [index, payload] of turnEnds.entries()) yield prpEvent(index + 1, "turn.completed", payload); + } + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new TurnEndsSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }); + const terminals: Array = []; + for await (const _event of session.events()) terminals.push((await session.snapshot()).terminal ?? null); + return terminals; + } + + it("leaves the run terminal open when a turn ends waiting for the provider's own next turn", async () => { + await expect(terminalsAfterEachTurnEnd([{ status: "completed", awaitingProviderTurn: true }])).resolves.toEqual([null]); + }); + + it("sets the run terminal when a turn ends without waiting for the provider", async () => { + const [terminal] = await terminalsAfterEachTurnEnd([{ status: "completed" }]); + expect(terminal).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + }); + + it("sets the run terminal at the first turn end that does not wait for the provider", async () => { + const [waiting, settled] = await terminalsAfterEachTurnEnd([{ status: "completed", awaitingProviderTurn: true }, { status: "completed" }]); + expect(waiting).toBeNull(); + expect(settled).toMatchObject({ turnTerminalState: "completed", runTerminalState: "succeeded" }); + }); + it("reconstructs a missing top-level terminal from a completed semantic turn", async () => { const recoveryDriver: HarnessDriver = { ...driver, diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index b45e0b9bf50..899106d04e5 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -10,6 +10,7 @@ import type { NativeSessionBackend, NativeSessionBackendDescriptor, NativeSessionRecoveryOptions, + NativeSessionSnapshotOptions, OpenNativeSessionInput, PersistedNativeSession, } from "../contracts/native-session-backend.js"; @@ -420,10 +421,11 @@ class HarnessNativeSession implements NativeSession { throw this.#protocolIntegrityFailure; } - async #harnessSnapshot(): Promise { + async #harnessSnapshot(sourceSeq?: number): Promise { this.#assertProtocolIntegrity(); try { - const snapshot = await this.#session.snapshot(); + const boundary = sourceSeq === undefined ? null : this.#session.boundarySnapshot?.(sourceSeq) ?? null; + const snapshot = boundary ?? await this.#session.snapshot(); this.#assertProtocolIntegrity(); return snapshot; } catch (error) { @@ -633,7 +635,7 @@ class HarnessNativeSession implements NativeSession { const disposition = snapshot.semanticResult?.result.reportedWorkDisposition ?? "yielded"; - this.#terminal = { + if (event.payload.awaitingProviderTurn !== true) this.#terminal = { schema: "paperclip.prp.terminal.v1", turnTerminalState: event.eventType === "turn.completed" @@ -874,9 +876,9 @@ class HarnessNativeSession implements NativeSession { }; } - async snapshot(): Promise { + async snapshot(options?: NativeSessionSnapshotOptions): Promise { this.#assertProtocolIntegrity(); - const snapshot = await this.#harnessSnapshot(); + const snapshot = await this.#harnessSnapshot(options?.sourceSeq); return { backendKind: "runner", driverKind: snapshot.driverKind, diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index 37091f4042d..494fc7da83f 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -669,6 +669,7 @@ export interface HarnessSession { usage?(): Promise | null>; transcript?(): Promise; snapshot(): Promise; + boundarySnapshot?(sourceSeq: number): PersistedHarnessSession | null; /** Return the confirmed released source, or false when active use was not released. */ release?(): Promise; /** Relinquish controller authority without semantically closing the session. */ diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index e8ce03c19dc..2973f95d232 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -152,6 +152,7 @@ export interface NativeSessionCancellation { export interface NativeSessionSnapshotOptions { /** Stop provider snapshot work that outlives the execution deadline. */ signal: AbortSignal; + sourceSeq?: number; } /** A dispatched operation may have taken effect, but has no proven result. */ diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index cd70ff1c0b2..b705e8ee579 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -25,6 +25,7 @@ import { describe, expect, it, vi } from "vitest"; import { configuredEnvironmentProjection } from "../configured-environment.js"; import { NativeSessionProtocolIntegrityError } from "../contracts/native-session-backend.js"; import { createCapabilityRunnerdCodexTransport, createCapabilityRunnerdProviderEnvironment, waitForRunnerCommand } from "../live/runnerd-codex-transport.js"; +import { providerTurnStartBefore } from "../live/provider-turn-tool-order.js"; import { ACPX_CREDENTIAL_BINDING_ENV, createAcpxSidecarHostEnvironment } from "../drivers/acpx/environment.js"; import { validatePrpEvent } from "../protocol/replay-contract.js"; import { digestPaperclipSemanticContent } from "../semantic-tools/receipts.js"; @@ -637,6 +638,89 @@ it("excludes streamed output from checkpoints even when the live window is full" } }); +function turnEvent(sourceSeq: number, eventType: string, payload: Record, priority = 0): Record { + const envelope = semanticInputEvent(sourceSeq); + Object.assign(envelope.payload as Record, { sourceEventId: `turn-event-${sourceSeq}`, eventType, priority, payload }); + return envelope; +} + +function storedTurnEvent(sourceSeq: number, eventType: string, payload: Record, priority: 0 | 1 | 2 = 0) { + return { sourceSeq, sourceEventId: `turn-event-${sourceSeq}`, eventType, priority, + envelope: turnEvent(sourceSeq, eventType, payload, priority), deliveryCount: 1, logicalEffectCount: 1 as const }; +} + +const providerStart = (turnId: string) => ({ provider: "acpx", providerTurnId: turnId, status: "inProgress", + turn: { id: turnId, status: "inProgress" }, origin: "provider" }); +const failedTurn = (turnId: string) => ({ provider: "acpx", providerTurnId: turnId, status: "failed" }); + +async function commitThroughFullWindow(prefix: ReturnType[], sent: Array<[string, Record]>) { + const root = mkdtempSync(resolve(tmpdir(), "runner-provider-turn-window-")); + const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); + core.store.state.committedEvents = [...prefix, ...Array.from({ length: 4096 - prefix.length }, (_, index) => + storedTurnEvent(prefix.length + index + 1, "item.delta", { text: "streaming output" }, 2))]; + core.store.state.ackedSourceSeq = 4096; + (core.store as WritableCoreStore).save(); + try { + await core.start(); + const client = (await authenticate(core, core.issueBootstrapTicket()))!; + for (const [index, [eventType, payload]] of sent.entries()) { + sendSecure(client, turnEvent(4097 + index, eventType, payload)); + expect(await receiveSecure(client)).toMatchObject({ kind: "ack", payload: { ackedSourceSeq: 4097 + index } }); + } + client.socket.destroy(); + return core.store.state.committedEvents.map(event => ({ ...event })); + } finally { + await core.stop(); + await core.drainPendingConnectionProcessing(); + rmSync(root, { recursive: true, force: true }); + } +} + +it("finds a provider turn's start across ordinary events the full window removed", async () => { + const window = await commitThroughFullWindow([storedTurnEvent(1, "turn.started", providerStart("provider-a"))], + [["item.completed", { provider: "acpx", itemId: "message-a", kind: "agentMessage", status: "completed", text: "Done." }]]); + expect(window.map(event => event.sourceSeq).slice(0, 3)).toEqual([1, 3, 4]); + expect(providerTurnStartBefore(window, 4097)?.turnId).toBe("provider-a"); +}); + +it("finds no provider turn start once the window removed that turn's terminal for a later startless one", async () => { + const window = await commitThroughFullWindow( + [storedTurnEvent(1, "turn.started", providerStart("provider-a")), storedTurnEvent(2, "turn.failed", failedTurn("provider-a"))], + [["harness.diagnostic", {}], ["turn.failed", failedTurn("controller-b")], ["harness.diagnostic", {}]]); + expect(window.map(event => event.sourceSeq).slice(0, 2)).toEqual([1, 5]); + expect(window.slice(-3).map(event => event.eventType)).toEqual(["harness.diagnostic", "turn.failed", "harness.diagnostic"]); + for (const sourceSeq of [4097, 4098, 4099]) expect(providerTurnStartBefore(window, sourceSeq)).toBeNull(); +}); + +it("finds a provider turn's start in a restored checkpoint that dropped the turn's streamed deltas", async () => { + const root = mkdtempSync(resolve(tmpdir(), "runner-provider-turn-checkpoint-")); + const options = { stateDirectory: resolve(root, "control-plane"), identity, expectedRunnerVersion, expectedRunnerDigest }; + const core = new DurablePrpControlPlane(options); + core.store.state.committedEvents = [storedTurnEvent(1, "turn.started", providerStart("provider-a")), + storedTurnEvent(2, "item.delta", { text: "streaming output" }, 2), storedTurnEvent(3, "item.delta", { text: "streaming output" }, 2), + storedTurnEvent(4, "item.completed", { provider: "acpx", itemId: "message-a", kind: "agentMessage", status: "completed", text: "Done." }, 1)]; + core.store.state.ackedSourceSeq = 4; + (core.store as WritableCoreStore).save(); + const snapshots: DurablePrpCheckpoint[] = []; + core.setCheckpointPersistence(async saved => { snapshots.push(saved); }); + const restoredRoot = mkdtempSync(resolve(tmpdir(), "runner-provider-turn-restored-")); + let restored: DurablePrpControlPlane | undefined; + try { + core.queueCommand("session.snapshot", {}, "provider-turn-checkpoint"); + await core.flushCheckpoint(); + decodeDurablePrpCheckpoint(snapshots.at(-1)!.records).restoreSessionDirectory(restoredRoot); + restored = new DurablePrpControlPlane({ ...options, stateDirectory: resolve(restoredRoot, "control-plane") }); + const window = restored.store.state.committedEvents; + expect(window.map(event => event.sourceSeq)).toEqual([1, 4]); + expect(providerTurnStartBefore(window, 4)?.turnId).toBe("provider-a"); + } finally { + await restored?.stop(); + await core.stop(); + rmSync(root, { recursive: true, force: true }); + rmSync(restoredRoot, { recursive: true, force: true }); + } +}); + it.each(["unsupported", "invalid cursor", "invalid proof"])("does not advance recovery from an unauthenticated cursor: %s", async fault => { const root = mkdtempSync(resolve(tmpdir(), "runner-recovery-cursor-auth-")); const core = new DurablePrpControlPlane({ stateDirectory: root, identity, expectedRunnerVersion, expectedRunnerDigest }); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 83feb7983a0..2f763b1d92e 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -604,6 +604,8 @@ export class CodexAppServerDriver implements HarnessDriver { } recoveredActiveTurnId = active.length === 1 ? text(active[0]?.id) : recoveredActiveTurnId; } + const awaitingProviderTurn = snapshot.awaitingProviderTurn === true && snapshot.semanticResult == null + && recoveredActiveTurnId === null && (snapshot.terminalTurns?.length ?? 0) > 0; if ( !recoveringAutonomousGoal && !this.#direct() && @@ -639,7 +641,7 @@ export class CodexAppServerDriver implements HarnessDriver { ); } const laterTurns = - lastKnownTerminalIndex < 0 + lastKnownTerminalIndex < 0 || awaitingProviderTurn ? [] : turns.slice(lastKnownTerminalIndex + 1); if (laterTurns.length > 1) { @@ -716,6 +718,7 @@ export class CodexAppServerDriver implements HarnessDriver { let lostAcceptedTurnId: string | null = null; if ( !recoveringAutonomousGoal && + !awaitingProviderTurn && recoveredActiveTurnId === null && !reconcileUncheckpointedDispositionTurn && snapshot.semanticResult == null diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts index ad30df81663..4c6660624ef 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts @@ -11,6 +11,7 @@ import { result, } from "./codex-app-server-driver.test-support.js"; import { RUNNERD_RUN_TERMINAL } from "./codex-driver-values.js"; +import type { HarnessSessionRecoveryResult } from "../../contracts/harness-driver.js"; async function completedFirstTurn(awaiting: boolean) { const transport = new FakeCodexTransport(); @@ -230,3 +231,350 @@ describe("Codex app-server driver with provider-started turns", () => { await session.close({ reason: "fixture complete" }); }); }); + +type StreamEvent = { eventType: string; payload: Record; turnId?: string }; + +async function restartedDuringWait(history: Array>, awaiting = true) { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Start a background install." } }); + first.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); + first.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, + ...(awaiting ? { awaitingProviderTurn: true } : {}) }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ reason: "controller restarted" }); + second.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [], ...(awaiting ? { awaitingProviderTurn: true } : {}) }, ...history] } }; + return { recovery: await driver.recoverSession!(checkpoint), transport: second }; +} + +async function restartedDuringActiveTurn(turns: Array>, beforeRecovery?: (transport: FakeCodexTransport) => void) { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Start a background install." } }); + const checkpoint = await original.snapshot(); + expect(checkpoint.activeTurnId).toBe("turn-1"); + await original.close({ reason: "controller restarted" }); + second.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, turns } }; + beforeRecovery?.(second); + return { recovery: await driver.recoverSession!(checkpoint).catch((error: unknown): HarnessSessionRecoveryResult => ({ recovered: false, reason: String(error) })), transport: second }; +} + +const agentMessage = (turnId: string) => ({ id: `msg-${turnId}`, type: "agentMessage", text: `work in ${turnId}` }); + +function releaseProviderTurn(transport: FakeCodexTransport, turnId: string, options: { complete: boolean; awaiting?: boolean; withResult?: boolean }) { + transport.push("turn/started", { threadId: "thread-1", turn: { id: turnId, status: "inProgress" }, origin: "provider" }); + transport.push("item/completed", { threadId: "thread-1", turnId, item: agentMessage(turnId) }); + transport.push("thread/tokenUsage/updated", { threadId: "thread-1", turnId, + tokenUsage: { total: { inputTokens: turnId === "provider-1" ? 100 : 200, outputTokens: 10 } } }); + if (options.withResult) transport.push("paperclip/runResult", { threadId: "thread-1", turnId, result }); + if (options.complete) { + transport.push("turn/completed", { threadId: "thread-1", turn: { id: turnId, status: "completed", items: [] }, + ...(options.awaiting ? { awaitingProviderTurn: true } : {}) }); + } +} + +async function streamUntil(events: AsyncIterator, done: (event: StreamEvent) => boolean) { + const collected: StreamEvent[] = []; + for (;;) { + const next = await events.next(); + if (next.done) return collected; + collected.push(next.value); + if (next.value.eventType === "session.failed" || done(next.value)) return collected; + } +} + +const startedTurns = (events: StreamEvent[]) => events.filter(event => event.eventType === "turn.started").map(event => ({ turnId: event.turnId, origin: event.payload.origin })); +const itemTurns = (events: StreamEvent[]) => events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage").map(event => event.turnId); +const usageTurns = (events: StreamEvent[]) => events.filter(event => event.payload.kind === "usage").map(event => event.turnId); +const completedTurn = (turnId: string) => (event: StreamEvent) => event.eventType === "turn.completed" && event.turnId === turnId; +const startedTurn = (turnId: string) => (event: StreamEvent) => event.eventType === "turn.started" && event.turnId === turnId; + +describe("Codex app-server driver after a controller restart during a wait for the provider's turn", () => { + it("lets the stream deliver one later provider-started turn that is still in progress", async () => { + const { recovery, transport } = await restartedDuringWait([{ id: "provider-1", status: "inProgress", items: [agentMessage("provider-1")] }]); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + expect(await session.snapshot()).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true, dispositionOnlyRecoveryConsumed: false }); + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + releaseProviderTurn(transport, "provider-1", { complete: true, withResult: true }); + const seen = await streamUntil(events, completedTurn("provider-1")); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }]); + expect(itemTurns(seen)).toEqual(["provider-1"]); + expect(usageTurns(seen)).toEqual(["provider-1"]); + const snapshot = await session.snapshot(); + expect(snapshot.semanticResult).toMatchObject({ result, turnId: "provider-1" }); + expect(snapshot).toMatchObject({ dispositionOnlyRecoveryConsumed: false, dispositionOnlyRecoveryTurnId: null }); + await session.close({ reason: "fixture complete" }); + }); + + it("lets the stream deliver a finished provider-started turn and the next one in progress", async () => { + const { recovery, transport } = await restartedDuringWait([ + { id: "provider-1", status: "completed", items: [agentMessage("provider-1")], awaitingProviderTurn: true }, + { id: "provider-2", status: "inProgress", items: [agentMessage("provider-2")] }, + ]); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + releaseProviderTurn(transport, "provider-1", { complete: true, awaiting: true }); + releaseProviderTurn(transport, "provider-2", { complete: false }); + const seen = await streamUntil(events, event => event.payload.kind === "usage" && event.turnId === "provider-2"); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }, { turnId: "provider-2", origin: "provider" }]); + expect(itemTurns(seen)).toEqual(["provider-1", "provider-2"]); + expect(usageTurns(seen)).toEqual(["provider-1", "provider-2"]); + expect(seen.find(completedTurn("provider-1"))).toMatchObject({ payload: { awaitingProviderTurn: true } }); + expect(await session.snapshot()).toMatchObject({ activeTurnId: "provider-2", dispositionOnlyRecoveryConsumed: false }); + await session.close({ reason: "fixture complete" }); + }); + + it("lets the stream deliver two finished provider-started turns and settles the last one's result", async () => { + const { recovery, transport } = await restartedDuringWait([ + { id: "provider-1", status: "completed", items: [agentMessage("provider-1")], awaitingProviderTurn: true }, + { id: "provider-2", status: "completed", items: [agentMessage("provider-2")] }, + ]); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + releaseProviderTurn(transport, "provider-1", { complete: true, awaiting: true }); + releaseProviderTurn(transport, "provider-2", { complete: true, withResult: true }); + const seen = await streamUntil(events, completedTurn("provider-2")); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }, { turnId: "provider-2", origin: "provider" }]); + expect(itemTurns(seen)).toEqual(["provider-1", "provider-2"]); + expect(usageTurns(seen)).toEqual(["provider-1", "provider-2"]); + expect(seen.find(completedTurn("provider-1"))).toMatchObject({ payload: { awaitingProviderTurn: true } }); + expect(seen.find(completedTurn("provider-2"))!.payload).not.toHaveProperty("awaitingProviderTurn"); + const snapshot = await session.snapshot(); + expect(snapshot.semanticResult).toMatchObject({ result, turnId: "provider-2" }); + expect(snapshot).toMatchObject({ dispositionOnlyRecoveryConsumed: false }); + await session.close({ reason: "fixture complete" }); + }); + + it("still refuses two later turns after a checkpoint that was not waiting", async () => { + const { recovery } = await restartedDuringWait([ + { id: "turn-2", status: "completed", items: [] }, + { id: "turn-3", status: "inProgress", items: [] }, + ], false); + expect(recovery).toEqual({ recovered: false, reason: "provider exposed multiple uncheckpointed disposition recovery turns" }); + }); + + it("settles a persisted active turn the read reports awaited and lets the stream deliver the provider's turn after it", async () => { + const { recovery, transport } = await restartedDuringActiveTurn([ + { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }, + { id: "provider-1", status: "inProgress", items: [] }, + ]); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + const reconciled = await streamUntil(events, completedTurn("turn-1")); + expect(reconciled.at(-1)).toMatchObject({ eventType: "turn.completed", turnId: "turn-1", payload: { awaitingProviderTurn: true } }); + expect(await session.snapshot()).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true }); + releaseProviderTurn(transport, "provider-1", { complete: true, withResult: true }); + const adopted = await streamUntil(events, completedTurn("provider-1")); + expect(adopted.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(adopted.find(startedTurn("provider-1"))).toMatchObject({ payload: { origin: "provider" } }); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); + + it("reconciles the persisted turn before admitting a provider start queued before the recovery", async () => { + const closeReasons: unknown[] = []; + const { recovery } = await restartedDuringActiveTurn([ + { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }, + { id: "provider-1", status: "inProgress", items: [] }, + ], transport => { + const close = transport.close.bind(transport); + transport.close = async (reason?: string) => { closeReasons.push(reason); await close(); }; + releaseProviderTurn(transport, "provider-1", { complete: true, withResult: true }); + }); + expect(closeReasons).toEqual([]); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + const seen = await streamUntil(events, completedTurn("provider-1")); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(seen.find(completedTurn("turn-1"))).toMatchObject({ payload: { awaitingProviderTurn: true } }); + expect(seen.findIndex(completedTurn("turn-1"))).toBeLessThan(seen.findIndex(startedTurn("provider-1"))); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }]); + expect(itemTurns(seen)).toEqual(["provider-1"]); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); + + it("accepts the provider's turn in progress when the stream already settled the persisted turn into a wait before the reconciliation judged it", async () => { + const { recovery, transport } = await restartedDuringActiveTurn([ + { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }, + { id: "provider-1", status: "inProgress", items: [] }, + ], second => second.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, awaitingProviderTurn: true })); + expect(recovery.reason).toBeUndefined(); + const session = recovery.session!; + expect(await session.snapshot()).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true }); + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + releaseProviderTurn(transport, "provider-1", { complete: true, withResult: true }); + const seen = await streamUntil(events, completedTurn("provider-1")); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(seen.filter(completedTurn("turn-1"))).toHaveLength(1); + expect(seen.find(completedTurn("turn-1"))).toMatchObject({ payload: { awaitingProviderTurn: true } }); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }]); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); + + it("admits at once a provider start that arrives after the stream settled the persisted turn into a wait, while the reconciliation is still reading", async () => { + const { recovery, transport } = await restartedDuringActiveTurn([ + { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }, + { id: "provider-1", status: "inProgress", items: [] }, + ], second => { + second.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, awaitingProviderTurn: true }); + second.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + }); + expect(recovery.reason).toBeUndefined(); + const session = recovery.session!; + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + const seen = await streamUntil(events, event => event.eventType === "session.reconciled"); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }]); + expect(seen.at(-1)).toMatchObject({ eventType: "session.reconciled", payload: { activeTurnId: "provider-1" } }); + transport.push("paperclip/runResult", { threadId: "thread-1", turnId: "provider-1", result }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "provider-1", status: "completed", items: [] } }); + await streamUntil(events, completedTurn("provider-1")); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); + + it.each([ + ["two turns are in progress", [ + { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }, + { id: "provider-1", status: "inProgress", items: [] }, { id: "provider-2", status: "inProgress", items: [] }]], + ["the turn in progress is listed before the settled turn", [ + { id: "provider-1", status: "inProgress", items: [] }, { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }]], + ])("still refuses another active turn after the stream settled the persisted turn into a wait when %s", async (_case, turns) => { + const { recovery } = await restartedDuringActiveTurn(turns, + second => second.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, awaitingProviderTurn: true })); + expect(recovery).toMatchObject({ recovered: false, reason: expect.stringContaining("thread/read exposed active turn provider-1 instead of persisted active turn none") }); + }); + + it.each([ + ["the persisted turn finished without the wait marker", [ + { id: "turn-1", status: "completed", items: [] }, { id: "provider-1", status: "inProgress", items: [] }]], + ["the persisted turn is still in progress", [ + { id: "turn-1", status: "inProgress", items: [] }, { id: "provider-1", status: "inProgress", items: [] }]], + ["the other active turn is listed before the persisted turn", [ + { id: "provider-1", status: "inProgress", items: [] }, { id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }]], + ["the reconciled persisted turn settles a result", [ + { id: "turn-1", status: "completed", awaitingProviderTurn: true, + items: [{ id: "result-1", type: "agentMessage", text: JSON.stringify(result) }] }, + { id: "provider-1", status: "inProgress", items: [] }]], + ])("still refuses another active turn when %s", async (_case, turns) => { + const { recovery } = await restartedDuringActiveTurn(turns); + expect(recovery).toMatchObject({ recovered: false, reason: expect.stringContaining("thread/read exposed active turn provider-1 instead of persisted active turn turn-1") }); + }); +}); + +describe("Codex app-server driver after a restart during a wait whose disposition marker is consumed", () => { + it("keeps reading notifications, admits the provider's own turn and still refuses a controller turn", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Start a background install." } }); + first.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); + first.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, awaitingProviderTurn: true }); + await collectUntilTerminal(original.events()); + const checkpoint = { ...await original.snapshot(), dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-1" }; + expect(checkpoint).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true, semanticResult: null }); + await original.close({ reason: "controller restarted" }); + second.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [], awaitingProviderTurn: true }] } }; + const recovery = await driver.recoverSession!(checkpoint); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + expect(await session.snapshot()).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true, dispositionOnlyRecoveryConsumed: true }); + await expect(session.startTurn({ message: { role: "user", text: "Report the disposition." } })).rejects.toThrow("session cannot start another turn"); + expect(second.calls.filter(call => call.method === "turn/start")).toEqual([]); + const events = session.events()[Symbol.asyncIterator]() as AsyncIterator; + releaseProviderTurn(second, "provider-1", { complete: true, withResult: true }); + const seen = await streamUntil(events, completedTurn("provider-1")); + expect(seen.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(startedTurns(seen)).toEqual([{ turnId: "provider-1", origin: "provider" }]); + expect(itemTurns(seen)).toEqual(["provider-1"]); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); +}); + +describe("Codex app-server driver boundary snapshots for provider-started turns", () => { + const sourceSeq = (value: unknown) => (value as { sourceSeq: number }).sourceSeq; + + it("keeps the state as of the awaiting terminal and of the provider start after the driver moved on", async () => { + const { transport, session, events, first } = await completedFirstTurn(true); + const terminalSeq = sourceSeq(first.at(-1)); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + transport.push("paperclip/runResult", { threadId: "thread-1", turnId: "provider-1", result }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "provider-1", status: "completed", items: [] } }); + const second = await nextUntil(events, eventType => eventType === "turn.completed"); + const startSeq = sourceSeq(second.find(value => value.eventType === "turn.started")); + const current = await session.snapshot(); + expect(current).toMatchObject({ activeTurnId: null, semanticResult: { turnId: "provider-1" } }); + expect(current.lastSourceSequence).toBeGreaterThan(startSeq); + expect(session.boundarySnapshot!(terminalSeq - 1)).toBeNull(); + expect(session.boundarySnapshot!(terminalSeq)).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true, semanticResult: null, lastSourceSequence: terminalSeq }); + expect(session.boundarySnapshot!(terminalSeq)).toBeNull(); + const atStart = session.boundarySnapshot!(startSeq); + expect(atStart).toMatchObject({ activeTurnId: "provider-1", semanticResult: null, lastSourceSequence: startSeq }); + expect(atStart).not.toHaveProperty("awaitingProviderTurn"); + expect(session.boundarySnapshot!(startSeq)).toBeNull(); + await session.close({ reason: "fixture complete" }); + }); + + it("drops an earlier boundary once a later one is asked for", async () => { + const { transport, session, events, first } = await completedFirstTurn(true); + const terminalSeq = sourceSeq(first.at(-1)); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const started = await nextUntil(events, eventType => eventType === "turn.started"); + expect(session.boundarySnapshot!(sourceSeq(started.at(-1)))).toMatchObject({ activeTurnId: "provider-1" }); + expect(session.boundarySnapshot!(terminalSeq)).toBeNull(); + await session.close({ reason: "fixture complete" }); + }); + + it("captures no boundary for a terminal that does not await", async () => { + const { session, first } = await completedFirstTurn(false); + expect(session.boundarySnapshot!(sourceSeq(first.at(-1)))).toBeNull(); + await session.close({ reason: "fixture complete" }); + }); + + it("keeps the disposition request owed in the boundary of a restart continuation that ends awaiting", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-awaiting-continuation", normalizedSessionId: "session-awaiting-continuation", workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Finish the request." } }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "failed", + error: { code: "provider_turn_lost_on_restore", recoverable: true }, items: [] } }); + await collectUntilTerminal(session.events()); + transport.turnStartResponse = Promise.resolve({ turn: { id: "turn-2", status: "inProgress", items: [] } }); + await expect(session.startTurn({ message: { role: "user", text: "Continue." }, continuation: true })) + .resolves.toMatchObject({ borrowedDispositionMarker: true }); + expect(await session.snapshot()).toMatchObject({ dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-2" }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-2", status: "completed", items: [] }, awaitingProviderTurn: true }); + const continuation = await collectUntilTerminal(session.events()); + const terminal = continuation.at(-1)!; + expect(terminal).toMatchObject({ eventType: "turn.completed", turnId: "turn-2", payload: { awaitingProviderTurn: true } }); + expect(session.boundarySnapshot!(terminal.sourceSeq)).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true, + dispositionOnlyRecoveryConsumed: false, dispositionOnlyRecoveryTurnId: null, lastSourceSequence: terminal.sourceSeq }); + await session.close({ reason: "fixture complete" }); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index ac4cd9d581e..97f64423f94 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -83,7 +83,7 @@ export class CodexHarnessSession "runtime_request_recovery_failed", error instanceof Error ? error.message : String(error), )); } - if (this.terminal) { + if (this.terminal && !this.awaitingProviderTurn) { this.#deferredNotificationPump = true; this.eventQueue.close(); } else { @@ -760,7 +760,13 @@ export class CodexHarnessSession } } - async reconcile(): Promise> { + reconcile(): Promise> { + const reconciled = this.#reconcile(); + this.recoveryReconciled = reconciled.then(() => undefined, () => undefined); + return reconciled; + } + + async #reconcile(): Promise> { this.assertProtocolIntegrity(); this.requireCapability("reconciliation"); const snapshot = await this.read(); @@ -816,11 +822,17 @@ export class CodexHarnessSession const unexpectedActive = activeTurns.find( (turn) => text(turn.id) !== expectedTurnId, ); - if (unexpectedActive !== undefined) { - throw new HarnessReconciliationError( - `thread/read exposed active turn ${text(unexpectedActive.id)} instead of persisted active turn ${expectedTurnId ?? "none"}`, - ); - } + const expectedIndex = turns.findIndex((turn) => text(turn.id) === expectedTurnId); + const awaitedProviderTurn = unexpectedActive !== undefined && activeTurns.length === 1 && expectedIndex >= 0 + && turns.indexOf(unexpectedActive) > expectedIndex && text(turns[expectedIndex]!.status) === "completed" + && turns[expectedIndex]!.awaitingProviderTurn === true; + const unexpectedActiveTurn = () => new HarnessReconciliationError( + `thread/read exposed active turn ${text(unexpectedActive?.id)} instead of persisted active turn ${expectedTurnId ?? "none"}`, + ); + const lastTerminalIndex = turns.findLastIndex((turn) => this.terminalTurns.has(text(turn.id))); + const awaitedWhileWaiting = unexpectedActive !== undefined && activeTurns.length === 1 && expectedTurnId === null + && this.awaitingProviderTurn && lastTerminalIndex >= 0 && turns.indexOf(unexpectedActive) > lastTerminalIndex; + if (unexpectedActive !== undefined && !awaitedProviderTurn && !awaitedWhileWaiting) throw unexpectedActiveTurn(); let reconciledTerminalTurnId: string | null = null; if (expectedTurnId !== null) { @@ -857,6 +869,7 @@ export class CodexHarnessSession } } } + if (awaitedProviderTurn && !this.awaitingProviderTurn) throw unexpectedActiveTurn(); for (const [turnId, fingerprint] of this.terminalTurns) { const reported = turns.find((turn) => text(turn.id) === turnId); @@ -899,6 +912,10 @@ export class CodexHarnessSession async snapshot(): Promise { this.assertProtocolIntegrity(); + return this.persistedSnapshot(); + } + + protected persistedSnapshot(): PersistedHarnessSession { return { driverKind: this.driverKind, workingDirectory: this.opened.context.workingDirectory, diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts index e33481395e9..f1ca06bc1c4 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts @@ -543,6 +543,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex return; } if (notification.method === "turn/started") { + if (params.origin === "provider" && !state.awaitingProviderTurn) await state.recoveryReconciled; // A start replayed for an earlier generation's turn (for example, one // the runner recorded after the previous controller detached and then // lost on restore) is history, never a start of this session's turn. A @@ -590,6 +591,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex { status: text(turn.status, "inProgress"), ...(providerStartedTurn ? { origin: "provider" } : {}) }, { turnId }, ); + if (providerStartedTurn) state.captureBoundary(state.sourceSequence); return; } if (notification.method === "turn/completed") { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts index 3a7ca9270db..8dc7c9ae091 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -7,6 +7,7 @@ import type { HarnessThreadGoal, HarnessThreadLineageEntry, PersistedHarnessSemanticResult, + PersistedHarnessSession, PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; import { @@ -91,7 +92,9 @@ class AsyncQueue implements AsyncIterable { } } -export class CodexSessionState { +const MAX_BOUNDARY_SNAPSHOTS = 8; + +export abstract class CodexSessionState { readonly transport: CodexAppServerTransport; runId: string; readonly normalizedSessionId: string; @@ -145,6 +148,7 @@ export class CodexSessionState { * the turn/start response arrives. */ turnStartSettled: Promise = Promise.resolve(); + recoveryReconciled: Promise = Promise.resolve(); protocolFailed = false; protocolFailureCode: string | null = null; protocolFailureMessage: string | null = null; @@ -578,6 +582,25 @@ export class CodexSessionState { }); } + protected abstract persistedSnapshot(): PersistedHarnessSession; + + readonly #boundarySnapshots = new Map(); + + captureBoundary(sourceSeq: number): void { + this.#boundarySnapshots.set(sourceSeq, structuredClone({ ...this.persistedSnapshot(), lastSourceSequence: sourceSeq })); + for (const seq of this.#boundarySnapshots.keys()) { + if (this.#boundarySnapshots.size <= MAX_BOUNDARY_SNAPSHOTS) break; + this.#boundarySnapshots.delete(seq); + } + } + + boundarySnapshot(sourceSeq: number): PersistedHarnessSession | null { + this.assertProtocolIntegrity(); + const snapshot = this.#boundarySnapshots.get(sourceSeq); + for (const seq of [...this.#boundarySnapshots.keys()]) if (seq <= sourceSeq) this.#boundarySnapshots.delete(seq); + return snapshot === undefined ? null : structuredClone(snapshot); + } + emitGoalCapabilities(): void { this.emitV2("session.capabilities.updated", { sessionGoals: diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts index 736c0166349..01feb00692c 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts @@ -225,6 +225,7 @@ export function mapTerminalTurn( payload.error = { ...record(payload.error), providerFailure }; } state.emit(eventType, payload, { turnId }); + const terminalSourceSeq = state.sourceSequence; finalize(state, status); if (status === "failed" && isNativeRestartInterruption(turn.error) && state.result === null && (!state.dispositionOnlyRecoveryConsumed || state.dispositionOnlyRecoveryTurnId === turnId)) { @@ -252,6 +253,7 @@ export function mapTerminalTurn( state.terminal = false; state.dispositionOnlyRecoveryAvailable = true; } + if (state.awaitingProviderTurn) state.captureBoundary(terminalSourceSeq); } function replayFingerprint(state: CodexSessionState, value: unknown): string { diff --git a/packages/paperclip-runner/src/live/provider-turn-tool-order.test.ts b/packages/paperclip-runner/src/live/provider-turn-tool-order.test.ts new file mode 100644 index 00000000000..40bcca677b0 --- /dev/null +++ b/packages/paperclip-runner/src/live/provider-turn-tool-order.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { providerTurnStartBefore } from "./provider-turn-tool-order.js"; + +type Stored = [number, string, Record?]; + +const start = (turnId: string, origin: string | null = "provider") => + ({ provider: "acpx", providerTurnId: turnId, status: "inProgress", turn: { id: turnId, status: "inProgress" }, ...(origin ? { origin } : {}) }); +const terminal = (turnId: string | null) => ({ provider: "acpx", providerTurnId: turnId, status: "failed" }); +const window = (stored: Stored[]) => stored.map(([sourceSeq, eventType, payload = {}]) => + ({ sourceSeq, eventType, envelope: { runId: "run-1", turnId: "turn-1", payload: { eventType, turnId: "turn-1", payload } } })); +const owner = (stored: Stored[], sourceSeq: number) => providerTurnStartBefore(window(stored), sourceSeq)?.turnId ?? null; + +describe("providerTurnStartBefore", () => { + it("returns the stored start when the window removed ordinary events after it", () => { + expect(owner([[10, "turn.started", start("a")], [40, "item.completed"], [41, "item.completed"]], 41)).toBe("a"); + }); + + it("returns the stored start across a dropped item delta", () => { + expect(owner([[10, "turn.started", start("a")], [11, "item.started"], [13, "item.completed"]], 13)).toBe("a"); + }); + + it("returns the start with its payload in the contiguous case", () => { + expect(providerTurnStartBefore(window([[10, "turn.started", start("a")], [11, "item.started"]]), 12)) + .toEqual({ turnId: "a", payload: start("a") }); + }); + + it("returns nothing for an event after the turn's own terminal", () => { + expect(owner([[10, "turn.started", start("a")], [20, "turn.completed", terminal("a")], [21, "item.completed"]], 21)).toBeNull(); + }); + + it("returns the start for an event before the turn's own terminal", () => { + expect(owner([[10, "turn.started", start("a")], [15, "item.completed"], [20, "turn.completed", terminal("a")]], 15)).toBe("a"); + }); + + it("returns the start for the turn's own terminal", () => { + expect(owner([[10, "turn.started", start("a")], [20, "turn.interrupted", terminal("a")]], 20)).toBe("a"); + }); + + it("returns nothing when the turn's terminal was removed and a startless terminal of another turn follows", () => { + const stored: Stored[] = [[10, "turn.started", start("a")], [30, "harness.diagnostic"], [31, "session.resumed"], [32, "turn.failed", terminal("controller-b")], [33, "item.completed"]]; + expect(owner(stored, 30)).toBeNull(); + expect(owner(stored, 32)).toBeNull(); + expect(owner(stored, 33)).toBeNull(); + }); + + it("returns nothing when the first terminal after the start names no turn", () => { + expect(owner([[10, "turn.started", start("a")], [30, "item.completed"], [32, "turn.failed", terminal(null)]], 30)).toBeNull(); + }); + + it("returns the start when its own terminal comes before a later foreign terminal", () => { + expect(owner([[10, "turn.started", start("a")], [15, "item.completed"], [20, "turn.completed", terminal("a")], [25, "turn.failed", terminal("controller-b")]], 15)).toBe("a"); + }); + + it("returns nothing when the nearest start is a controller start", () => { + expect(owner([[10, "turn.started", start("a")], [20, "turn.completed", terminal("a")], [30, "turn.started", start("b", null)], [31, "item.completed"]], 31)).toBeNull(); + }); + + it("returns nothing when no start is stored", () => { + expect(owner([[30, "item.completed"], [31, "item.completed"]], 31)).toBeNull(); + }); +}); diff --git a/packages/paperclip-runner/src/live/provider-turn-tool-order.ts b/packages/paperclip-runner/src/live/provider-turn-tool-order.ts index 95a93446d98..9ef6cabb64d 100644 --- a/packages/paperclip-runner/src/live/provider-turn-tool-order.ts +++ b/packages/paperclip-runner/src/live/provider-turn-tool-order.ts @@ -14,3 +14,21 @@ export function providerTurnToolCallSourceSeq( } return null; } + +const eventPayload = (event: Pick) => record(record(event.envelope.payload).payload); + +const terminalEventTypes = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]); + +export function providerTurnStartBefore( + events: readonly Pick[], + sourceSeq: number, +): { turnId: string; payload: Record } | null { + const startAt = events.findLastIndex(event => event.sourceSeq < sourceSeq && event.eventType === "turn.started"); + if (startAt < 0) return null; + const payload = eventPayload(events[startAt]!); + const turnId = payload.providerTurnId; + if (payload.origin !== "provider" || typeof turnId !== "string" || turnId.length === 0) return null; + const terminal = events.slice(startAt + 1).find(event => terminalEventTypes.has(event.eventType)); + if (terminal && (terminal.sourceSeq < sourceSeq || eventPayload(terminal).providerTurnId !== turnId)) return null; + return { turnId, payload }; +} diff --git a/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn.test.ts b/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn.test.ts index d1db45947a4..e7587253d45 100644 --- a/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-acpx-provider-turn.test.ts @@ -8,12 +8,13 @@ import { join, resolve } from "node:path"; import { expect, it, vi } from "vitest"; import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; import { QUALIFIED_ACPX_PROFILES, resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; -import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding } from "./runnerd-codex-transport.js"; -import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest } from "./runner-provider-launch.js"; +import { createCapabilityRunnerdCodexTransport, readRunnerdArtifactBinding, type CapabilityRunnerdCodexTransport, type CapabilityRunnerdCodexTransportOptions } from "./runnerd-codex-transport.js"; +import { createRunnerProviderLaunch, runnerLaunchRequirementsDigest, type RunnerProviderLaunchRequirements } from "./runner-provider-launch.js"; import { defaultCapabilityRunnerdBinary } from "../../test/support/runnerd-transport.js"; -import type { PrpEvent } from "../protocol/replay-contract.js"; +import type { PrpEvent, PrpStructuredRunResult } from "../protocol/replay-contract.js"; import { createNativeSessionBackend } from "../backends/native-backend-factory.js"; import type { NativeExecutionInput } from "../contracts/native-execution.js"; +import type { NativeSession, PersistedNativeSession } from "../contracts/native-session-backend.js"; import { executeNativeSession } from "../native-session-runtime.js"; const sha = (value: string | Buffer) => `sha256:${createHash("sha256").update(value).digest("hex")}`; @@ -38,6 +39,17 @@ let scenario = 0; const durableState = async (root: string) => (await Promise.all(["control-plane/control-plane-state.json", "runner/runner-state.json"] .map(path => readFile(join(root, path), "utf8").catch(() => "")))).join("\n"); +function storeSourceEvent(committed: PrpEvent[], value: PrpEvent) { + const source = () => committed.filter(event => event.sourceInstanceId === value.sourceInstanceId); + const existing = source().find(event => event.sourceEventId === value.sourceEventId || event.sourceSeq === value.sourceSeq); + if (existing && (existing.sourceEventId !== value.sourceEventId || JSON.stringify(existing.payload) !== JSON.stringify(value.payload))) throw new Error("source_event_replay_conflict"); + if (!existing) committed.push(structuredClone(value)); + const stored = new Set(source().map(event => event.sourceSeq)); + let highestContiguousSourceSeq = 0; + while (stored.has(highestContiguousSourceSeq + 1)) highestContiguousSourceSeq += 1; + return { cursor: value.sourceSeq, highestContiguousSourceSeq, disposition: existing ? "duplicate" as const : "committed" as const }; +} + async function runProviderTurnScenario(tokens: string, options: { agent?: "claude" | "codex"; capMs?: number; providerStartedTurns?: boolean } = {}) { const agent = options.agent ?? "claude"; const providerStartedTurns = options.providerStartedTurns ?? agent === "claude"; @@ -103,7 +115,7 @@ async function runProviderTurnScenario(tokens: string, options: { agent?: "claud } }); const calls = async () => (await readFile(journal, "utf8").catch(() => "")).trim().split("\n").filter(Boolean) .flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); - const appended: PrpEvent[] = []; + const appended: PrpEvent[] = [], committed: PrpEvent[] = []; const completeRun = vi.fn(async () => undefined); const resolveMissingResult = vi.fn(async () => null); try { @@ -116,7 +128,7 @@ async function runProviderTurnScenario(tokens: string, options: { agent?: "claud async openRun() {}, async checkpointSession() {}, async appendEvent(value: PrpEvent) { appended.push(structuredClone(value)); - return { cursor: value.sourceSeq, highestContiguousSourceSeq: value.sourceSeq, disposition: "committed" }; + return storeSourceEvent(committed, value); }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, completeRun, @@ -237,3 +249,558 @@ it.skipIf(process.platform === "win32")("never opts a non-Claude ACPX agent in", expect(sidecarCommands(run.journalled)).not.toContain("session.provider_turn"); expectCleanClose(run); }, 60_000); + +async function restartFixture(tokens: string, sidecarArgs = "", optIn = true) { + const n = ++scenario; + const root = await realpath(await mkdtemp(join(tmpdir(), "runner-acpx-provider-turn-restart-"))); + const journal = join(root, "calls"), fault = join(root, "fault"), wrapper = join(root, "sidecar"), script = join(root, "sidecar-entry"); + const background = join(root, "background"); + const binary = defaultCapabilityRunnerdBinary(), sidecar = resolve(import.meta.dirname, "../../runner/target/debug/fake-acpx-sidecar"); + await mkdir(join(root, "acpx"), { mode: 0o700 }); await mkdir(join(root, "diagnostics"), { mode: 0o700 }); + await writeFile(script, "qualified fixture entry"); + await writeFile(background, tokens); + await writeFile(wrapper, `#!/bin/sh\nexec ${quote(sidecar)} --mode source-reconcile --profile-digest ${quote(QUALIFIED_ACPX_PROFILES.claude.commandDigest)} --journal ${quote(journal)} --source-fault ${quote(fault)} --background-tasks ${quote(background)} --spawn-owned-child${sidecarArgs}\n`, { mode: 0o700 }); + let authority: DurablePrpControlPlane | null = null, registrations = 0; + const diagnostics: string[] = []; + const server = createServer(); + server.on("upgrade", (request, socket, head) => authority ? authority.handleUpgrade(request, socket, "/runner", head) : socket.destroy()); + await new Promise(done => server.listen(0, "127.0.0.1", done)); + const address = server.address(); if (!address || typeof address === "string") throw new Error("missing fixture listener"); + const identity = { runnerInstanceId: `runner-provider-turn-restart-${n}`, environmentLeaseId: `lease-provider-turn-restart-${n}`, runId: `run-provider-turn-restart-${n}`, + normalizedSessionId: `session-provider-turn-restart-${n}`, turnId: `turn-provider-turn-restart-${n}`, itemId: `item-provider-turn-restart-${n}` }; + const nativeIdentity = { companyId: `company-restart-${n}`, agentId: "agent", issueId: "issue", runId: identity.runId, sessionId: identity.normalizedSessionId }; + const execution: NativeExecutionInput = { schema: "paperclip.native-execution-input.v1", + binding: { companyId: nativeIdentity.companyId, agentId: "agent", issueId: "issue", runId: identity.runId, executionWorkspaceId: "workspace" }, + task: { identifier: "TASK-1", title: "Install and test", description: null, prompt: "Install and test", workMode: "standard" }, + workspace: { cwd: root, repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: identity.normalizedSessionId, driverKind: "acpx_runtime", protocolVersion: 1, lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null } }, + provider: { kind: "acpx", agent: "claude", model: "explicit-source-model", permissionPolicy: "interactive", profile: qualifiedProfile("claude") }, + completionContract: { id: "contract", sha256: sha("contract"), schemaVersion: "paperclip.completion-contract.v1", + contract: { revision: "acpx-provider-turns-v1", objective: "Install and test", criteria: [{ id: "objective", requirement: "Install and test" }] } }, + interactionResponses: [], credentialBindings: [] }; + const runnerPids = new Set(); + let requirements: RunnerProviderLaunchRequirements | undefined; + const transportOptions: CapabilityRunnerdCodexTransportOptions = { + provider: "acpx", acpxAgent: "claude", ...(optIn ? { providerStartedTurns: true } : {}), onDiagnostic: message => diagnostics.push(message), runnerBinary: binary, stateDirectory: root, prpIdentity: identity, + runnerArtifact: readRunnerdArtifactBinding(binary), codexSourceRecoveryVersion: 1, acpxSourceRecoveryVersion: 1, + lifecyclePolicy: { mode: "per_turn", idleTimeoutMs: null }, runnerProcessReconnectGraceMs: null, runnerReconnectGraceMs: 10_000, + controlPlaneRegistration: async next => { + const generation = ++registrations; authority = next; + return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: () => { if (generation === registrations) authority = null; } }; + }, + prepareLaunch: async launch => { + requirements ??= structuredClone(launch); + return { configurationId: `initial-${launch.identity.runId}`, requirementsDigest: runnerLaunchRequirementsDigest(launch), + provider: createRunnerProviderLaunch(launch, { options: {}, root, cwd: root, codexHome: root, codexCommand: "", providerNodeCommand: wrapper, opencodeProxyPath: "", acpxSidecarPath: script }) }; + }, + runnerProcessLauncher: spec => { + const stderr = openSync(join(root, "native.stderr"), "a", 0o600); + const child = spawn(binary, [...spec.args, "--acpx-launch-authority-digest", sha("fixture authority"), + "--acpx-sidecar-command", wrapper, "--acpx-sidecar-command-sha256", sha(execFileSync("cat", [wrapper])), + "--acpx-sidecar-script", script, "--acpx-sidecar-script-sha256", sha("qualified fixture entry")], + { env: { PATH: process.env.PATH, HOME: root, ...spec.environment }, stdio: ["ignore", "ignore", stderr], detached: true }); + closeSync(stderr); + if (child.pid) runnerPids.add(child.pid); + return { child, completion: new Promise((done, reject) => { child.once("error", reject); child.once("exit", (code, signal) => done({ code, signal, stdout: "", stderr: "" })); }) }; + }, + }; + const calls = async () => (await readFile(journal, "utf8").catch(() => "")).trim().split("\n").filter(Boolean) + .flatMap(line => { try { return [JSON.parse(line)]; } catch { return []; } }); + const backend = (transport: CapabilityRunnerdCodexTransport) => createNativeSessionBackend(execution, { codexTransportFactory: () => transport.transport, + runnerInstanceId: identity.runnerInstanceId, environment: { PAPERCLIP_WORKSPACE_CWD: root }, dynamicTools: [] }); + const first = (holdFirstToolCall: boolean | undefined, onHeld: () => void = () => undefined) => { + const transport = createCapabilityRunnerdCodexTransport(transportOptions); + const install = transport.transport.setServerRequestHandler.bind(transport.transport); + if (holdFirstToolCall) transport.transport.setServerRequestHandler = handler => install(request => { + if (request.method !== "item/tool/call") return handler(request); + onHeld(); + return new Promise(() => undefined); + }); + return transport; + }; + const restarted = (first: CapabilityRunnerdCodexTransport, checkpoint: PersistedNativeSession, extra: Partial = {}) => { + const pid = first.evidence().runnerPid!, startedAt = first.transport.processInfo!().startedAt; + return () => { + const second = createCapabilityRunnerdCodexTransport({ ...transportOptions, recoveryLaunch: { requirements: requirements! }, + resumeProviderSession: { driverSessionId: checkpoint.sessionId, providerSessionId: checkpoint.providerSessionId!, providerIdentity: checkpoint.providerIdentity }, + authorityRunId: identity.runId, runnerProcessLauncher: () => { throw new Error("duplicate runner launch"); }, + adoptExistingRunner: { pid, processGroupId: pid, startedAt, isAlive: () => alive(pid), signal: () => false }, ...extra }); + const serverRequests: Array> = []; + const install = second.transport.setServerRequestHandler.bind(second.transport); + second.transport.setServerRequestHandler = handler => install(async request => { + const response = await handler(request); + serverRequests.push(structuredClone({ method: request.method, params: request.params, response })); + return response; + }); + return { second, pid, serverRequests }; + }; + }; + const atRestart = async () => ({ journalled: await calls(), durable: await durableState(root) }); + const fail = async (error: unknown) => new Error(`${String(error)}\n${await readFile(join(root, "native.stderr"), "utf8").catch(() => "")}`, { cause: error }); + const dispose = async () => { + for (const pid of runnerPids) kill(pid); + for (const entry of await calls()) if (entry.processStart) kill(entry.processStart); + server.closeAllConnections(); await new Promise(done => server.close(() => done())); + await rm(root, { recursive: true, force: true }); + }; + return { root, identity, nativeIdentity, execution, diagnostics, calls, backend, first, restarted, atRestart, fail, dispose }; +} + +async function runRestartScenario(tokens: string, options: { waitMs: number; lastTurnId: string; detachDelayMs?: number; holdFirstToolCall?: boolean; snapshotAtProviderTurn?: boolean }) { + const fixture = await restartFixture(tokens); + try { + const first = fixture.first(options.holdFirstToolCall); + const session = await fixture.backend(first).openSession({ identity: fixture.nativeIdentity, workingDirectory: fixture.root }); + await session.startTurn({ message: { role: "user", text: "Install and test." } }); + for await (const event of session.events()) if (options.snapshotAtProviderTurn ? event.payload.origin === "provider" : event.eventType === "turn.completed") break; + const checkpoint = await session.snapshot(); + await new Promise(done => setTimeout(done, options.detachDelayMs ?? 0)); + const restart = fixture.restarted(first, checkpoint, { resumeActiveTurnId: checkpoint.activeTurnId ?? null }); + await session.detachControllerForRestart!(); + await new Promise(done => setTimeout(done, options.waitMs)); + const atRestart = await fixture.atRestart(); + const { second, pid, serverRequests } = restart(); + const recovery = await fixture.backend(second).recoverSession!(checkpoint, { signal: new AbortController().signal }); + const result = { checkpoint, pid, recoveredPid: second.evidence().runnerPid, recovered: recovery.recovered, reason: recovery.reason, atRestart, serverRequests, + delivered: [] as PrpEvent[], snapshot: null as unknown, closeOutcome: "not closed", journalled: [] as Array>, diagnostics: fixture.diagnostics, durable: "" }; + if (recovery.session) { + const events = recovery.session.events()[Symbol.asyncIterator](); + const timeout = new Promise(done => setTimeout(() => done(null), 15_000)); + for (let next = await Promise.race([events.next(), timeout]); next && !next.done; next = await Promise.race([events.next(), timeout])) { + result.delivered.push(next.value); + if (next.value.eventType === "session.failed" || (next.value.eventType === "turn.completed" && next.value.turnId === options.lastTurnId)) break; + } + result.snapshot = await recovery.session.snapshot(); + result.closeOutcome = await recovery.session.close({ reason: "fixture complete" }).then(() => "closed", (error: unknown) => String(error instanceof Error ? error.message : error)); + } + const final = await fixture.atRestart(); + return { ...result, ...{ journalled: final.journalled, durable: final.durable } }; + } catch (error) { + throw await fixture.fail(error); + } finally { + await fixture.dispose(); + } +} + +const replyResult: PrpStructuredRunResult = { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Claude finished its own work.", + completionClaim: { contractRevision: "acpx-provider-turns-v1", objectiveSatisfied: true, criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], remainingWork: [] }, + evidence: [], verification: [], attentionRequests: [], artifacts: [] }; + +async function runConsumerRestartScenario(tokens: string, options: { at: "wait" | "provider-turn" | "turn"; waitMs: number; replyResult?: boolean; holdFirstToolCall?: boolean; sidecarArgs?: string; optIn?: boolean; + slowAppend?: (event: PrpEvent) => boolean; holdWaitCheckpoint?: boolean }) { + const fixture = await restartFixture(tokens, options.sidecarArgs, options.optIn ?? true); + const { identity, execution } = fixture; + const appended: PrpEvent[] = [], committed: PrpEvent[] = [], appendedAt: number[] = [], checkpoints: PersistedNativeSession[] = []; + const completeRun = vi.fn(async (_value: unknown) => undefined); + let fenced = false, held = false, slowed = false; + let releaseQueued = () => {}; + const fence = new Promise(done => { releaseQueued = done; }); + const queuedAtRestart: PrpEvent[] = []; + let heldCheckpoint: PersistedNativeSession | null = null; + const controlPlane = (fencedController: boolean) => { + const guard = () => { if (fencedController && fenced) throw new Error("controller fenced for restart"); }; + return { + async openRun() { guard(); }, + async checkpointSession(value: PersistedNativeSession) { + guard(); + if (fencedController && options.holdWaitCheckpoint && value.awaitingProviderTurn === true) { heldCheckpoint = structuredClone(value); await fence; guard(); } + checkpoints.push(structuredClone(value)); + }, + async appendEvent(value: PrpEvent) { + guard(); + if (fencedController && options.slowAppend) { + if (slowed) { queuedAtRestart.push(structuredClone(value)); await fence; guard(); } + if (options.slowAppend(value)) { slowed = true; await new Promise(done => setTimeout(done, 1_000)); } + } + appended.push(structuredClone(value)); appendedAt.push(Date.now()); + return storeSourceEvent(committed, value); + }, + async replayEvents(request: { sourceInstanceId: string; afterSourceSeq: number; limit?: number }) { + const events = committed.filter(event => event.sourceInstanceId === request.sourceInstanceId && event.sourceSeq > request.afterSourceSeq) + .sort((left, right) => left.sourceSeq - right.sourceSeq).slice(0, request.limit ?? 1_000); + let highestContiguousSourceSeq = request.afterSourceSeq; + for (const event of events) if (event.sourceSeq === highestContiguousSourceSeq + 1) highestContiguousSourceSeq += 1; + return { events: structuredClone(events), highestContiguousSourceSeq }; + }, + async completeRun(value: unknown) { guard(); await completeRun(value); }, + }; + }; + const execute = (backend: ReturnType, fencedController: boolean, extra: Partial[0]>) => executeNativeSession({ + input: execution, backend, runnerInstanceId: identity.runnerInstanceId, controlPlaneInstanceId: "control-provider-turn-restart", + controlPlane: controlPlane(fencedController), timeoutMs: 30_000, ...extra, + resolveMissingResult: async ({ terminalEvent }) => options.replyResult && terminalEvent.turnId === "provider-1" && !terminalEvent.payload.awaitingProviderTurn ? structuredClone(replyResult) : null, + }).then(() => "finalized", (error: unknown) => String(error instanceof Error ? error.message : error)); + try { + const first = fixture.first(options.holdFirstToolCall, () => { held = true; }); + let session: NativeSession | null = null; + const stopFirst = new AbortController(); + const firstOutcome = execute(fixture.backend(first), true, { signal: stopFirst.signal, onSession: value => { session = value ?? session; } }); + const reached = (event: PrpEvent) => options.at === "wait" ? event.payload.awaitingProviderTurn === true + : options.at === "turn" ? event.eventType === "turn.started" : event.eventType === "turn.started" && event.payload.origin === "provider"; + const shaped = (value: PersistedNativeSession) => options.at === "wait" ? value.awaitingProviderTurn === true + : options.at === "turn" ? typeof value.activeTurnId === "string" : value.activeTurnId === "provider-1"; + await vi.waitFor(() => expect(appended.some(reached) && (!options.holdFirstToolCall || held)).toBe(true), { timeout: 20_000, interval: 20 }); + await vi.waitFor(() => expect(checkpoints.some(shaped)).toBe(true), { timeout: options.slowAppend ? 10_000 : 1_000, interval: 20 }).catch(() => undefined); + if (options.slowAppend) await vi.waitFor(() => expect(queuedAtRestart.length).toBeGreaterThan(0), { timeout: 5_000, interval: 20 }); + if (options.holdWaitCheckpoint) await vi.waitFor(() => expect(heldCheckpoint).not.toBeNull(), { timeout: 5_000, interval: 20 }); + fenced = true; + releaseQueued(); + const committedAtRestart = structuredClone(committed); + const checkpoint = checkpoints.at(-1)!; + const restart = fixture.restarted(first, checkpoint, { resumeActiveTurnId: checkpoint.activeTurnId ?? null }); + await (session as NativeSession | null)!.detachControllerForRestart!(); + const firstSettled = await Promise.race([firstOutcome, new Promise(done => setTimeout(() => done(null), 3_000))]); + if (firstSettled === null) stopFirst.abort(new Error("controller process exited")); + await firstOutcome; + await new Promise(done => setTimeout(done, options.waitMs)); + const atRestart = await fixture.atRestart(); + const { second, pid, serverRequests } = restart(); + const awaitingAt = appendedAt[appended.findIndex(event => event.payload.awaitingProviderTurn === true)] ?? null; + const firstAppended = appended.length; + const outcome = await execute(fixture.backend(second), false, { persistedSession: checkpoint, providerTurnWait: { startedAtMs: awaitingAt } }); + const recoveredPid = second.evidence().runnerPid; + const closeOutcome = await second.transport.close().then(() => "closed", (error: unknown) => String(error instanceof Error ? error.message : error)); + const final = await fixture.atRestart(); + return { runnerInstanceId: identity.runnerInstanceId, checkpoint, pid, recoveredPid, atRestart, serverRequests, appended, appendedAfterRestart: appended.slice(firstAppended), checkpoints, outcome, completeRun, closeOutcome, + diagnostics: fixture.diagnostics, journalled: final.journalled, durable: final.durable, committed, committedAtRestart, queuedAtRestart, heldCheckpoint }; + } catch (error) { + throw await fixture.fail(error); + } finally { + await fixture.dispose(); + } +} + +type RestartRun = Awaited>; +const ofType = (run: RestartRun, eventType: string) => run.delivered.filter(event => event.eventType === eventType); +const resolvedCalls = (journalled: Array>) => journalled.filter(entry => entry.command === "tool.resolve") + .map(entry => [entry.params.turnId, entry.params.callId, entry.params.error ?? null]); +const acked = (run: RestartRun, eventType: string) => (JSON.parse(run.atRestart.durable.split("\n")[0]!).committedEvents as Array>) + .filter(event => event.eventType === eventType).map(event => event.envelope.payload.payload); +const ackedStarts = (run: RestartRun) => acked(run, "turn.started").map(payload => [payload.providerTurnId ?? null, payload.origin ?? null]); +const ackedTerminals = (run: RestartRun) => acked(run, "turn.completed").map(payload => payload.providerTurnId); +const expectProviderTurnsInOrder = (run: RestartRun, turnIds: string[]) => { + expect(ofType(run, "turn.started").map(event => [event.turnId, event.payload.origin])).toEqual(turnIds.map(turnId => [turnId, "provider"])); + for (const turnId of turnIds) { + const turn = run.delivered.filter(event => event.turnId === turnId); + expect(turn[0]?.eventType).toBe("turn.started"); + expect(turn.at(-1)?.eventType).toBe("turn.completed"); + } + const owners = run.delivered.filter(event => event.turnId?.startsWith("provider-")).map(event => event.turnId); + expect(owners).toEqual([...owners].sort()); + const sequence = run.delivered.map(event => event.sourceSeq); + expect(sequence).toEqual([...new Set(sequence)].sort((left, right) => left - right)); +}; +const armedWaits = (journalled: Array>) => journalled.filter(entry => entry.command === "session.provider_turn" && entry.params.armed === true).length; +const expectRecoveredCleanly = (run: RestartRun) => { + expect(run.recovered, run.reason).toBe(true); + expect(run.recoveredPid).toBe(run.pid); + expect(ofType(run, "session.failed")).toEqual([]); + expect(run.closeOutcome).toBe("closed"); + expect(unsettled(run.diagnostics)).toEqual([]); + expect(run.durable).not.toContain("conflicting duplicate"); +}; + +it.skipIf(process.platform === "win32").each([true, false])("delivers the provider's own turn that ran while the controller was away, after a restart during the wait (result: %s)", async withResult => { + const run = await runRestartScenario(`live provider-turn ${withResult ? "provider-result-resolved " : ""}provider-goal-at-terminal provider-turn-delay-ms=1500`, { waitMs: 2_500, lastTurnId: "provider-1" }); + expect(run.checkpoint).toMatchObject({ activeTurnId: null, semanticResult: null, awaitingProviderTurn: true }); + expectRecoveredCleanly(run); + expect(ofType(run, "turn.started").map(event => [event.turnId, event.payload.origin])).toEqual([["provider-1", "provider"]]); + expect(ofType(run, "item.completed").filter(event => event.payload.kind === "agentMessage").map(event => event.turnId)).toEqual(["provider-1"]); + expect(run.delivered.filter(event => event.payload.kind === "usage").map(event => event.turnId)).toEqual(["provider-1"]); + if (withResult) { + expect(ofType(run, "run.result.proposed").map(event => event.turnId)).toEqual(["provider-1"]); + expect(run.snapshot).toMatchObject({ semanticResult: { reportedWorkDisposition: "done" }, terminal: { turnTerminalState: "completed", runTerminalState: "succeeded" } }); + expect(resolvedCalls(run.journalled)).toEqual([["provider-1", "call-provider-finish", null]]); + expect(run.serverRequests.map(request => [request.params.turnId, request.params.tool, request.response.success])).toEqual([["provider-1", "paperclip_finish", true]]); + } else expect(ofType(run, "run.result.proposed")).toEqual([]); +}, 60_000); + +it.skipIf(process.platform === "win32").each([0, 1_500])("delivers two provider turns that the runner started while the controller was away, in order (detach delay: %i ms)", async detachDelayMs => { + const run = await runRestartScenario("live provider-turn provider-turns=live/resolved provider-goal-at-terminal provider-turn-delay-ms=1000", { waitMs: 3_500 - detachDelayMs, lastTurnId: "provider-2", detachDelayMs }); + expect(run.atRestart.journalled.filter(entry => entry.command === "session.provider_turn").map(entry => [entry.params.turnId, entry.params.armed])) + .toEqual([[expect.any(String), true], ["provider-1", true]]); + expect(resolvedCalls(run.atRestart.journalled)).toEqual([]); + const pending = ["provider-1", "provider-2"].filter(turnId => !ackedTerminals(run).includes(turnId)); + expect(pending).toContain("provider-2"); + expectRecoveredCleanly(run); + expectProviderTurnsInOrder(run, pending); + expect(ofType(run, "turn.completed").map(event => [event.turnId, event.payload.awaitingProviderTurn ?? false])) + .toEqual([["provider-1", true], ["provider-2", false]].filter(([turnId]) => pending.includes(turnId as string))); + expect(ofType(run, "item.completed").filter(event => event.payload.kind === "agentMessage").map(event => event.turnId)).toEqual(pending); + expect(run.delivered.filter(event => event.payload.kind === "usage").map(event => event.turnId)).toEqual(pending); + expect(ofType(run, "run.result.proposed").map(event => event.turnId)).toEqual(["provider-2"]); + expect(run.serverRequests.map(request => [request.params.turnId, request.params.tool, request.response.success])).toEqual([["provider-2", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-2", "call-provider-finish", null]]); + expect(run.snapshot).toMatchObject({ semanticResult: { reportedWorkDisposition: "done" }, terminal: { turnTerminalState: "completed", runTerminalState: "succeeded" } }); +}, 60_000); + +it.skipIf(process.platform === "win32")("delivers the start of a provider turn that the old controller acknowledged, before the rest of the turn", async () => { + const run = await runRestartScenario("live provider-turn provider-goal-at-terminal provider-turn-delay-ms=300 provider-turn-until-snapshot", { waitMs: 500, lastTurnId: "provider-1", detachDelayMs: 1_500 }); + expect(run.checkpoint).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true }); + expect(ackedStarts(run)).toContainEqual(["provider-1", "provider"]); + expect(run.atRestart.durable.split("\n")[0]).toContain("Claude is still working."); + expect(run.atRestart.durable).not.toContain("Claude finished its own work."); + expectRecoveredCleanly(run); + expectProviderTurnsInOrder(run, ["provider-1"]); + expect(run.delivered.filter(event => event.turnId === "provider-1").map(event => [event.eventType, event.turnId, event.payload.kind ?? null])).toEqual([["turn.started", "provider-1", null], + ["item.delta", "provider-1", "agentMessage"], ["item.completed", "provider-1", "usage"], ["item.completed", "provider-1", "agentMessage"], ["turn.completed", "provider-1", null]]); + expect(ofType(run, "run.result.proposed")).toEqual([]); + expect(run.serverRequests).toEqual([]); +}, 60_000); + +it.skipIf(process.platform === "win32")("delivers two provider turns after a restart when the old controller acknowledged the first turn's start", async () => { + const run = await runRestartScenario("live provider-turn provider-turns=live/resolved provider-goal-at-terminal provider-turn-delay-ms=300 provider-turn-gap-ms=2500", { waitMs: 2_500, lastTurnId: "provider-2", detachDelayMs: 1_500 }); + expect(run.checkpoint).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true }); + expect(ackedStarts(run)).toContainEqual(["provider-1", "provider"]); + expect(run.atRestart.durable.split("\n")[0]).toContain("Claude is still working."); + expect(run.atRestart.journalled.filter(entry => entry.command === "session.provider_turn").map(entry => entry.params.turnId)).toEqual([expect.any(String), "provider-1"]); + expectRecoveredCleanly(run); + expectProviderTurnsInOrder(run, ["provider-1", "provider-2"]); + expect(ofType(run, "turn.completed").map(event => [event.turnId, event.payload.awaitingProviderTurn ?? false])).toEqual([["provider-1", true], ["provider-2", false]]); + expect(run.delivered.filter(event => event.payload.kind === "usage").map(event => event.turnId)).toEqual(["provider-1", "provider-2"]); + expect(ofType(run, "run.result.proposed").map(event => event.turnId)).toEqual(["provider-2"]); + expect(run.serverRequests.map(request => [request.params.turnId, request.params.tool, request.response.success])).toEqual([["provider-2", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-2", "call-provider-finish", null]]); + expect(run.snapshot).toMatchObject({ semanticResult: { reportedWorkDisposition: "done" }, terminal: { turnTerminalState: "completed", runTerminalState: "succeeded" } }); +}, 60_000); + +it.skipIf(process.platform === "win32")("answers the open finish call of a provider turn whose start the old controller acknowledged", async () => { + const run = await runRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal provider-turn-delay-ms=300", { waitMs: 500, lastTurnId: "provider-1", detachDelayMs: 1_500, holdFirstToolCall: true }); + expect(run.checkpoint).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true }); + expect(ackedStarts(run)).toContainEqual(["provider-1", "provider"]); + expect(resolvedCalls(run.atRestart.journalled)).toEqual([]); + expect(run.atRestart.durable).toContain("call-provider-finish"); + expectRecoveredCleanly(run); + expectProviderTurnsInOrder(run, ["provider-1"]); + expect(run.serverRequests.map(request => [request.params.turnId, request.params.tool, request.response.success])).toEqual([["provider-1", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-1", "call-provider-finish", null]]); + expect(ofType(run, "run.result.proposed").map(event => event.turnId)).toEqual(["provider-1"]); + expect(run.snapshot).toMatchObject({ activeTurnId: null, semanticResult: { reportedWorkDisposition: "done" }, terminal: { turnTerminalState: "completed", runTerminalState: "succeeded" } }); +}, 60_000); + +type ConsumerRun = Awaited>; +const waitingCheckpoint = { activeTurnId: null, awaitingProviderTurn: true }; +const providerTurnCheckpoint = { activeTurnId: "provider-1" }; +const expectSettledAfterRestart = (run: ConsumerRun, turnIds: string[]) => { + expect(run.outcome, run.diagnostics.slice(-12).join("\n")).toBe("finalized"); + expect(run.recoveredPid).toBe(run.pid); + expect(run.completeRun).toHaveBeenCalledOnce(); + expect(run.completeRun.mock.calls[0]![0]).toMatchObject({ turnId: turnIds.at(-1), result: { reportedWorkDisposition: "done" } }); + expect(run.appended.filter(event => event.eventType === "turn.started").map(event => [event.turnId, event.payload.origin ?? null])) + .toEqual([[expect.any(String), null], ...turnIds.map(turnId => [turnId, "provider"])]); + expect(run.appended.filter(event => event.eventType === "session.failed")).toEqual([]); + expect(run.appended.filter(event => event.eventType === "turn.completed" && event.turnId?.startsWith("provider-")).map(event => event.turnId)).toEqual(turnIds); + expect(run.appended.filter(event => event.payload.kind === "usage" && event.turnId?.startsWith("provider-")).map(event => event.turnId)).toEqual(turnIds); + const sequence = run.appended.filter(event => event.sourceInstanceId === run.runnerInstanceId).map(event => event.sourceSeq); + expect(sequence).toEqual([...new Set(sequence)].sort((left, right) => left - right)); + expect(run.closeOutcome).toBe("closed"); + expect(unsettled(run.diagnostics)).toEqual([]); + expect(run.durable).not.toContain("conflicting duplicate"); +}; +const expectNoTerminalPersistedWithoutResult = (run: ConsumerRun) => + expect(run.checkpoints.filter(checkpoint => (checkpoint.semanticResult ?? null) === null && (checkpoint.terminal ?? null) !== null)).toEqual([]); + +it.skipIf(process.platform === "win32")("settles a provider turn that starts after the new controller attached, from the checkpoint saved when the wait began", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal provider-turn-after-snapshot", { at: "wait", waitMs: 200 }); + expect(run.checkpoint).toMatchObject(waitingCheckpoint); + expect(run.atRestart.durable).not.toContain("call-provider-finish"); + expectSettledAfterRestart(run, ["provider-1"]); + expectNoTerminalPersistedWithoutResult(run); +}, 60_000); + +it.skipIf(process.platform === "win32")("settles a provider turn that was running at a restart, from the checkpoint saved when it started", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-goal-at-terminal provider-turn-delay-ms=300 provider-turn-until-snapshot", { at: "provider-turn", waitMs: 1_000, replyResult: true }); + expect(run.checkpoint).toMatchObject(providerTurnCheckpoint); + expect(run.atRestart.durable).toContain("Claude is still working."); + expect(run.atRestart.durable).not.toContain("Claude finished its own work."); + expectSettledAfterRestart(run, ["provider-1"]); + expect(run.serverRequests).toEqual([]); +}, 60_000); + +it.skipIf(process.platform === "win32")("delivers the rest of a provider turn active in the checkpoint without a second start, then the next provider turn", async () => { + const run = await runRestartScenario("live provider-turn provider-turns=live/resolved provider-goal-at-terminal provider-turn-delay-ms=300 provider-turn-gap-ms=2500", { waitMs: 3_500, lastTurnId: "provider-2", snapshotAtProviderTurn: true }); + expect(run.checkpoint).toMatchObject({ activeTurnId: "provider-1" }); + expect(ackedStarts(run)).toContainEqual(["provider-1", "provider"]); + expect(ackedTerminals(run)).not.toContain("provider-1"); + expectRecoveredCleanly(run); + expect(ofType(run, "turn.started").map(event => [event.turnId, event.payload.origin])).toEqual([["provider-2", "provider"]]); + const owners = run.delivered.filter(event => event.turnId?.startsWith("provider-")).map(event => event.turnId); + expect(owners).toEqual([...owners].sort()); + expect(run.delivered.filter(event => event.eventType.startsWith("turn.")).map(event => [event.eventType, event.turnId])) + .toEqual([["turn.completed", "provider-1"], ["turn.started", "provider-2"], ["turn.completed", "provider-2"]]); + expect(run.delivered.filter(event => event.payload.kind === "usage").map(event => event.turnId)).toEqual(["provider-1", "provider-2"]); + expect(run.serverRequests.map(request => [request.params.turnId, request.params.tool, request.response.success])).toEqual([["provider-2", "paperclip_finish", true]]); + const sequence = run.delivered.map(event => event.sourceSeq); + expect(sequence).toEqual([...new Set(sequence)].sort((left, right) => left - right)); +}, 60_000); + +const answeredCalls = (run: ConsumerRun) => run.serverRequests.map(request => [request.params.turnId, request.params.tool, request.response.success]); +const providerTurnEvents = (events: PrpEvent[]) => events + .filter(event => event.turnId?.startsWith("provider-") && !event.eventType.startsWith("session.") && event.eventType !== "item.delta") + .map(event => [event.eventType, event.turnId, event.payload.kind ?? event.payload.origin ?? null]); +const settledProviderTurn = (turnId: string, opening: unknown[][]) => [ + ...opening, + ["item.completed", turnId, "usage"], + ["item.completed", turnId, "agentMessage"], + ["turn.completed", turnId, null], +]; +const acceptedResult = (turnId: string) => [["run.result.accepted", turnId, null], ["run.terminal", turnId, null]]; +const startedWithResult = (turnId: string) => [["turn.started", turnId, "provider"], ["run.result.proposed", turnId, null]]; + +it.skipIf(process.platform === "win32")("R0 settles an ordinary turn whose finish call was open at a restart and appends its result, completion and terminal once", async () => { + const run = await runConsumerRestartScenario("quiet", { at: "turn", waitMs: 500, holdFirstToolCall: true, sidecarArgs: " --wait-completion-feedback", optIn: false }); + const turnId = run.checkpoint.activeTurnId!; + expect(turnId).toEqual(expect.any(String)); + expect(run.outcome, run.diagnostics.slice(-12).join("\n")).toBe("finalized"); + expect(run.recoveredPid).toBe(run.pid); + expect(run.completeRun).toHaveBeenCalledOnce(); + expect(run.completeRun.mock.calls[0]![0]).toMatchObject({ turnId, result: { reportedWorkDisposition: "done" } }); + expect(answeredCalls(run)).toEqual([[turnId, "paperclip_finish", true]]); + expect(run.appendedAfterRestart.filter(event => event.turnId === turnId && !event.eventType.startsWith("session.")).map(event => event.eventType)) + .toEqual(["run.result.proposed", "turn.completed", "run.result.accepted", "run.terminal"]); + const sequence = run.appended.filter(event => event.sourceInstanceId === run.runnerInstanceId).map(event => event.sourceSeq); + expect(sequence).toEqual([...new Set(sequence)].sort((left, right) => left - right)); + expect(run.closeOutcome).toBe("closed"); + expect(unsettled(run.diagnostics)).toEqual([]); +}, 60_000); + +it.skipIf(process.platform === "win32")("R1 appends a provider turn that ran while the controller was away, from the checkpoint saved when the wait began", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal provider-turn-delay-ms=1500", { at: "wait", waitMs: 3_000 }); + expect(run.checkpoint).toMatchObject({ ...waitingCheckpoint, terminal: null }); + expect(run.atRestart.durable).toContain("call-provider-finish"); + expect(run.appended.filter(event => event.turnId === "provider-1")).toEqual(run.appendedAfterRestart.filter(event => event.turnId === "provider-1")); + expect(providerTurnEvents(run.appendedAfterRestart)).toEqual([...settledProviderTurn("provider-1", startedWithResult("provider-1")), ...acceptedResult("provider-1")]); + expectSettledAfterRestart(run, ["provider-1"]); + expectNoTerminalPersistedWithoutResult(run); +}, 60_000); + +it.skipIf(process.platform === "win32")("R4 appends the rest of a provider turn whose finish call was open at a restart, from the checkpoint saved when the turn started", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal provider-turn-delay-ms=300", { at: "provider-turn", waitMs: 500, holdFirstToolCall: true }); + expect(run.checkpoint).toMatchObject(providerTurnCheckpoint); + expect(resolvedCalls(run.atRestart.journalled)).toEqual([]); + expect(providerTurnEvents(run.appendedAfterRestart)).toEqual([...settledProviderTurn("provider-1", [["run.result.proposed", "provider-1", null]]), ...acceptedResult("provider-1")]); + expectSettledAfterRestart(run, ["provider-1"]); + expectNoTerminalPersistedWithoutResult(run); + expect(answeredCalls(run)).toEqual([["provider-1", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-1", "call-provider-finish", null]]); +}, 60_000); + +it.skipIf(process.platform === "win32")("R5 appends both provider turns after a restart during the first, from the checkpoint saved when it started", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-turns=live/resolved provider-goal-at-terminal provider-turn-delay-ms=300 provider-turn-gap-ms=2500", { at: "provider-turn", waitMs: 3_500 }); + expect(run.checkpoint).toMatchObject(providerTurnCheckpoint); + expect(run.atRestart.journalled.filter(entry => entry.command === "session.provider_turn").map(entry => entry.params.turnId)).toEqual([expect.any(String), "provider-1"]); + expect(providerTurnEvents(run.appended)).toEqual([ + ...settledProviderTurn("provider-1", [["turn.started", "provider-1", "provider"]]), + ...settledProviderTurn("provider-2", startedWithResult("provider-2")), + ...acceptedResult("provider-2"), + ]); + expectSettledAfterRestart(run, ["provider-1", "provider-2"]); + expectNoTerminalPersistedWithoutResult(run); + expect(answeredCalls(run)).toEqual([["provider-2", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-2", "call-provider-finish", null]]); +}, 60_000); + +const providerTurnStarts = (run: ConsumerRun) => run.committed.filter(event => event.eventType === "turn.started" && event.payload.origin === "provider").map(event => event.turnId); +const storedSequence = (run: ConsumerRun, events: PrpEvent[]) => events.filter(event => event.sourceInstanceId === run.runnerInstanceId) + .map(event => event.sourceSeq).sort((left, right) => left - right); +const expectCheckpointAtStoredEvent = (run: ConsumerRun) => { + expect(run.queuedAtRestart.length).toBeGreaterThan(0); + expect(run.checkpoint.cursor).toBe(String(storedSequence(run, run.committedAtRestart).at(-1))); +}; +const expectNoHoleInCore = (run: ConsumerRun) => { + const sequence = storedSequence(run, run.committed); + expect(sequence).toEqual(sequence.map((_value, index) => index + 1)); +}; +const expectSettledOnce = (run: ConsumerRun) => { + expect(run.outcome, run.diagnostics.slice(-12).join("\n")).toBe("finalized"); + expect(run.completeRun).toHaveBeenCalledOnce(); + expect(run.committed.filter(event => event.eventType === "turn.completed" && event.turnId === "provider-1")).toHaveLength(1); + expect(run.closeOutcome).toBe("closed"); + expect(unsettled(run.diagnostics)).toEqual([]); + expect(run.durable).not.toContain("conflicting duplicate"); +}; +const providerStart = (event: PrpEvent) => event.eventType === "turn.started" && event.payload.origin === "provider"; +const awaitingTerminal = (event: PrpEvent) => event.eventType === "turn.completed" && event.payload.awaitingProviderTurn === true; + +it.skipIf(process.platform === "win32")("R6 saves the provider-start checkpoint at the start Core stored, so the recovered controller continues Core's sequence without a hole", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-goal-at-terminal provider-turn-delay-ms=300 provider-turn-until-snapshot", + { at: "provider-turn", waitMs: 1_000, replyResult: true, slowAppend: providerStart }); + expect(run.queuedAtRestart.map(event => [event.eventType, event.turnId])).toEqual([["item.delta", "provider-1"]]); + expectNoHoleInCore(run); + expectSettledOnce(run); + expect(run.checkpoint).toMatchObject({ activeTurnId: "provider-1", semanticResult: null }); + expectCheckpointAtStoredEvent(run); + expect(providerTurnStarts(run)).toEqual(["provider-1"]); +}, 60_000); + +it.skipIf(process.platform === "win32")("R7 saves the waiting checkpoint at the awaited terminal Core stored, so the provider turn's start reaches Core exactly once", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal", + { at: "wait", waitMs: 500, holdFirstToolCall: true, slowAppend: awaitingTerminal }); + expect(providerTurnStarts(run)).toEqual(["provider-1"]); + expect(run.checkpoint).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true, semanticResult: null }); + expectCheckpointAtStoredEvent(run); + expect(run.committed.filter(event => event.eventType === "run.result.proposed").map(event => event.turnId)).toEqual(["provider-1"]); + expectNoHoleInCore(run); + expectSettledOnce(run); + expect(answeredCalls(run)).toEqual([["provider-1", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-1", "call-provider-finish", null]]); +}, 60_000); + +it.skipIf(process.platform === "win32")("R8 characterization: after the old controller answered the provider turn's finish, recovery resumes at the stored start without a result, closes the turn in Core and ends on the missing-result path", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal provider-turn-delay-ms=300", + { at: "provider-turn", waitMs: 500, slowAppend: providerStart }); + expect(run.queuedAtRestart.map(event => event.eventType)).toEqual(["run.result.proposed"]); + expect(run.outcome).toContain("native_finalization_missing"); + expect(run.checkpoint).toMatchObject({ activeTurnId: "provider-1", semanticResult: null }); + expectCheckpointAtStoredEvent(run); + expect(providerTurnStarts(run)).toEqual(["provider-1"]); + expect(run.committed.filter(event => event.eventType === "run.result.proposed")).toEqual([]); + expect(run.committed.filter(event => event.eventType === "turn.completed" && event.turnId === "provider-1")).toHaveLength(1); + expectNoHoleInCore(run); + expect(answeredCalls(run)).toEqual([]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-1", "call-provider-finish", null]]); + expect(run.completeRun).not.toHaveBeenCalled(); + expect(run.durable).not.toContain("conflicting duplicate"); +}, 60_000); + +it.skipIf(process.platform === "win32")("R10 recovers from the checkpoint saved at the controller turn's start when the wait checkpoint was not saved, and admits the provider turn's start only after reconciling that turn", async () => { + const run = await runConsumerRestartScenario("live provider-turn provider-result-resolved provider-goal-at-terminal provider-turn-delay-ms=300", + { at: "wait", waitMs: 1_000, holdFirstToolCall: true, holdWaitCheckpoint: true }); + const controllerTurn = run.checkpoint.activeTurnId!; + expect(controllerTurn).toEqual(expect.any(String)); + expect(run.checkpoint).toMatchObject({ semanticResult: null }); + expect(run.heldCheckpoint).toMatchObject({ activeTurnId: null, awaitingProviderTurn: true }); + expect(run.committedAtRestart.filter(awaitingTerminal).map(event => event.turnId)).toEqual([controllerTurn]); + expect(acked(run as unknown as RestartRun, "turn.completed").filter(payload => payload.awaitingProviderTurn === true)).toHaveLength(1); + expect(run.atRestart.durable).toContain("call-provider-finish"); + expect(run.appended.filter(event => event.eventType === "session.failed")).toEqual([]); + expectSettledOnce(run); + expect(run.completeRun.mock.calls[0]![0]).toMatchObject({ turnId: "provider-1", result: { reportedWorkDisposition: "done" } }); + expect(providerTurnStarts(run)).toEqual(["provider-1"]); + expect(answeredCalls(run)).toEqual([["provider-1", "paperclip_finish", true]]); + expect(resolvedCalls(run.journalled)).toEqual([["provider-1", "call-provider-finish", null]]); +}, 60_000); + +it.skipIf(process.platform === "win32")("R9 characterization: an ordinary turn whose start checkpoint is ahead of Core settles after a restart, and Core lacks only the event the old controller did not store", async () => { + const run = await runConsumerRestartScenario("quiet", { at: "turn", waitMs: 500, holdFirstToolCall: true, sidecarArgs: " --wait-completion-feedback", optIn: false, + slowAppend: event => event.eventType === "turn.started" }); + const turnId = run.checkpoint.activeTurnId!; + const stored = (events: PrpEvent[]) => events.filter(event => event.sourceInstanceId === run.runnerInstanceId).map(event => event.sourceSeq); + const lost = run.queuedAtRestart.map(event => event.sourceSeq); + expect(run.queuedAtRestart.map(event => [event.eventType, event.turnId])).toEqual([["turn.accepted", turnId]]); + expect(run.checkpoint.cursor).toBe(String(lost[0])); + expect(stored(run.committedAtRestart).at(-1)).toBe(lost[0]! - 1); + expect(run.outcome, run.diagnostics.slice(-12).join("\n")).toBe("finalized"); + expect(run.recoveredPid).toBe(run.pid); + expect(run.completeRun).toHaveBeenCalledOnce(); + expect(run.completeRun.mock.calls[0]![0]).toMatchObject({ turnId, result: { reportedWorkDisposition: "done" } }); + expect(answeredCalls(run)).toEqual([[turnId, "paperclip_finish", true]]); + expect(run.appendedAfterRestart.filter(event => event.turnId === turnId && !event.eventType.startsWith("session.")).map(event => event.eventType)) + .toEqual(["run.result.proposed", "turn.completed", "run.result.accepted", "run.terminal"]); + const sequence = stored(run.committed); + expect(sequence).toEqual(Array.from({ length: sequence.at(-1)! }, (_value, index) => index + 1).filter(sourceSeq => !lost.includes(sourceSeq))); + expect(run.committed.filter(event => event.eventType === "turn.accepted")).toEqual([]); + expect(run.closeOutcome).toBe("closed"); + expect(unsettled(run.diagnostics)).toEqual([]); + expect(run.durable).not.toContain("conflicting duplicate"); +}, 60_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 5169606853d..bf44c8eaea4 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -7280,3 +7280,140 @@ describe("a Runner command that gets no answer", () => { .toBe("runner disconnected 125s; pending ahead 0; last runner event none"); }); }); + +const providerTurnStart = (turnId: string, origin: string | null = "provider") => ({ provider: "acpx", providerTurnId: turnId, status: "inProgress", + turn: { id: turnId, status: "inProgress" }, ...(origin ? { origin } : {}) }); +const providerTurnItem = (turnId: string) => ({ provider: "acpx", itemId: `message-${turnId}`, kind: "agentMessage", status: "completed", text: `${turnId} is done.` }); +const providerTurnTerminal = (turnId: string) => ({ provider: "acpx", providerTurnId: turnId, status: "completed", error: null }); +const optedInClaude = { provider: "acpx" as const, acpxAgent: "claude" as const, providerStartedTurns: true }; + +async function deliverProviderTurnEventsAfterRestart(input: { + options: Record; + resumeActiveTurnId?: string; + acknowledged: Array<[string, Record]>; + redelivered: Array<[string, Record]>; + removed?: number[]; + finishCall?: boolean; +}) { + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-provider-turn-start-")); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + let callbacks!: ConstructorParameters[0]; + let core!: DurablePrpControlPlane; + const coreSpy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) { + callbacks = options; + return core = new OriginalCore(options); + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, codexArgs: fakeCodexArgs(stateDirectory), stateDirectory, + ...(input.resumeActiveTurnId ? { resumeActiveTurnId: input.resumeActiveTurnId } : {}), + }); + const toolCalls: Array> = []; + bundle.transport.setServerRequestHandler(async request => { + toolCalls.push(structuredClone(request.params)); + return { success: true, contentItems: [{ type: "inputText" as const, text: "{\"done\":true}" }] }; + }); + const delivered: Array<{ method: string; params: Record }> = []; + try { + await bundle.transport.request("thread/start", { cwd: tmpdir() }); + const reader = bundle.transport.notifications()[Symbol.asyncIterator](); + void (async () => { for (let next = await reader.next(); !next.done; next = await reader.next()) delivered.push(next.value); })().catch(() => undefined); + await new Promise(done => setTimeout(done, 100)); + const before = delivered.length; + const transport = bundle.transport as unknown as { options: Record }; + transport.options = { ...transport.options, ...input.options }; + const identity = core.store.state.identity; + const window = core.store.state.committedEvents; + const envelope = (eventType: string, payload: Record) => + ({ runId: identity.runId, turnId: identity.turnId, payload: { eventType, turnId: identity.turnId, payload } }); + input.acknowledged.forEach(([eventType, payload], index) => { + const at = window.length - input.acknowledged.length + index; + window[at] = { ...window[at]!, eventType, envelope: envelope(eventType, payload) }; + }); + for (const at of [...input.removed ?? []].sort((left, right) => right - left)) window.splice(window.length - input.acknowledged.length + at, 1); + let seq = window.at(-1)!.sourceSeq; + const commit = (eventType: string, payload: Record) => { + seq += 1; + window.push({ sourceSeq: seq, sourceEventId: `provider-turn-start-${seq}`, eventType, priority: 1, + envelope: envelope(eventType, payload), deliveryCount: 1, logicalEffectCount: 1 }); + return `provider-turn-start-${seq}`; + }; + for (const [eventType, payload] of input.redelivered) commit(eventType, payload); + (core.store as unknown as { onChange: (() => void) | null }).onChange?.(); + if (input.finishCall) { + const sourceEventId = commit("tool.execution.started", { executionId: "exec-provider-finish", transport: "dynamic", name: "paperclip_finish", status: "running" }); + const outcome = await callbacks.onSemanticToolInput!({ callId: "exec-provider-finish", operationId: "paperclip_finish", + input: { summary: "Done" }, sourceEventId, sourceEventType: "semantic_tool.input", correlation: identity }); + expect(outcome.isError).not.toBe(true); + } + await new Promise(done => setTimeout(done, 200)); + return { notifications: delivered.slice(before).filter(notification => notification.method.startsWith("turn/") || notification.method.startsWith("item/")), toolCalls }; + } finally { + coreSpy.mockRestore(); + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +} + +const notificationTurns = (notifications: Array<{ method: string; params: Record }>) => + notifications.map(notification => [notification.method, notification.params.turnId, notification.params.origin ?? null]); + +it("gives the driver a provider turn's start before its next event when the start was acknowledged before a restart", async () => { + const { notifications } = await deliverProviderTurnEventsAfterRestart({ options: optedInClaude, + acknowledged: [["turn.started", providerTurnStart("provider-1")]], + redelivered: [["item.completed", providerTurnItem("provider-1")], ["turn.completed", providerTurnTerminal("provider-1")]] }); + expect(notificationTurns(notifications)).toEqual([["turn/started", "provider-1", "provider"], ["item/completed", "provider-1", null], ["turn/completed", "provider-1", null]]); + expect(notifications[0]!.params).toEqual(rehydrateRunnerdTurnNotification(providerTurnStart("provider-1"), notifications[1]!.params.threadId, "provider-1", "turn/started")); +}, 20_000); + +it("gives the driver a provider turn's start before its next event when the window removed events after the start", async () => { + const { notifications } = await deliverProviderTurnEventsAfterRestart({ options: optedInClaude, + acknowledged: [["turn.started", providerTurnStart("provider-1")], ["item.completed", providerTurnItem("provider-1")], ["item.completed", providerTurnItem("provider-1")]], + removed: [1], + redelivered: [["item.completed", providerTurnItem("provider-1")], ["turn.completed", providerTurnTerminal("provider-1")]] }); + expect(notificationTurns(notifications)).toEqual([["turn/started", "provider-1", "provider"], ["item/completed", "provider-1", null], ["turn/completed", "provider-1", null]]); +}, 20_000); + +it("labels each provider turn's events with that turn when the recovery read names a later turn", async () => { + const { notifications } = await deliverProviderTurnEventsAfterRestart({ options: optedInClaude, resumeActiveTurnId: "provider-2", + acknowledged: [["turn.started", providerTurnStart("provider-1")]], + redelivered: [["item.completed", providerTurnItem("provider-1")], ["turn.completed", providerTurnTerminal("provider-1")], + ["turn.started", providerTurnStart("provider-2")], ["item.completed", providerTurnItem("provider-2")]] }); + expect(notificationTurns(notifications)).toEqual([["turn/started", "provider-1", "provider"], ["item/completed", "provider-1", null], + ["turn/completed", "provider-1", null], ["turn/started", "provider-2", "provider"], ["item/completed", "provider-2", null]]); +}, 20_000); + +it("gives no second start to the provider turn that the recovered driver already has as its active turn", async () => { + const { notifications } = await deliverProviderTurnEventsAfterRestart({ options: optedInClaude, resumeActiveTurnId: "provider-1", + acknowledged: [["turn.started", providerTurnStart("provider-1")]], + redelivered: [["item.completed", providerTurnItem("provider-1")], ["turn.completed", providerTurnTerminal("provider-1")]] }); + expect(notificationTurns(notifications)).toEqual([["item/completed", "provider-1", null], ["turn/completed", "provider-1", null]]); +}, 20_000); + +it("gives no second start to a provider turn whose start is delivered again", async () => { + const { notifications } = await deliverProviderTurnEventsAfterRestart({ options: optedInClaude, acknowledged: [], + redelivered: [["turn.started", providerTurnStart("provider-1")], ["item.completed", providerTurnItem("provider-1")], ["turn.completed", providerTurnTerminal("provider-1")]] }); + expect(notificationTurns(notifications)).toEqual([["turn/started", "provider-1", "provider"], ["item/completed", "provider-1", null], ["turn/completed", "provider-1", null]]); +}, 20_000); + +it.each([ + ["not opted in", { provider: "acpx", acpxAgent: "claude" }, "provider"], + ["a non-Claude agent", { provider: "acpx", acpxAgent: "codex", providerStartedTurns: true }, "provider"], + ["another provider", { providerStartedTurns: true }, "provider"], + ["a controller-started turn", optedInClaude, null], +] as const)("delivers a restarted turn's events as before for %s", async (_case, options, origin) => { + const { notifications } = await deliverProviderTurnEventsAfterRestart({ options, resumeActiveTurnId: "controller-turn", + acknowledged: [["turn.started", providerTurnStart("provider-1", origin)]], + redelivered: [["item.completed", providerTurnItem("provider-1")], ["turn.completed", providerTurnTerminal("provider-1")]] }); + const threadId = notifications[0]!.params.threadId; + expect(notifications).toEqual([ + { method: "item/completed", params: rehydrateRunnerdItemNotification(providerTurnItem("provider-1"), threadId, "controller-turn") }, + { method: "turn/completed", params: rehydrateRunnerdTurnNotification(providerTurnTerminal("provider-1"), threadId, "controller-turn", "turn/completed") }, + ]); +}, 20_000); + +it("binds the finish call of a provider turn whose start was acknowledged before a restart to that turn", async () => { + const { notifications, toolCalls } = await deliverProviderTurnEventsAfterRestart({ options: optedInClaude, resumeActiveTurnId: "provider-2", + acknowledged: [["turn.started", providerTurnStart("provider-1")]], redelivered: [], finishCall: true }); + expect(notificationTurns(notifications)).toEqual([["turn/started", "provider-1", "provider"]]); + expect(toolCalls).toEqual([expect.objectContaining({ turnId: "provider-1", tool: "paperclip_finish", callId: "exec-provider-finish" })]); +}, 20_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index d42325ca874..a9e56ff8f2e 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1,4 +1,4 @@ -import { providerTurnToolCallSourceSeq } from "./provider-turn-tool-order.js"; +import { providerTurnStartBefore, providerTurnToolCallSourceSeq } from "./provider-turn-tool-order.js"; import type { RunnerAttachmentDownloadGrant } from "../contracts/harness-driver.js"; import { readProviderFailure } from "../contracts/provider-failure.generated.js"; import { confirmAntigravityProbeNativeRetirement } from "../drivers/acpx/antigravity-probe-retirement.js"; @@ -4219,6 +4219,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } | null = null; #checkpointProviderIdentityConfirmed = false; #turnId = ""; + readonly #deliveredProviderTurnStarts = new Set(); #turnStartResponsePending = false; #turnStartResponseSettled: Promise = Promise.resolve(); #turnStartResponseEpoch = 0; @@ -4388,6 +4389,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { void this.#failureSignal.catch(() => undefined); this.#ownsRoot = options.stateDirectory === undefined; this.#turnId = options.resumeActiveTurnId ?? ""; + if (options.resumeActiveTurnId) this.#deliveredProviderTurnStarts.add(options.resumeActiveTurnId); this.#root = options.stateDirectory ?? mkdtempSync(resolve(tmpdir(), "paperclip-runner-lab-prp-")); @@ -7794,6 +7796,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#pumpEvents(); await new Promise(resolveWait => setTimeout(resolveWait, 10)); } + this.#deliverProviderTurnStart(this.#core?.store.state.committedEvents ?? [], sourceSeq); let delivered!: () => void; const barrier = new Promise(resolveBarrier => { delivered = resolveBarrier; }); this.#queue.pushControl({ id: `provider-turn-order:${sourceSeq}`, method: "paperclip/providerTurnOrder", params: {} }, () => delivered()); @@ -7804,6 +7807,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ]).catch(() => { throw new SemanticToolNotDispatchedError(); }); } + #deliverProviderTurnStart(events: readonly DurableRecoveryCommittedEvent[], sourceSeq: number) { + const start = providerTurnStartBefore(events, sourceSeq); + if (start === null) return; + this.#turnId = start.turnId; + if (this.#deliveredProviderTurnStarts.has(start.turnId)) return; + this.#deliveredProviderTurnStarts.add(start.turnId); + this.#queue.push({ method: "turn/started", params: rehydrateRunnerdTurnNotification(start.payload, this.#threadId, start.turnId, "turn/started") }); + } + #bindSemanticToolOutcome(call: { operationId: string; callId: string }, outcome: { result: unknown; isError?: boolean }) { if (call.operationId === "call_api") { const result = record(outcome.result); @@ -8468,6 +8480,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#applyProviderIdentityEvent(event); continue; } + if (this.#providerStartedTurns() && event.eventType !== "turn.started") this.#deliverProviderTurnStart(events, event.sourceSeq); if (event.eventType === "session.capabilities.updated" && this.options.provider === "acpx") { const capabilities = record(record(event.envelope.payload).payload); if (capabilities.turnControls !== undefined) { @@ -8702,6 +8715,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } if (typeof providerTurnId === "string" && providerTurnId.length > 0) { this.#turnId = providerTurnId; + this.#deliveredProviderTurnStarts.add(providerTurnId); if (this.#turnStartResponsePending) { this.#observedTurnStartEpoch = this.#turnStartResponseEpoch; } diff --git a/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts b/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts index 79f14a7ffdf..7a46646bc0c 100644 --- a/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.provider-turn.test.ts @@ -107,6 +107,8 @@ function harness(options: { withoutInterrupt?: boolean; restart?: RestartContinu const queue: Array = []; let wake: (() => void) | null = null; let activeProviderTurn: string | null = null; + const observed = { activeTurnId: null as string | null, awaiting: false }; + const snapshotSequences: Array = []; const push = (...values: Array) => { for (const value of values) { if (value !== "end" && value.eventType === "turn.started") activeProviderTurn = value.turnId ?? null; @@ -138,7 +140,12 @@ function harness(options: { withoutInterrupt?: boolean; restart?: RestartContinu for (;;) { const next = queue.shift(); if (next === "end") return; - if (next) { yield structuredClone(next); continue; } + if (next) { + if (next.eventType === "turn.started") Object.assign(observed, { activeTurnId: next.turnId ?? null, awaiting: false }); + else if (next.eventType.startsWith("turn.")) Object.assign(observed, { activeTurnId: null, awaiting: next.payload.awaitingProviderTurn === true }); + yield structuredClone(next); + continue; + } await new Promise(resolve => { wake = resolve; }); wake = null; } @@ -146,10 +153,11 @@ function harness(options: { withoutInterrupt?: boolean; restart?: RestartContinu startTurn, ...(options.withoutInterrupt ? {} : { interrupt }), async result() { return null; }, - async snapshot() { + async snapshot(snapshotOptions?: { sourceSeq?: number }) { + snapshotSequences.push(snapshotOptions?.sourceSeq); const base: PersistedNativeSession = { backendKind: "mock", sessionId: identity.sessionId, identity, providerSessionId: "provider-session", cursor: String(sequence), activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; - if (!options.restart) return base; + if (!options.restart) return { ...base, activeTurnId: observed.activeTurnId, ...(observed.awaiting ? { awaitingProviderTurn: true } : {}) }; if (options.restart.recovery) return { ...base, ...options.restart.recovery.snapshot }; const starts = startTurn.mock.calls.length; return { ...base, semanticResult: null, @@ -159,7 +167,7 @@ function harness(options: { withoutInterrupt?: boolean; restart?: RestartContinu }, async close() { push("end"); }, }; - return { session, push, interrupt, startTurn }; + return { session, push, interrupt, startTurn, snapshotSequences }; } async function run(options: { @@ -175,6 +183,7 @@ async function run(options: { const control = harness({ withoutInterrupt: options.withoutInterrupt, restart: options.restart }); const appended: PrpEvent[] = []; const checkpoints: PersistedNativeSession[] = []; + const checkpointedAfter: number[] = []; let queued: Array<() => void> = []; const flushStreamingEvents = vi.fn(() => { const flushed = queued; @@ -194,7 +203,7 @@ async function run(options: { ...(options.restart ? { async recoverSession() { return { recovered: true, session: control.session }; } } : {}) }, controlPlane: { ...(options.pipelinesDeltas ? { pipelinesStreamingEvents: true, flushStreamingEvents } : {}), - async openRun() {}, async checkpointSession(value: PersistedNativeSession) { checkpoints.push(structuredClone(value)); }, + async openRun() {}, async checkpointSession(value: PersistedNativeSession) { checkpoints.push(structuredClone(value)); checkpointedAfter.push(appended.length); }, async appendEvent(value: PrpEvent) { appended.push(structuredClone(value)); if (options.pipelinesDeltas && value.eventType === "item.delta") await new Promise(resolve => queued.push(resolve)); @@ -216,7 +225,7 @@ async function run(options: { await options.after?.(control, appended); const outcome = await execution; return { outcome, returned, appended, completeRun, resolveMissingResult, interrupt: control.interrupt, flushStreamingEvents, - startTurn: control.startTurn, checkpoints, elapsedMs: Date.now() - started }; + startTurn: control.startTurn, checkpoints, checkpointedAfter, snapshotSequences: control.snapshotSequences, elapsedMs: Date.now() - started }; } const types = (events: PrpEvent[]) => events.map(value => value.eventType); @@ -231,6 +240,48 @@ describe("the controller consumer with provider-started turn sequences", () => { expect(outcome.interrupt).not.toHaveBeenCalled(); }); + const ordinaryTurn = () => [ + event("turn-1", "turn.started", { status: "inProgress" }), + event("turn-1", "item.completed", usage), + event("turn-1", "run.result.proposed", result as unknown as Record), + event("turn-1", "turn.completed", { status: "completed", error: null }), + ]; + const checkpointsAfter = (outcome: Awaited>, matches: (value: PrpEvent) => boolean) => { + const index = outcome.appended.findIndex(matches); + return outcome.checkpoints.filter((_value, at) => outcome.checkpointedAfter[at] === index + 1); + }; + + it("persists a checkpoint with the waiting shape right after the awaiting terminal", async () => { + const outcome = await run({ stream: turnOne(true), providerTurnWait: { capMs: 100 } }); + expect(checkpointsAfter(outcome, value => value.payload.awaitingProviderTurn === true)) + .toEqual([expect.objectContaining({ activeTurnId: null, awaitingProviderTurn: true })]); + }); + + it("persists a checkpoint naming a provider-started turn active right after its start", async () => { + const outcome = await run({ stream: [...turnOne(true), ...providerTurn("provider-1", { awaiting: true }), ...providerTurn("provider-2", { result: true })] }); + expect(outcome.outcome).toBe("finalized"); + for (const turnId of ["provider-1", "provider-2"]) { + expect(checkpointsAfter(outcome, value => value.turnId === turnId && value.eventType === "turn.started")) + .toEqual([expect.objectContaining({ activeTurnId: turnId })]); + } + }); + + it("asks for the snapshot as of the awaiting terminal and the provider start it has just appended", async () => { + const outcome = await run({ stream: [...turnOne(true), ...providerTurn("provider-1", { result: true })] }); + expect(outcome.outcome).toBe("finalized"); + const awaited = outcome.appended.find(value => value.payload.awaitingProviderTurn === true)!; + const started = outcome.appended.find(value => value.payload.origin === "provider")!; + expect(outcome.snapshotSequences.filter(value => value !== undefined)).toEqual([awaited.sourceSeq, started.sourceSeq]); + }); + + it("persists no checkpoint of either kind for a turn that does not await", async () => { + const outcome = await run({ stream: ordinaryTurn() }); + expect(outcome.outcome).toBe("finalized"); + const checkpointsWithoutProviderTurns = 4; + expect(outcome.checkpoints).toHaveLength(checkpointsWithoutProviderTurns); + expect(outcome.checkpoints.filter(value => value.awaitingProviderTurn || value.activeTurnId?.startsWith("provider-"))).toEqual([]); + }); + it("consumes the provider-started turn as the next turn of the same run and settles its result normally", async () => { const outcome = await run({ stream: [...turnOne(true), ...providerTurn("provider-1", { result: true })] }); expect(outcome.outcome).toBe("finalized"); @@ -595,6 +646,41 @@ describe("the controller consumer with provider-started turn sequences", () => { expect(second.returned).toMatchObject({ result, turnId: "provider-1" }); }); + it.each([true, false])("resumes a restored wait whose disposition marker is consumed instead of settling on that turn's earlier terminal (provider turn arrives: %s)", async (arrives) => { + const awaitedTurn = [ + event("turn-disposition", "turn.started", { status: "inProgress" }), + event("turn-disposition", "item.completed", usage), + event("turn-disposition", "turn.completed", { status: "completed", error: null, awaitingProviderTurn: true }), + ]; + const terminalTurns = [completedTerminal("turn-1"), completedTerminal("turn-disposition")]; + const waiting = { terminalTurns, semanticResult: null, awaitingProviderTurn: true, dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-disposition" }; + const recovery = { checkpoint: { ...restartCheckpoint, ...waiting }, snapshot: waiting, replay: [...turnOne(false), ...awaitedTurn] }; + const turn = providerTurn("provider-1", { result: true }); + const outcome = await run({ + stream: [], restart: { recovery }, turnTimeoutMs: 10_000, + providerTurnWait: { startedAtMs: Date.now() - (arrives ? 599_000 : 599_700) }, + after: async (control) => { + if (!arrives) return; + await new Promise(resolve => setTimeout(resolve, 200)); + control.push(...turn); + }, + }); + expect(outcome.startTurn).not.toHaveBeenCalled(); + if (arrives) { + expect(outcome.outcome).toBe("finalized"); + expect(outcome.returned).toMatchObject({ result, turnId: "provider-1" }); + expect(outcome.resolveMissingResult).not.toHaveBeenCalled(); + expect(outcome.interrupt).not.toHaveBeenCalled(); + expect(appendedOf(outcome.appended, turn)).toEqual(ids(turn)); + } else { + expect(outcome.outcome).toContain("native_finalization_missing"); + expect(outcome.interrupt).toHaveBeenCalledOnce(); + expect(outcome.resolveMissingResult).toHaveBeenCalledOnce(); + expect(outcome.resolveMissingResult).toHaveBeenCalledWith(expect.objectContaining({ turnId: "turn-disposition", + terminalEvent: expect.objectContaining({ eventType: "turn.completed", payload: expect.objectContaining({ awaitingProviderTurn: true }) }) })); + } + }); + it("fails the run as today when the Runner's stream ends during the wait", async () => { const outcome = await run({ stream: turnOne(true), providerTurnWait: { capMs: 60_000 }, diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 73f8aaf9ac0..218f2ab52dc 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -1193,6 +1193,7 @@ async function consumeTurn( supersededTerminal?: (event: PrpEvent, signal: AbortSignal) => Promise, providerTurnWaitOptions?: ProviderTurnWaitOptions, resumedProviderTurnWait?: PrpEvent, + checkpointProviderTurn?: (signal: AbortSignal, sourceSeq: number) => Promise, ) { let timer: ReturnType | undefined; const providerTurnWait = createProviderTurnWait({ @@ -1475,6 +1476,10 @@ async function consumeTurn( if (ended !== null) return ended; } providerTurnWait.observe(event); + if (event.eventType === "turn.started" && payload.origin === "provider") { + await checkpointProviderTurn?.(appendAbort.signal, event.sourceSeq); + appendAbort.signal.throwIfAborted(); + } const eventGoal = goalFromEvent(event); const goalChanged = eventGoal !== undefined && goalLifecycleFingerprint(eventGoal) !== goalLifecycleFingerprint(previousGoal); @@ -1696,7 +1701,11 @@ async function consumeTurn( }; } if (semanticResultProposal === null && governedResult === null && inputTimers.size === 0 && - awaitsProviderTurn(event) && providerTurnWait.begin(event)) continue; + awaitsProviderTurn(event) && providerTurnWait.begin(event)) { + await checkpointProviderTurn?.(appendAbort.signal, event.sourceSeq); + appendAbort.signal.throwIfAborted(); + continue; + } if (semanticResultProposal === null && governedResult === null && inputTimers.size === 0 && await resumeInterruptedTurn?.(event, appendAbort.signal)) continue; await settlePendingDeltas(); @@ -2933,8 +2942,8 @@ export async function executeNativeSession( if (reconciledRecoveryCheckpoint !== null) { await persistCheckpoint(reconciledRecoveryCheckpoint); } - const checkpoint = async (signal?: AbortSignal) => { - const snapshot = await session.snapshot(signal ? { signal } : undefined); + const checkpoint = async (signal?: AbortSignal, sourceSeq?: number) => { + const snapshot = await session.snapshot(signal ? { signal, ...(sourceSeq === undefined ? {} : { sourceSeq }) } : undefined); signal?.throwIfAborted(); await persistCheckpoint(snapshot, signal); return snapshot; @@ -3151,7 +3160,8 @@ export async function executeNativeSession( recovered && persistedSession?.dispositionOnlyRecoveryConsumed && !recoveredSnapshot.semanticResult && - !recoveredActiveTurnId, + !recoveredActiveTurnId && + recoveredSnapshot.awaitingProviderTurn !== true, ); const dispositionRecoveryTurnId = persistedSession?.dispositionOnlyRecoveryTurnId ?? @@ -3251,6 +3261,7 @@ export async function executeNativeSession( supersededByRestartContinuation, options.providerTurnWait, resumedProviderTurnWait, + checkpoint, ) : Promise.resolve({ event: recoveryTerminal,