diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 33f017f8e0f..ac6562d55d6 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -219,6 +219,30 @@ describe("HarnessDriverBackend", () => { expect(recover.mock.calls[0]![0].codexUsageBaseline!.latest.inputTokens).toBe(150); }); + it.each([true, false])("carries the provider-turn wait marker through a serialized native checkpoint only while waiting (%s)", async (awaiting) => { + class WaitSession extends FakeHarnessSession { + override async snapshot(): Promise { + return { ...(await super.snapshot()), semanticResult: undefined, activeTurnId: null, + ...(awaiting ? { awaitingProviderTurn: true } : {}) }; + } + } + const original = new HarnessDriverBackend({ ...driver, openSession: async () => new WaitSession() }); + const session = await original.openSession({ identity: { + runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1", + } }); + const checkpoint = JSON.parse(JSON.stringify(await session.snapshot())); + const recover = vi.fn(async (_snapshot: PersistedHarnessSession) => ({ recovered: true, session: new WaitSession() })); + await new HarnessDriverBackend({ ...driver, recoverSession: recover }) + .recoverSession(checkpoint, { signal: new AbortController().signal }); + if (awaiting) { + expect(checkpoint.awaitingProviderTurn).toBe(true); + expect(recover.mock.calls[0]![0].awaitingProviderTurn).toBe(true); + } else { + expect(checkpoint).not.toHaveProperty("awaitingProviderTurn"); + expect(recover.mock.calls[0]![0]).not.toHaveProperty("awaitingProviderTurn"); + } + }); + it("rejects and closes a provider session without a durable provider identity", async () => { let closed = false; class MissingProviderIdentitySession extends FakeHarnessSession { diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index e3922fb36b9..b45e0b9bf50 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -153,6 +153,7 @@ export class HarnessDriverBackend implements NativeSessionBackend { pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], goal: snapshot.goal ?? null, lineage: snapshot.lineage ?? [], + ...(snapshot.awaitingProviderTurn === true ? { awaitingProviderTurn: true } : {}), }; const recoveryOptions: HarnessSessionRecoveryOptions = { signal: options.signal, @@ -918,6 +919,7 @@ class HarnessNativeSession implements NativeSession { pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], goal: snapshot.goal ?? null, lineage: snapshot.lineage ?? [], + ...(snapshot.awaitingProviderTurn === true ? { awaitingProviderTurn: true } : {}), }; } diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index 14b3aae458f..37091f4042d 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -505,6 +505,7 @@ export interface PersistedHarnessSession { | "same_session_only" | "allow_replacement_after_governed_wait" | "allow_replacement_after_resume_failure"; + awaitingProviderTurn?: boolean; } /** Runner proof that saved provider state cannot be restored by this runner diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index fcd43490e2f..e8ce03c19dc 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -80,6 +80,7 @@ export interface PersistedNativeSession { pendingRuntimeRequests?: HarnessRuntimeRequest[]; goal?: HarnessThreadGoal | null; lineage?: HarnessThreadLineageEntry[]; + awaitingProviderTurn?: boolean; } export interface NativeSessionRecoveryResult { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index f5fa26a6a28..83feb7983a0 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -771,6 +771,7 @@ export class CodexAppServerDriver implements HarnessDriver { codexUsageBaseline: snapshot.codexUsageBaseline, dispositionOnlyRecoveryConsumed, dispositionOnlyRecoveryTurnId, + awaitingProviderTurn: snapshot.awaitingProviderTurn, adoptedAcceptedTurnId, lostAcceptedTurnId, stalePendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], @@ -1108,6 +1109,7 @@ export class CodexAppServerDriver implements HarnessDriver { codexUsageBaseline?: PersistedHarnessSession["codexUsageBaseline"]; dispositionOnlyRecoveryConsumed?: boolean; dispositionOnlyRecoveryTurnId?: string | null; + awaitingProviderTurn?: boolean; adoptedAcceptedTurnId?: string | null; lostAcceptedTurnId?: string | null; stalePendingRuntimeRequests?: HarnessRuntimeRequest[]; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts new file mode 100644 index 00000000000..ad30df81663 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.provider-turn.test.ts @@ -0,0 +1,232 @@ +import { + FakeCodexTransport, + HarnessOperationAlreadyTerminalError, + HarnessStaleTurnError, + WORKSPACE, + collectUntilTerminal, + describe, + expect, + it, + makeDriver, + result, +} from "./codex-app-server-driver.test-support.js"; +import { RUNNERD_RUN_TERMINAL } from "./codex-driver-values.js"; + +async function completedFirstTurn(awaiting: boolean) { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Start a background install." } }); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, + ...(awaiting ? { awaitingProviderTurn: true } : {}) }); + const events = session.events()[Symbol.asyncIterator](); + const first = await collectUntilTerminal({ [Symbol.asyncIterator]: () => events }); + return { transport, session, events, first }; +} + +async function restoredAfterFirstTurn(awaiting: boolean) { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Start a background install." } }); + first.push("turn/started", { threadId: "thread-1", turn: { id: "turn-1", status: "inProgress" } }); + first.push("turn/completed", { threadId: "thread-1", turn: { id: "turn-1", status: "completed", items: [] }, + ...(awaiting ? { awaitingProviderTurn: true } : {}) }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ reason: "controller restarted" }); + second.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [] }] } }; + const recovery = await driver.recoverSession!(checkpoint); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + return { transport: second, session, checkpoint, events: session.events()[Symbol.asyncIterator]() }; +} + +async function reconciledActiveTurn(awaiting: boolean) { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Start a background install." } }); + const checkpoint = await original.snapshot(); + expect(checkpoint.activeTurnId).toBe("turn-1"); + await original.close({ reason: "controller restarted" }); + second.readResponse = { thread: { id: "thread-1", sessionId: "provider-session-1", cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [], ...(awaiting ? { awaitingProviderTurn: true } : {}) }] } }; + const recovery = await driver.recoverSession!(checkpoint); + expect(recovery).toMatchObject({ recovered: true }); + const session = recovery.session!; + return { transport: second, session, events: session.events()[Symbol.asyncIterator]() }; +} + +async function nextUntil(events: AsyncIterator<{ eventType: string; payload: Record }>, done: (eventType: string) => boolean) { + const collected: Array<{ eventType: string; payload: Record; turnId?: string }> = []; + for (;;) { + const next = await events.next(); + if (next.done) return collected; + collected.push(next.value); + if (done(next.value.eventType)) return collected; + } +} + +describe("Codex app-server driver with provider-started turns", () => { + it("marks an awaited terminal and adopts the provider's own next turn with its result", async () => { + const { transport, session, events, first } = await completedFirstTurn(true); + expect(first.at(-1)).toMatchObject({ eventType: "turn.completed", payload: { status: "completed", awaitingProviderTurn: true } }); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + transport.push("paperclip/runResult", { threadId: "thread-1", turnId: "provider-1", result }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "provider-1", status: "completed", items: [] } }); + const second = await nextUntil(events, eventType => eventType === "turn.completed"); + expect(second.some(event => event.eventType === "session.failed")).toBe(false); + expect(second.find(event => event.eventType === "turn.started")).toMatchObject({ turnId: "provider-1", payload: { origin: "provider" } }); + expect(second.at(-1)).toMatchObject({ eventType: "turn.completed", turnId: "provider-1" }); + expect(second.at(-1)!.payload).not.toHaveProperty("awaitingProviderTurn"); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); + + it("still fails closed on a provider-started turn the Runner did not await", async () => { + const { transport, session, events, first } = await completedFirstTurn(false); + expect(first.at(-1)!.payload).not.toHaveProperty("awaitingProviderTurn"); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const failed = await nextUntil(events, eventType => eventType === "session.failed"); + expect(failed.at(-1)).toMatchObject({ eventType: "session.failed", payload: { code: "turn_binding_mismatch" } }); + await session.close({ reason: "fixture complete" }).catch(() => undefined); + }); + + it("acknowledges an interrupt during the wait without sending one to the provider", async () => { + const { transport, session, events } = await completedFirstTurn(true); + await session.interrupt!({ reason: "Stop" }); + const acknowledged = await nextUntil(events, eventType => eventType === "item.completed"); + expect(acknowledged.at(-1)).toMatchObject({ payload: { kind: "interrupt_acknowledgement", status: "provider_turn_wait_ended" } }); + expect(transport.calls.some(request => request.method === "turn/interrupt")).toBe(false); + await session.close({ reason: "fixture complete" }); + }); + + it("never interrupts a running provider-started turn for the wait's own end", async () => { + const { transport, session, events } = await completedFirstTurn(true); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + await nextUntil(events, eventType => eventType === "turn.started"); + await expect(session.interrupt!({ turnId: "turn-1", reason: "cap" })).rejects.toBeInstanceOf(HarnessStaleTurnError); + expect(transport.calls.some(request => request.method === "turn/interrupt")).toBe(false); + await session.close({ reason: "fixture complete" }); + }); + + it("ends the wait on the Runner's run.terminal only while awaiting with no turn, and ignores a forged one", async () => { + const forged = await completedFirstTurn(true); + forged.transport.push("paperclip/runTerminal", { threadId: "thread-1" }); + forged.transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const untouched = await nextUntil(forged.events, eventType => eventType === "turn.started"); + expect(untouched.some(event => event.payload.kind === "interrupt_acknowledgement")).toBe(false); + await forged.session.close({ reason: "fixture complete" }); + + const awaiting = await completedFirstTurn(true); + awaiting.transport.push("paperclip/runTerminal", { [RUNNERD_RUN_TERMINAL]: true, threadId: "thread-1" }); + const ended = await nextUntil(awaiting.events, eventType => eventType === "item.completed"); + expect(ended.filter(event => event.payload.kind === "interrupt_acknowledgement")).toHaveLength(1); + expect(ended.at(-1)).toMatchObject({ payload: { kind: "interrupt_acknowledgement", status: "provider_turn_wait_ended", cause: "run_terminal" } }); + expect(await awaiting.session.snapshot()).not.toHaveProperty("awaitingProviderTurn"); + awaiting.transport.push("paperclip/runTerminal", { [RUNNERD_RUN_TERMINAL]: true, threadId: "thread-1" }); + awaiting.transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const afterSecond = await nextUntil(awaiting.events, eventType => eventType === "session.failed"); + expect(afterSecond.some(event => event.payload.kind === "interrupt_acknowledgement")).toBe(false); + await awaiting.session.close({ reason: "fixture complete" }).catch(() => undefined); + + const ordinary = await completedFirstTurn(false); + ordinary.transport.push("paperclip/runTerminal", { [RUNNERD_RUN_TERMINAL]: true, threadId: "thread-1" }); + ordinary.transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const after = await nextUntil(ordinary.events, eventType => eventType === "session.failed"); + expect(after.some(event => event.payload.kind === "interrupt_acknowledgement")).toBe(false); + await ordinary.session.close({ reason: "fixture complete" }).catch(() => undefined); + }); + + it("keeps the wait across a controller restart and adopts the provider's own next turn", async () => { + const { transport, session, checkpoint, events } = await restoredAfterFirstTurn(true); + expect(checkpoint).toMatchObject({ activeTurnId: null, semanticResult: null, awaitingProviderTurn: true }); + expect(await session.snapshot()).toMatchObject({ awaitingProviderTurn: true }); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + transport.push("paperclip/runResult", { threadId: "thread-1", turnId: "provider-1", result }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "provider-1", status: "completed", items: [] } }); + const adopted = await nextUntil(events, eventType => eventType === "turn.completed" || eventType === "session.failed"); + expect(adopted.some(event => event.eventType === "session.failed")).toBe(false); + expect(adopted.find(event => event.eventType === "turn.started")).toMatchObject({ turnId: "provider-1", payload: { origin: "provider" } }); + expect(adopted.at(-1)).toMatchObject({ eventType: "turn.completed", turnId: "provider-1" }); + const settled = await session.snapshot(); + expect(settled.semanticResult).toMatchObject({ result, turnId: "provider-1" }); + expect(settled).not.toHaveProperty("awaitingProviderTurn"); + await session.close({ reason: "fixture complete" }); + }); + + it("marks a persisted active turn that a restart reconciles as awaiting, then adopts the provider's own next turn", async () => { + const { transport, session, events } = await reconciledActiveTurn(true); + const reconciled = await nextUntil(events, eventType => eventType === "turn.completed"); + expect(reconciled.at(-1)).toMatchObject({ eventType: "turn.completed", turnId: "turn-1", + payload: { status: "completed", awaitingProviderTurn: true } }); + expect(await session.snapshot()).toMatchObject({ activeTurnId: null, semanticResult: null, awaitingProviderTurn: true }); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + transport.push("paperclip/runResult", { threadId: "thread-1", turnId: "provider-1", result }); + transport.push("turn/completed", { threadId: "thread-1", turn: { id: "provider-1", status: "completed", items: [] } }); + const adopted = await nextUntil(events, eventType => eventType === "turn.completed" || eventType === "session.failed"); + expect(adopted.some(event => event.eventType === "session.failed")).toBe(false); + expect(adopted.find(event => event.eventType === "turn.started")).toMatchObject({ turnId: "provider-1", payload: { origin: "provider" } }); + expect(adopted.at(-1)).toMatchObject({ eventType: "turn.completed", turnId: "provider-1" }); + expect((await session.snapshot()).semanticResult).toMatchObject({ result, turnId: "provider-1" }); + await session.close({ reason: "fixture complete" }); + }); + + it("reconciles an unmarked persisted active turn as an ordinary terminal", async () => { + const { transport, session, events } = await reconciledActiveTurn(false); + const reconciled = await nextUntil(events, eventType => eventType === "turn.completed"); + expect(reconciled.at(-1)).toMatchObject({ eventType: "turn.completed", turnId: "turn-1" }); + expect(reconciled.at(-1)!.payload).not.toHaveProperty("awaitingProviderTurn"); + expect(await session.snapshot()).not.toHaveProperty("awaitingProviderTurn"); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const failed = await nextUntil(events, eventType => eventType === "session.failed"); + expect(failed.at(-1)).toMatchObject({ eventType: "session.failed", payload: { code: "turn_binding_mismatch" } }); + await session.close({ reason: "fixture complete" }).catch(() => undefined); + }); + + it("writes no wait marker in a snapshot taken outside a wait", async () => { + const ordinary = await completedFirstTurn(false); + expect(await ordinary.session.snapshot()).not.toHaveProperty("awaitingProviderTurn"); + await ordinary.session.close({ reason: "fixture complete" }); + const opened = await makeDriver([new FakeCodexTransport()]).openSession({ + runId: "provider-turn", normalizedSessionId: "provider-turn-session", workingDirectory: WORKSPACE, + }); + expect(await opened.snapshot()).not.toHaveProperty("awaitingProviderTurn"); + await opened.close({ reason: "fixture complete" }); + }); + + it("still fails closed on a provider-started turn after a restore with no wait", async () => { + const { transport, session, checkpoint, events } = await restoredAfterFirstTurn(false); + expect(checkpoint).not.toHaveProperty("awaitingProviderTurn"); + transport.push("turn/started", { threadId: "thread-1", turn: { id: "provider-1", status: "inProgress" }, origin: "provider" }); + const failed = await nextUntil(events, eventType => eventType === "session.failed"); + expect(failed.at(-1)).toMatchObject({ eventType: "session.failed", payload: { code: "turn_binding_mismatch" } }); + await session.close({ reason: "fixture complete" }).catch(() => undefined); + }); + + it("acknowledges an interrupt during a restored wait without sending one to the provider", async () => { + const { transport, session, events } = await restoredAfterFirstTurn(true); + await session.interrupt!({ reason: "Stop" }); + let acknowledged = await events.next(); + while (!acknowledged.done && acknowledged.value.payload.kind !== "interrupt_acknowledgement") acknowledged = await events.next(); + expect(acknowledged.value).toMatchObject({ payload: { kind: "interrupt_acknowledgement", status: "provider_turn_wait_ended" } }); + expect(transport.calls.some(request => request.method === "turn/interrupt")).toBe(false); + await session.close({ reason: "fixture complete" }); + }); + + it("keeps rejecting an interrupt after a terminal that does not await", async () => { + const { session } = await completedFirstTurn(false); + await expect(session.interrupt!({ reason: "Stop" })).rejects.toBeInstanceOf(HarnessOperationAlreadyTerminalError); + await session.close({ reason: "fixture complete" }); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts index 241d91e5663..02283ba02ce 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts @@ -43,6 +43,7 @@ export const RUNNERD_CANONICAL_ITEM = Symbol.for( ); export const RUNNERD_RUNTIME_ACTIVITY = Symbol("paperclip.runnerd.runtime-activity"); +export const RUNNERD_RUN_TERMINAL = Symbol("paperclip.runnerd.run-terminal"); export function text(value: unknown, fallback = ""): string { return typeof value === "string" ? value : fallback; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts index ca2de721d59..ac4cd9d581e 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -476,6 +476,19 @@ export class CodexHarnessSession ); return; } + if (this.awaitingProviderTurn && this.activeTurnId === null && !this.turnStartPending) { + this.emit( + "item.completed", + { + kind: "interrupt_acknowledgement", + text: "Paperclip stopped waiting for the provider's own next turn.", + status: "provider_turn_wait_ended", + reason: boundedText(input.reason), + }, + { itemId: `interrupt:provider-turn-wait:${++this.interruptSequence}` }, + ); + return; + } if (this.terminal || this.activeTurnId === null) { throw new HarnessOperationAlreadyTerminalError("interruption"); } @@ -836,7 +849,7 @@ export class CodexHarnessSession ["completed", "failed", "interrupted", "cancelled"].includes(status) ) { reconciledTerminalTurnId = expectedTurnId; - mapTerminalTurn(this, expectedTurn, expectedTurnId, true); + mapTerminalTurn(this, expectedTurn, expectedTurnId, true, expectedTurn.awaitingProviderTurn === true); } else { throw new HarnessReconciliationError( `persisted active turn ${expectedTurnId} has unreconcilable status ${status || "missing"}`, @@ -920,6 +933,7 @@ export class CodexHarnessSession this.currentGoal === null ? null : structuredClone(this.currentGoal), lineage: this.lineage(), lastSourceSequence: this.sourceSequence, + ...(this.awaitingProviderTurn ? { awaitingProviderTurn: true } : {}), }; } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts index 5a2ba36ad28..e33481395e9 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts @@ -30,6 +30,7 @@ import { recordWorkspaceChanges, } from "./codex-session-workspace.js"; import { + RUNNERD_RUN_TERMINAL, RUNNERD_RUNTIME_ACTIVITY, boundedText, differingJsonPaths, @@ -152,6 +153,19 @@ async function mapNotification(state: CodexSessionState, notification: CodexRpcN } async function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): Promise { + if (notification.method === "paperclip/runTerminal") { + if ((notification.params as Record)[RUNNERD_RUN_TERMINAL] !== true) return; + if (state.awaitingProviderTurn && state.activeTurnId === null && !state.turnStartPending) { + state.awaitingProviderTurn = false; + state.emit("item.completed", { + kind: "interrupt_acknowledgement", + text: "The Runner ended the wait for the provider's own next turn.", + status: "provider_turn_wait_ended", + cause: "run_terminal", + }, { itemId: `interrupt:provider-turn-wait:run-terminal:${state.sourceSequence + 1}` }); + } + return; + } if (notification.method === "paperclip/runtimeActivityRecorded") { const params = notification.params; if ((params as Record)[RUNNERD_RUNTIME_ACTIVITY] !== true) return; @@ -538,7 +552,8 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex state.recordHistoryEvent(notification.method, turnId); return; } - const autonomousGoalTurn = state.currentGoal?.status === "active" + const providerStartedTurn = params.origin === "provider" && state.awaitingProviderTurn; + const autonomousGoalTurn = (state.currentGoal?.status === "active" || providerStartedTurn) && state.activeTurnId === null && !state.turnStartPending && !state.protocolFailed @@ -569,9 +584,10 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex state.activeTurnId = turnId; state.turnStartPending = false; state.turnStarted = true; + state.awaitingProviderTurn = false; state.emit( "turn.started", - { status: text(turn.status, "inProgress") }, + { status: text(turn.status, "inProgress"), ...(providerStartedTurn ? { origin: "provider" } : {}) }, { turnId }, ); return; @@ -608,7 +624,7 @@ async function mapNotificationBody(state: CodexSessionState, notification: Codex } if (ignoreUnboundRestoredTurnLoss(state, turn, turnId)) return; if (!state.notificationNamesActiveTurn(turnId, "turn terminal")) return; - mapTerminalTurn(state, turn, turnId); + mapTerminalTurn(state, turn, turnId, false, params.awaitingProviderTurn === true); return; } if (notification.method === "item/started") { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts index 5314dd5ae66..3a7ca9270db 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -161,6 +161,7 @@ export class CodexSessionState { * stricter consumed marker. */ restartContinuationTurnId: string | null = null; + awaitingProviderTurn = false; turnStarted = false; readonly terminalTurns = new Map(); /** @@ -217,6 +218,7 @@ export class CodexSessionState { codexUsageBaseline?: CodexUsageBaseline; dispositionOnlyRecoveryConsumed?: boolean; dispositionOnlyRecoveryTurnId?: string | null; + awaitingProviderTurn?: boolean; /** A recovered turn whose turn.accepted the previous controller never emitted. */ adoptedAcceptedTurnId?: string | null; /** A turn the runner accepted for an earlier generation that the restore lost. */ @@ -309,6 +311,8 @@ export class CodexSessionState { if (this.activeTurnId && this.terminalTurns.has(this.activeTurnId)) { this.activeTurnId = null; } + this.awaitingProviderTurn = input.awaitingProviderTurn === true && this.activeTurnId === null + && this.result === null && this.terminalTurns.size > 0; const dispositionOnlyRecoveryPreviouslyConsumed = input.dispositionOnlyRecoveryConsumed ?? false; const dispositionOnlyRecoveryTurnId = diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts index f344b5af03b..736c0166349 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts @@ -153,6 +153,7 @@ export function mapTerminalTurn( turn: Record, fallbackTurnId: string, reconciling = false, + awaitingProviderTurn = false, ): void { const turnId = text(turn.id, fallbackTurnId || state.activeTurnId || ""); const reported = turn; @@ -214,9 +215,11 @@ export function mapTerminalTurn( state.cancelPendingRequests("turn_terminal"); state.activeTurnId = null; state.turnStarted = false; + state.awaitingProviderTurn = awaitingProviderTurn && eventType === "turn.completed" && state.result === null; const providerFailure = status === "failed" ? readProviderFailure(record(turn.error).providerFailure) ?? (state.driverKind === "codex_app_server" ? codexProviderFailure(turn.error, "turn", state.diagnosticEnvironment) : null) : null; - const payload = boundedPayload({ status, error: turn.error ?? null }); + const payload = boundedPayload({ status, error: turn.error ?? null, + ...(state.awaitingProviderTurn ? { awaitingProviderTurn: true } : {}) }); if (providerFailure) { payload.status = status; payload.error = { ...record(payload.error), providerFailure }; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index b983446b565..5169606853d 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1962,6 +1962,27 @@ it("settles a restored turn from the runner's reconciled result terminal", () => expect(recovered([runTerminal], "turn-1")).toBeNull(); }); +it("carries the provider-turn wait marker on a recovered turn terminal only when the PRP turn.completed has it", () => { + const { runnerdRecoveredTurnTerminal: recovered, runnerdPriorTurnTerminals: prior } = runnerdRecoveryInternals; + const event = (eventType: string, payload: Record) => ({ eventType, envelope: { runId: "run-1", payload: { payload } } }); + const awaited = event("turn.completed", { providerTurnId: "turn-1", awaitingProviderTurn: true }); + const plain = event("turn.completed", { providerTurnId: "turn-1" }); + const failed = event("turn.failed", { providerTurnId: "turn-1", awaitingProviderTurn: true }); + const forged = event("turn.completed", { providerTurnId: "turn-1", awaitingProviderTurn: "true" }); + expect(recovered([awaited], "turn-1")).toEqual({ status: "completed", error: null, awaitingProviderTurn: true }); + expect(recovered([plain], "turn-1")).toStrictEqual({ status: "completed", error: null }); + expect(recovered([failed], "turn-1")).toStrictEqual({ status: "failed", error: null }); + expect(recovered([forged], "turn-1")).toStrictEqual({ status: "completed", error: null }); + expect([...prior([awaited], "run-1", "turn-2").values()]) + .toEqual([{ id: "turn-1", status: "completed", error: null, awaitingProviderTurn: true }]); + for (const unmarked of [plain, failed, forged]) { + expect([...prior([unmarked], "run-1", "turn-2").values()]) + .toStrictEqual([{ id: "turn-1", status: unmarked.eventType.slice("turn.".length), error: null }]); + } + expect(prior([awaited], "run-1", "turn-1").size).toBe(0); + expect(prior([awaited], "run-other", "turn-2").size).toBe(0); +}); + it("requires ACPX command results to preserve the requested turn identity", () => { expect( runnerdRecoveryInternals.turnStartCommandResultValid({ diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 0fbecce161f..6fc75db84f2 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -886,8 +886,35 @@ function turnStartResponseReady(input: { type RunnerdRecoveredTurnTerminal = { status: "completed" | "failed" | "interrupted" | "cancelled"; error: unknown; + awaitingProviderTurn?: true; }; +function awaitingProviderTurnMarker(eventType: string, payload: Record): { awaitingProviderTurn?: true } { + return eventType === "turn.completed" && payload.awaitingProviderTurn === true ? { awaitingProviderTurn: true } : {}; +} + +function runnerdPriorTurnTerminals( + events: readonly Pick[], + runId: string | undefined, + boundTurnId: string, +): Map> { + const priorTerminals = new Map>(); + for (const event of events) { + if (event.envelope.runId !== runId || + !["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) continue; + const payload = record(record(event.envelope.payload).payload); + const turnId = payload.providerTurnId ?? payload.turnId ?? record(payload.turn).id; + if (typeof turnId !== "string" || !turnId || turnId === boundTurnId) continue; + priorTerminals.set(turnId, { + id: turnId, + status: event.eventType.slice("turn.".length), + error: payload.error ?? record(payload.turn).error ?? null, + ...awaitingProviderTurnMarker(event.eventType, payload), + }); + } + return priorTerminals; +} + const RUNNERD_TURN_TERMINAL_STATUSES = new Set(["completed", "failed", "interrupted", "cancelled"]); /** @@ -916,7 +943,8 @@ function runnerdRecoveredTurnTerminal( // Reconciliation must retain the cause, including the runner's explicit // process-loss marker, rather than inventing error:null. terminal = { status: event.eventType.slice("turn.".length) as RunnerdRecoveredTurnTerminal["status"], - error: payload.error ?? record(payload.turn).error ?? null }; + error: payload.error ?? record(payload.turn).error ?? null, + ...awaitingProviderTurnMarker(event.eventType, payload) }; reconciledEndedTurn = false; } else if (event.eventType === "run.terminal" && reconciledEndedTurn) { const status = String(payload.turnTerminalState); @@ -4652,19 +4680,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // Keep its prior terminal as the history anchor, so driver recovery can // adopt the later accepted turn instead of submitting it again. Items // remain lazy and fail closed if their start left the retained window. - const priorTerminals = new Map>(); - for (const event of this.#core?.store.state.committedEvents ?? []) { - if (event.envelope.runId !== this.#core?.store.state.identity.runId || - !["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) continue; - const payload = record(record(event.envelope.payload).payload); - const turnId = payload.providerTurnId ?? payload.turnId ?? record(payload.turn).id; - if (typeof turnId !== "string" || !turnId || turnId === this.#turnId) continue; - priorTerminals.set(turnId, { - id: turnId, - status: event.eventType.slice("turn.".length), - error: payload.error ?? record(payload.turn).error ?? null, - }); - } + const priorTerminals = runnerdPriorTurnTerminals( + this.#core?.store.state.committedEvents ?? [], this.#core?.store.state.identity.runId, this.#turnId); recoveredTurns.unshift(...priorTerminals.values()); this.#recoveryTurnBindingPending = false; this.#pumpEvents(); @@ -9786,6 +9803,7 @@ export const runnerdRecoveryInternals = Object.freeze({ rotatedRunAttachPayload, localRunnerStateArchive, rotateAuthorityEpoch, + runnerdPriorTurnTerminals, runnerdRecoveredTurnTerminal, runnerdTerminalProviderTurnId, terminalMayBelongToPendingTurnStart,