diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index e8dcb86f91d..d34e98fe922 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -1795,6 +1795,18 @@ fn run() -> Result<(), Box> { "tokenUsage": {"total": {"inputTokens": 100, "outputTokens": 20}, "last": {"inputTokens": 50, "outputTokens": 10}} }}))?; } + // A notification for a turn the runner never owned fails the + // accepted turn before it starts. + if args + .iter() + .any(|value| value == "--foreign-usage-before-turn-started") + { + send(json!({"method": "thread/tokenUsage/updated", "params": { + "threadId": state.thread_id, + "turnId": "provider-turn-foreign", + "tokenUsage": {"total": {"inputTokens": 100, "outputTokens": 20}, "last": {"inputTokens": 50, "outputTokens": 10}} + }}))?; + } // A loaded host can accept a turn long before it starts it. if let Some(delay) = argument(&args, "--turn-started-delay-ms") { thread::sleep(Duration::from_millis(delay.parse()?)); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts index aa67fc52484..b03eee9506e 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts @@ -80,6 +80,32 @@ it("does not adopt an accepted turn the runner no longer reports active", async } }, 60_000); +it("fails a pending turn start at once with the runner's own terminal failure", async () => { + // The runner can fail an accepted turn before its turn/started arrives + // (RC-20d: turn_binding_mismatch). That terminal is final, so the start must + // report it at once rather than wait out the bound under a generic message. + const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-failed-turn-start-")); + const callsPath = join(stateDirectory, "calls.log"); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(stateDirectory, "--foreign-usage-before-turn-started", + "--turn-started-delay-ms", "20000", "--call-log", callsPath), + stateDirectory, turnStartTimeoutMs: 15_000, + }); + try { + await bundle.transport.request("thread/start", { cwd: tmpdir(), dynamicTools: [] }); + const started = Date.now(); + await expect(bundle.transport.request("turn/start", { input: [{ type: "text", text: "Fail before start." }] })) + .rejects.toThrow("turn_binding_mismatch"); + expect(Date.now() - started).toBeLessThan(5_000); + const calls = (await readFile(callsPath, "utf8")).trim().split(/\r?\n/); + expect(calls.filter(call => call === "turn/start")).toHaveLength(1); + } finally { + await bundle.transport.close().catch(() => undefined); + await rm(stateDirectory, { recursive: true, force: true }); + } +}, 60_000); + it("keeps a quiet active Codex turn in the same process across connection lease expiry", async () => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-renew-active-")); const callsPath = join(stateDirectory, "calls.log"); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 9df1aba9e7a..d1c83fc9221 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -867,6 +867,34 @@ function terminalMayBelongToPendingTurnStart(input: { /** Part of the turn-start bound kept for the accepted-turn adoption check. */ const TURN_START_ADOPTION_CHECK_RESERVE_MS = 5_000; +/** + * The runner's own final failure of a turn whose start is still pending: a + * non-recoverable turn terminal held for that start, or a failed run terminal. + * Neither can be followed by a live start, so the start reports it at once. + */ +export function runnerdPendingTurnStartFailure( + events: ReadonlyArray<{ eventType: string; envelope: { payload?: unknown } }>, +): string | null { + for (const event of events) { + const payload = record(record(event.envelope.payload).payload); + const error = record(payload.error); + if ( + event.eventType === "turn.failed" && + (payload.recoverable === false || error.recoverable === false) + ) { + const code = typeof error.code === "string" ? error.code + : typeof payload.code === "string" ? payload.code : "provider_turn_failed"; + const message = typeof error.message === "string" ? error.message + : typeof payload.message === "string" ? payload.message : null; + return message ? `${code}: ${message}` : code; + } + if (event.eventType === "run.terminal" && payload.runTerminalState === "failed") { + return "run_terminal_failed"; + } + } + return null; +} + function turnStartNotificationDisposition(input: { responsePending: boolean; expectedProviderTurnId: string | null; @@ -7107,11 +7135,24 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { TURN_START_ADOPTION_CHECK_RESERVE_MS, Math.floor(Math.max(0, deadline - Date.now()) / 4), ); + // The runner can end the accepted turn before it starts. Its terminal is + // held for the pending start, so report that real cause at once. + const throwIfRunnerFailedStart = () => { + const failure = runnerdPendingTurnStartFailure([ + ...this.#deferredTurnStartEvents, + ...(this.#core?.store.state.committedEvents ?? []).filter( + (event) => event.sourceSeq > this.#eventSourceSeq, + ), + ]); + if (failure !== null) + throw new Error(`runnerd failed the provider turn before it started: ${failure}`); + }; const waitForStarted = async () => { while (!providerTurnStarted() && Date.now() < adoptionCheckAt) { this.#throwIfFailed(); this.#pumpEvents(); if (providerTurnStarted()) break; + throwIfRunnerFailedStart(); if (await this.#runnerHasExited()) throw new Error("runnerd exited before provider turn startup"); await new Promise((resolveWait) => setTimeout(resolveWait, 10)); @@ -7135,6 +7176,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // report a successful start on it. this.#pumpEventsSafely(); this.#throwIfFailed(); + if (!providerTurnStarted()) throwIfRunnerFailedStart(); if (!providerTurnStarted() && this.#acceptedTurnProvablyLive({ snapshot, expectedProviderTurnId, acceptingProviderPid, acceptingProviderStartedAt, acceptingCommandSeq, @@ -7151,6 +7193,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#throwIfFailed(); this.#pumpEvents(); if (providerTurnStarted()) break; + throwIfRunnerFailedStart(); if (await this.#runnerHasExited()) throw new Error("runnerd exited before provider turn startup"); await new Promise((resolveWait) => setTimeout(resolveWait, 10));